diff --git a/latest/.buildinfo b/latest/.buildinfo index 1f24c7367c..e1872b4a6c 100644 --- a/latest/.buildinfo +++ b/latest/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: a9c5c8c57021602368f541d74d22523d +config: 5b10b2153627779ea5be4dbb07d82396 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/latest/_cpp_gen/executor.html b/latest/_cpp_gen/executor.html index 182be9c38a..824e5f6543 100644 --- a/latest/_cpp_gen/executor.html +++ b/latest/_cpp_gen/executor.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -506,8 +514,8 @@

Executor#

-
-

transferAgent.h#

+
+

disaggServerUtil.h#

namespace tensorrt_llm#
@@ -515,6 +523,749 @@
namespace executor#
+
+namespace disagg_executor#
+
+
+class DisaggExecutorOrchestrator#
+
+

Public Functions

+
+
+DisaggExecutorOrchestrator( + +
+
std::vector<std::filesystem::path> const &ctxEnginePaths,
+
std::vector<std::filesystem::path> const &genEnginePaths,
+
std::vector<executor::ExecutorConfig> const &ctxExecutorConfigs,
+
std::vector<executor::ExecutorConfig> const &genExecutorConfigs,
+
bool hasContextAwaitThreads,
+
bool hasGenAwaitThreads,
+
+ +)#
+

Constructs a DisaggExecutorOrchestrator object.

+
+
Parameters:
+
    +

  • ctxEnginePaths – A vector of file paths to context engine files.

    • +

  • genEnginePaths – A vector of file paths to generation engine files.

    • +

  • ctxExecutorConfigs – A vector of ExecutorConfig for context executors.

    • +

  • genExecutorConfigs – A vector of ExecutorConfig for generation executors.

    • +

  • hasContextAwaitThreads – Whether or not there are threads that receive response for each generation executor.

    • +

  • hasGenAwaitThreads – Whether or not there are threads that receive response for each generation executor.

    • +
+
+
+
+ +
+
+std::vector<IdType> enqueueContext( + +
+
std::vector<texec::Request> const &requests,
+
std::optional<int> selectContextId = std::nullopt,
+
bool batch = false,
+
+ +)#
+

Enqueue context-only requests to context executors.

+
+
Parameters:
+
    +

  • requests – A vector of context-only requests.

    • +

  • selectContextId – The index of the context executor to use. If std::nullopt, the executor that has the smallest number of inflight requests will be used.

    • +

  • batch – If true,enqueue requests in same context executor.If false, will try to use a different executor for each request.

    • +
+
+
Returns:
+

A vector of global request ids, corresponding to the order of the requests in requests, the id returned may be different from the request id in each executor.

+
+
+
+ +
+
+void enqueueGeneration( + +
+
std::vector<texec::Request> const &requests,
+
std::vector<IdType> const &globalRequestIds,
+
std::optional<int> selectGenIdx = std::nullopt,
+
bool batch = false,
+
+ +)#
+

Enqueue generation-only requests to generation executors.

+
+
Parameters:
+
    +

  • requests – A vector of generation-only requests.

    • +

  • globalRequestIds – A vector of global request ids, corresponding to the order of the requests,and must be the ids returned by the enqueueContext function.

    • +

  • selectGenIdx – The index of the generation executor to use. If std::nullopt, the executor that has the smallest number of inflight requests will be used.

    • +

  • batch – If true,enqueue requests in same generation executor.If false, will try to use a different executor for each request.

    • +
+
+
+
+ +
+
+std::vector<ResponseWithId> awaitContextResponses( + +
+
std::optional<std::chrono::milliseconds> const &timeout,
+
std::optional<int> contextIdx = std::nullopt,
+
+ +)#
+

Await for context responses.

+
+
Parameters:
+
    +

  • timeout – The maximum time to wait for new responses

    • +

  • contextIdx – The index of the context executor to use. If std::nullopt, return ready responses in all context executors,if hasContextAwaitThreads is true, then this parameter must be std::nullopt.

    • +
+
+
Returns:
+

A vector of responses with corresponding global request ids

+
+
+
+ +
+
+std::vector<ResponseWithId> awaitGenerationResponses( + +
+
std::optional<std::chrono::milliseconds> const &timeout,
+
std::optional<int> genIdx = std::nullopt,
+
+ +)#
+

Await for generation responses.

+
+
Parameters:
+
    +

  • timeout – The maximum time to wait for new responses.

    • +

  • genIdx – The index of the generation executor to use. If std::nullopt, return ready responses in all generation executors,if hasGenAwaitThreads is true, then this parameter must be std::nullopt.

    • +
+
+
Returns:
+

A vector of responses with corresponding global request ids.

+
+
+
+ +
+
+bool canEnqueue() const#
+

Indicates if the current process is allowed to enqueueRequests.

+
+ +
+
+std::vector<std::unique_ptr<texec::Executor>> const &getContextExecutors( + +
+
+ +) const#
+

Get context executors.

+
+ +
+
+std::vector<std::unique_ptr<texec::Executor>> const &getGenExecutors( + +
+
+ +) const#
+

Get generation executors.

+
+ +
+
+~DisaggExecutorOrchestrator()#
+
+ +
+
+

Private Members

+
+
+std::unique_ptr<Impl> mImpl#
+
+ +
+
+ +
+
+struct ResponseWithId#
+
+

Public Functions

+
+
+inline ResponseWithId( + +
+
tensorrt_llm::executor::Response &&response,
+
IdType gid,
+
+ +)#
+
+ +
+
+inline ResponseWithId( + +
+
tensorrt_llm::executor::Response const &response,
+
IdType gid,
+
+ +)#
+
+ +
+
+inline ResponseWithId(ResponseWithId &&other) noexcept#
+
+ +
+
+ResponseWithId(ResponseWithId const &other) = default#
+
+ +
+
+inline ResponseWithId &operator=(ResponseWithId &&other) noexcept#
+
+ +
+
+inline ResponseWithId &operator=(ResponseWithId const &other)#
+
+ +
+
+~ResponseWithId() = default#
+
+ +
+
+

Public Members

+
+
+tensorrt_llm::executor::Response response#
+
+ +
+
+IdType gid#
+
+ +
+
+ +
+ +
+ + + +
+
+

tensor.h#

+
+
+namespace tensorrt_llm
+
+
+namespace executor
+
+
+class Shape : public tensorrt_llm::common::ArrayView<detail::DimType64 const>#
+
+

Public Types

+
+
+using Base = tensorrt_llm::common::ArrayView<detail::DimType64 const>#
+
+ +
+
+using DimType64 = typename std::remove_cv_t<Base::value_type>#
+
+ +
+
+

Public Functions

+
+
+inline Shape()#
+
+ +
+
+inline Shape(DimType64 const *data, Base::size_type size)#
+
+ +
+
+inline Shape(std::initializer_list<DimType64> dims)#
+
+ +
+
+ +
+
+class Tensor#
+
+

Public Types

+
+
+using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>#
+
+ +
+
+

Public Functions

+
+
+Tensor copyToCpu(Tensor::CudaStreamPtr stream = nullptr) const#
+
+ +
+
+Tensor copyToPinned(Tensor::CudaStreamPtr stream = nullptr) const#
+
+ +
+
+Tensor copyToPooledPinned(Tensor::CudaStreamPtr stream = nullptr) const#
+
+ +
+
+Tensor copyToManaged(Tensor::CudaStreamPtr stream = nullptr) const#
+
+ +
+
+Tensor copyToGpu(Tensor::CudaStreamPtr stream) const#
+
+ +
+
+Tensor() noexcept = default#
+
+ +
+
+~Tensor() = default#
+
+ +
+
+Tensor(Tensor const &other) noexcept = default#
+
+ +
+
+Tensor(Tensor &&other) noexcept = default#
+
+ +
+
+Tensor &operator=(Tensor const &other) noexcept = default#
+
+ +
+
+Tensor &operator=(Tensor &&other) noexcept = default#
+
+ +
+
+void *getData()#
+

Returns a pointer to underlying array.

+
+ +
+
+void const *getData() const#
+

Returns a pointer to underlying array.

+
+ +
+
+DataType getDataType() const#
+

Returns the data type of the buffer.

+
+ +
+
+MemoryType getMemoryType() const#
+

Returns the memory type of the buffer.

+
+ +
+
+Shape getShape() const#
+

Returns the tensor dimensions.

+
+ +
+
+std::size_t getSize() const#
+

Returns the number of elements in the tensor.

+
+ +
+
+std::size_t getSizeInBytes() const#
+

Returns the size of the tensor in bytes.

+
+ +
+
+void setZero(CudaStreamPtr stream = nullptr)#
+

Set the entire memory to zero.

+
+
Parameters:
+

stream – Must be a valid CUDA stream if the memory type is GPU.

+
+
+
+ +
+
+void setFrom(Tensor const &other, CudaStreamPtr stream = nullptr)#
+

Copy the data and shape from another tensor.

+
+
Parameters:
+
    +

  • other – A tensor to copy from.

    • +

  • stream – Must be a valid CUDA stream if the memory type is GPU.

    • +
+
+
+
+ +
+
+inline explicit operator bool() const#
+
+ +
+
+inline bool operator==(Tensor const &rhs) const#
+
+ +
+
+inline bool operator!=(Tensor const &rhs) const#
+
+ +
+
+

Public Static Functions

+
+
+static Tensor cpu(DataType dataType, Shape shape = {})#
+

Allocate a cpu tensor with the given shape and data type.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • dataType – The data type of the tensor.

    • +
+
+
+
+ +
+
+template<typename T>
static inline Tensor cpu(Shape shape = {})#
+
+ +
+
+static Tensor pinned(DataType dataType, Shape shape = {})#
+

Allocate a cpu tensor in pinned memory with the given shape and data type.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • dataType – The data type of the tensor.

    • +
+
+
+
+ +
+
+template<typename T>
static inline Tensor pinned(Shape shape = {})#
+
+ +
+
+static Tensor pooledPinned(DataType dataType, Shape shape = {})#
+

Allocate a cpu tensor in pooled pinned memory with the given shape and data type.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • dataType – The data type of the tensor.

    • +
+
+
+
+ +
+
+template<typename T>
static inline Tensor pooledPinned( + +
+
Shape shape = {},
+
+ +)#
+
+ +
+
+static Tensor managed(DataType dataType, Shape shape = {})#
+

Allocate a tensor in managed memory (UVM) with the given shape and data type.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • dataType – The data type of the tensor.

    • +
+
+
+
+ +
+
+template<typename T>
static inline Tensor managed(Shape shape = {})#
+
+ +
+
+static Tensor gpu( + +
+
DataType dataType,
+
CudaStreamPtr stream,
+
Shape shape = {},
+
+ +)#
+

Allocate a gpu tensor with the given shape and data type on a particular cuda stream.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • +

  • dataType – The data type of the tensor.

    • +
+
+
+
+ +
+
+template<typename T>
static inline Tensor gpu( + +
+
CudaStreamPtr stream,
+
Shape shape = {},
+
+ +)#
+
+ +
+
+static Tensor of(DataType dataType, void *data, Shape shape)#
+

Wrap a data pointer into a tensor without taking ownership.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • dataType – The data type of the tensor.

    • +

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • +
+
+
+
+ +
+
+template<typename T>
static inline Tensor of(T *data, Shape shape)#
+

Wrap a data pointer into a tensor without taking ownership.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • dataType – The data type of the tensor.

    • +

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • +
+
+
+
+ +
+
+template<typename T>
static inline Tensor of(T &data)#
+

Wrap any container into a tensor without taking ownership.

+
+
Parameters:
+
    +

  • shape – The shape of the tensor.

    • +

  • dataType – The data type of the tensor.

    • +

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • +
+
+
+
+ +
+
+

Private Types

+
+
+using Impl = runtime::ITensor#
+
+ +
+
+

Private Functions

+
+
+explicit Tensor(std::shared_ptr<runtime::ITensor> tensor)#
+
+ +
+
+Tensor copyTo( + +
+
std::shared_ptr<Impl> tensor,
+
CudaStreamPtr stream,
+
+ +) const#
+
+ +
+
+

Private Members

+
+
+std::shared_ptr<Impl> mTensor#
+
+ +
+
+

Private Static Functions

+
+
+template<typename T>
static inline DataType getRuntimeType()#
+
+ +
+
+

Friends

+
+
+friend class Serialization
+
+ +
+
+friend std::shared_ptr<runtime::ITensor> const &toITensor( + +
+
Tensor const &tensor,
+
+ +)#
+
+ +
+
+friend Tensor ofITensor( + +
+
std::shared_ptr<runtime::ITensor> tensor,
+
+ +)#
+
+ +
+
+ +
+
+namespace detail#
+
+

Typedefs

+
+
+using DimType64 = int64_t#
+
+ +
+
+

Functions

+
+
+std::shared_ptr<runtime::ITensor> const &toITensor( + +
+
Tensor const &tensor,
+
+ +)#
+
+ +
+
+Tensor ofITensor(std::shared_ptr<runtime::ITensor> tensor)#
+
+ +
+
+ +
+ +
+
+namespace runtime#
+
+ +
+ +
+
+

transferAgent.h#

+
+
+namespace tensorrt_llm
+
+
+namespace executor
+
namespace kv_cache#
@@ -1281,6 +2032,1741 @@
+
+
+

serialization.h#

+
+
+namespace tensorrt_llm
+
+
+namespace executor
+
+
+class Serialization#
+
+

Public Static Functions

+
+
+static size_t serializedSize( + +
+
tensorrt_llm::batch_manager::kv_cache_manager::BlockKey const &key,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
tensorrt_llm::batch_manager::kv_cache_manager::BlockKey const &key,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static tensorrt_llm::batch_manager::kv_cache_manager::BlockKey deserializeBlockKey( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static RequestPerfMetrics::TimePoint deserializeTimePoint( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
RequestPerfMetrics::TimePoint const &tp,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(RequestPerfMetrics::TimePoint const&)#
+
+ +
+
+static RequestPerfMetrics deserializeRequestPerfMetrics( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
RequestPerfMetrics const &metrics,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(RequestPerfMetrics const &metrics)#
+
+ +
+
+static SamplingConfig deserializeSamplingConfig(std::istream &is)#
+
+ +
+
+static void serialize(SamplingConfig const &config, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(SamplingConfig const &config)#
+
+ +
+
+static OutputConfig deserializeOutputConfig(std::istream &is)#
+
+ +
+
+static void serialize(OutputConfig const &config, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(OutputConfig const &config)#
+
+ +
+
+static AdditionalModelOutput deserializeAdditionalModelOutput( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
AdditionalModelOutput const &additionalModelOutput,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
AdditionalModelOutput const &additionalModelOutput,
+
+ +)#
+
+ +
+
+static ExternalDraftTokensConfig deserializeExternalDraftTokensConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
ExternalDraftTokensConfig const &config,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(ExternalDraftTokensConfig const &config)#
+
+ +
+
+static PromptTuningConfig deserializePromptTuningConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
PromptTuningConfig const &config,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(PromptTuningConfig const &config)#
+
+ +
+
+static MultimodalInput deserializeMultimodalInput(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
MultimodalInput const &multimodalInput,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(MultimodalInput const &multimodalInput)#
+
+ +
+
+static MropeConfig deserializeMropeConfig(std::istream &is)#
+
+ +
+
+static void serialize(MropeConfig const &config, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(MropeConfig const &config)#
+
+ +
+
+static LoraConfig deserializeLoraConfig(std::istream &is)#
+
+ +
+
+static void serialize(LoraConfig const &config, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(LoraConfig const &config)#
+
+ +
+
+static kv_cache::CommState deserializeCommState(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
kv_cache::CommState const &state,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(kv_cache::CommState const &state)#
+
+ +
+
+static kv_cache::SocketState deserializeSocketState(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
kv_cache::SocketState const &state,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(kv_cache::SocketState const &state)#
+
+ +
+
+static kv_cache::AgentState deserializeAgentState(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
kv_cache::AgentState const &state,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(kv_cache::AgentState const &state)#
+
+ +
+
+static kv_cache::CacheState deserializeCacheState(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
kv_cache::CacheState const &state,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(kv_cache::CacheState const &state)#
+
+ +
+
+static DataTransceiverState deserializeDataTransceiverState( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static DataTransceiverState deserializeDataTransceiverState( + +
+
std::vector<char> &buffer,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
DataTransceiverState const &dataTransceiverState,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static std::vector<char> serialize( + +
+
DataTransceiverState const &dataTransceiverState,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
DataTransceiverState const &dataTransceiverState,
+
+ +)#
+
+ +
+
+static ContextPhaseParams deserializeContextPhaseParams( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
ContextPhaseParams const &contextPhaseParams,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
ContextPhaseParams const &contextPhaseParams,
+
+ +)#
+
+ +
+
+static Request deserializeRequest(std::istream &is)#
+
+ +
+
+static void serialize(Request const &request, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(Request const &request)#
+
+ +
+
+static Tensor deserializeTensor(std::istream &is)#
+
+ +
+
+static void serialize(Tensor const &tensor, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(Tensor const &tensor)#
+
+ +
+
+static SpeculativeDecodingFastLogitsInfo deserializeSpecDecFastLogitsInfo( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
SpeculativeDecodingFastLogitsInfo const &info,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
SpeculativeDecodingFastLogitsInfo const &info,
+
+ +)#
+
+ +
+
+static Result deserializeResult(std::istream &is)#
+
+ +
+
+static void serialize(Result const &result, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(Result const &result)#
+
+ +
+
+static AdditionalOutput deserializeAdditionalOutput(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
AdditionalOutput const &additionalOutput,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
AdditionalOutput const &additionalOutput,
+
+ +)#
+
+ +
+
+static Response deserializeResponse(std::istream &is)#
+
+ +
+
+static void serialize(Response const &response, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(Response const &response)#
+
+ +
+
+static std::vector<Response> deserializeResponses( + +
+
std::vector<char> &buffer,
+
+ +)#
+
+ +
+
+static std::vector<char> serialize( + +
+
std::vector<Response> const &responses,
+
+ +)#
+
+ +
+
+static KvCacheConfig deserializeKvCacheConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
KvCacheConfig const &kvCacheConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(KvCacheConfig const &kvCacheConfig)#
+
+ +
+
+static DynamicBatchConfig deserializeDynamicBatchConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
DynamicBatchConfig const &dynamicBatchConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
DynamicBatchConfig const &dynamicBatchConfig,
+
+ +)#
+
+ +
+
+static SchedulerConfig deserializeSchedulerConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
SchedulerConfig const &schedulerConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(SchedulerConfig const &schedulerConfig)#
+
+ +
+
+static ExtendedRuntimePerfKnobConfig deserializeExtendedRuntimePerfKnobConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig,
+
+ +)#
+
+ +
+
+static ParallelConfig deserializeParallelConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
ParallelConfig const &parallelConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(ParallelConfig const &parallelConfig)#
+
+ +
+
+static PeftCacheConfig deserializePeftCacheConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
PeftCacheConfig const &peftCacheConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(PeftCacheConfig const &peftCacheConfig)#
+
+ +
+
+static OrchestratorConfig deserializeOrchestratorConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
OrchestratorConfig const &orchestratorConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
OrchestratorConfig const &orchestratorConfig,
+
+ +)#
+
+ +
+
+static DecodingMode deserializeDecodingMode(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
DecodingMode const &decodingMode,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(DecodingMode const &decodingMode)#
+
+ +
+
+static LookaheadDecodingConfig deserializeLookaheadDecodingConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
LookaheadDecodingConfig const &lookaheadDecodingConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
LookaheadDecodingConfig const &lookaheadDecodingConfig,
+
+ +)#
+
+ +
+
+static EagleConfig deserializeEagleConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
EagleConfig const &eagleConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(EagleConfig const &eagleConfig)#
+
+ +
+
+static SpeculativeDecodingConfig deserializeSpeculativeDecodingConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
SpeculativeDecodingConfig const &specDecConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
SpeculativeDecodingConfig const &specDecConfig,
+
+ +)#
+
+ +
+
+static GuidedDecodingConfig deserializeGuidedDecodingConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
GuidedDecodingConfig const &guidedDecodingConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
GuidedDecodingConfig const &guidedDecodingConfig,
+
+ +)#
+
+ +
+
+static GuidedDecodingParams deserializeGuidedDecodingParams( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
GuidedDecodingParams const &guidedDecodingParams,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
GuidedDecodingParams const &guidedDecodingParams,
+
+ +)#
+
+ +
+
+static KvCacheRetentionConfig deserializeKvCacheRetentionConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
KvCacheRetentionConfig const &kvCacheRetentionConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
KvCacheRetentionConfig const &kvCacheRetentionConfig,
+
+ +)#
+
+ +
+
+static KvCacheRetentionConfig::TokenRangeRetentionConfig deserializeTokenRangeRetentionConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig,
+
+ +)#
+
+ +
+
+static DecodingConfig deserializeDecodingConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
DecodingConfig const &decodingConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(DecodingConfig const &decodingConfig)#
+
+ +
+
+static DebugConfig deserializeDebugConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
DebugConfig const &debugConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(DebugConfig const &debugConfig)#
+
+ +
+
+static CacheTransceiverConfig deserializeCacheTransceiverConfig( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
CacheTransceiverConfig const &cacheTransceiverConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
CacheTransceiverConfig const &cacheTransceiverConfig,
+
+ +)#
+
+ +
+
+static ExecutorConfig deserializeExecutorConfig(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
ExecutorConfig const &executorConfig,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(ExecutorConfig const &executorConfig)#
+
+ +
+
+static KvCacheStats deserializeKvCacheStats(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
KvCacheStats const &kvCacheStats,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(KvCacheStats const &kvCacheStats)#
+
+ +
+
+static StaticBatchingStats deserializeStaticBatchingStats( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
StaticBatchingStats const &staticBatchingStats,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
StaticBatchingStats const &staticBatchingStats,
+
+ +)#
+
+ +
+
+static InflightBatchingStats deserializeInflightBatchingStats( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
InflightBatchingStats const &inflightBatchingStats,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
InflightBatchingStats const &inflightBatchingStats,
+
+ +)#
+
+ +
+
+static SpecDecodingStats deserializeSpecDecodingStats( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
SpecDecodingStats const &specDecodingStats,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
SpecDecodingStats const &specDecodingStats,
+
+ +)#
+
+ +
+
+static IterationStats deserializeIterationStats( + +
+
std::vector<char> &buffer,
+
+ +)#
+
+ +
+
+static IterationStats deserializeIterationStats(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
IterationStats const &iterStats,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static std::vector<char> serialize(IterationStats const &iterStats)#
+
+ +
+
+static size_t serializedSize(IterationStats const &iterStats)#
+
+ +
+
+static std::vector<char> serialize( + +
+
std::vector<IterationStats> const &iterStatsVec,
+
+ +)#
+
+ +
+
+static std::vector<IterationStats> deserializeIterationStatsVec( + +
+
std::vector<char> &buffer,
+
+ +)#
+
+ +
+
+static DisServingRequestStats deserializeDisServingRequestStats( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
DisServingRequestStats const &stats,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
DisServingRequestStats const &disServingRequestStats,
+
+ +)#
+
+ +
+
+static RequestStage deserializeRequestStage(std::istream &is)#
+
+ +
+
+static void serialize( + +
+
RequestStage const &requestStage,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static size_t serializedSize(RequestStage const &requestStage)#
+
+ +
+
+static RequestStats deserializeRequestStats(std::istream &is)#
+
+ +
+
+static void serialize(RequestStats const &state, std::ostream &os)#
+
+ +
+
+static size_t serializedSize(RequestStats const &state)#
+
+ +
+
+static RequestStatsPerIteration deserializeRequestStatsPerIteration( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static RequestStatsPerIteration deserializeRequestStatsPerIteration( + +
+
std::vector<char> &buffer,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
RequestStatsPerIteration const &state,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static std::vector<char> serialize( + +
+
RequestStatsPerIteration const &state,
+
+ +)#
+
+ +
+
+static size_t serializedSize(RequestStatsPerIteration const &state)#
+
+ +
+
+static std::vector<char> serialize( + +
+
std::vector<RequestStatsPerIteration> const &requestStatsVec,
+
+ +)#
+
+ +
+
+static std::vector<RequestStatsPerIteration> deserializeRequestStatsPerIterationVec( + +
+
std::vector<char> &buffer,
+
+ +)#
+
+ +
+
+static std::vector<char> serialize( + +
+
std::deque<KVCacheEvent> const &kvCacheEvents,
+
+ +)#
+
+ +
+
+static std::deque<KVCacheEvent> deserializeKVCacheEvents( + +
+
std::vector<char> &buffer,
+
+ +)#
+
+ +
+
+static size_t serializedSize(KVCacheEvent const &event)#
+
+ +
+
+static void serialize(KVCacheEvent const &event, std::ostream &os)#
+
+ +
+
+static KVCacheEvent deserializeKVCacheEvent(std::istream &is)#
+
+ +
+
+static size_t serializedSize(KVCacheCreatedData const &data)#
+
+ +
+
+static void serialize( + +
+
KVCacheCreatedData const &data,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static KVCacheCreatedData deserializeKVCacheCreatedData( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static size_t serializedSize(KVCacheStoredData const &data)#
+
+ +
+
+static void serialize( + +
+
KVCacheStoredData const &data,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static KVCacheStoredData deserializeKVCacheStoredData( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static size_t serializedSize(KVCacheStoredBlockData const &data)#
+
+ +
+
+static void serialize( + +
+
KVCacheStoredBlockData const &data,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static KVCacheStoredBlockData deserializeKVCacheStoredBlockData( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static size_t serializedSize(KVCacheRemovedData const &data)#
+
+ +
+
+static void serialize( + +
+
KVCacheRemovedData const &data,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static KVCacheRemovedData deserializeKVCacheRemovedData( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+template<typename T>
static size_t serializedSize( + +
+
KVCacheEventDiff<T> const &data,
+
+ +)#
+
+ +
+
+template<typename T>
static void serialize( + +
+
KVCacheEventDiff<T> const &data,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+template<typename T>
static KVCacheEventDiff<T> deserializeKVCacheEventDiff( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static size_t serializedSize(KVCacheUpdatedData const &data)#
+
+ +
+
+static void serialize( + +
+
KVCacheUpdatedData const &data,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static KVCacheUpdatedData deserializeKVCacheUpdatedData( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static size_t serializedSize( + +
+
tensorrt_llm::runtime::UniqueToken const &token,
+
+ +)#
+
+ +
+
+static void serialize( + +
+
tensorrt_llm::runtime::UniqueToken const &token,
+
std::ostream &os,
+
+ +)#
+
+ +
+
+static tensorrt_llm::runtime::UniqueToken deserializeUniqueToken( + +
+
std::istream &is,
+
+ +)#
+
+ +
+
+static std::string deserializeString(std::istream &is)#
+
+ +
+
+static bool deserializeBool(std::istream &is)#
+
+ +
+
+static ModelType deserializeModelType(std::istream &is)#
+
+ +
+
+ +
+
+namespace kv_cache
+
+ +
+ +
+

types.h#

@@ -3029,879 +5515,6 @@ -
-
-namespace runtime#
-
- - - -
-
-

cacheCommunicator.h#

-
-
-namespace tensorrt_llm
-
-
-namespace executor
-
-
-namespace kv_cache
-
-
-class Connection#
-
-

Public Functions

-
-
-virtual ~Connection() = default#
-
- -
-
-virtual void send( - -
-
DataContext const &ctx,
-
void const *data,
-
size_t size,
-
- -) const = 0#
-
- -
-
-virtual void recv( - -
-
DataContext const &ctx,
-
void *data,
-
size_t size,
-
- -) const = 0#
-
- -
-
-inline virtual bool isThreadSafe() const noexcept#
-
- -
-
- -
-
-class ConnectionManager#
-
-

Public Functions

-
-
-virtual ~ConnectionManager() = default#
-
- -
-
-virtual Connection const *recvConnect( - -
-
DataContext const &ctx,
-
void *data,
-
size_t size,
-
- -) = 0#
-
- -
-
-virtual std::vector<Connection const*> getConnections( - -
-
CommState const &state,
-
- -) = 0#
-
- -
-
-virtual CommState const &getCommState() const = 0#
-
- -
-
- -
-
-struct DataContext#
-
-

Public Functions

-
-
-inline explicit DataContext(int tag)#
-
- -
-
-inline int getTag() const noexcept#
-
- -
-
-

Private Members

-
-
-int const mTag#
-
- -
-
- -
- -
- -
- -
-
-

disaggServerUtil.h#

-
-
-namespace tensorrt_llm
-
-
-namespace executor
-
-
-namespace disagg_executor#
-
-
-class DisaggExecutorOrchestrator#
-
-

Public Functions

-
-
-DisaggExecutorOrchestrator( - -
-
std::vector<std::filesystem::path> const &ctxEnginePaths,
-
std::vector<std::filesystem::path> const &genEnginePaths,
-
std::vector<executor::ExecutorConfig> const &ctxExecutorConfigs,
-
std::vector<executor::ExecutorConfig> const &genExecutorConfigs,
-
bool hasContextAwaitThreads,
-
bool hasGenAwaitThreads,
-
- -)#
-

Constructs a DisaggExecutorOrchestrator object.

-
-
Parameters:
-
    -

  • ctxEnginePaths – A vector of file paths to context engine files.

    • -

  • genEnginePaths – A vector of file paths to generation engine files.

    • -

  • ctxExecutorConfigs – A vector of ExecutorConfig for context executors.

    • -

  • genExecutorConfigs – A vector of ExecutorConfig for generation executors.

    • -

  • hasContextAwaitThreads – Whether or not there are threads that receive response for each generation executor.

    • -

  • hasGenAwaitThreads – Whether or not there are threads that receive response for each generation executor.

    • -
-
-
-
- -
-
-std::vector<IdType> enqueueContext( - -
-
std::vector<texec::Request> const &requests,
-
std::optional<int> selectContextId = std::nullopt,
-
bool batch = false,
-
- -)#
-

Enqueue context-only requests to context executors.

-
-
Parameters:
-
    -

  • requests – A vector of context-only requests.

    • -

  • selectContextId – The index of the context executor to use. If std::nullopt, the executor that has the smallest number of inflight requests will be used.

    • -

  • batch – If true,enqueue requests in same context executor.If false, will try to use a different executor for each request.

    • -
-
-
Returns:
-

A vector of global request ids, corresponding to the order of the requests in requests, the id returned may be different from the request id in each executor.

-
-
-
- -
-
-void enqueueGeneration( - -
-
std::vector<texec::Request> const &requests,
-
std::vector<IdType> const &globalRequestIds,
-
std::optional<int> selectGenIdx = std::nullopt,
-
bool batch = false,
-
- -)#
-

Enqueue generation-only requests to generation executors.

-
-
Parameters:
-
    -

  • requests – A vector of generation-only requests.

    • -

  • globalRequestIds – A vector of global request ids, corresponding to the order of the requests,and must be the ids returned by the enqueueContext function.

    • -

  • selectGenIdx – The index of the generation executor to use. If std::nullopt, the executor that has the smallest number of inflight requests will be used.

    • -

  • batch – If true,enqueue requests in same generation executor.If false, will try to use a different executor for each request.

    • -
-
-
-
- -
-
-std::vector<ResponseWithId> awaitContextResponses( - -
-
std::optional<std::chrono::milliseconds> const &timeout,
-
std::optional<int> contextIdx = std::nullopt,
-
- -)#
-

Await for context responses.

-
-
Parameters:
-
    -

  • timeout – The maximum time to wait for new responses

    • -

  • contextIdx – The index of the context executor to use. If std::nullopt, return ready responses in all context executors,if hasContextAwaitThreads is true, then this parameter must be std::nullopt.

    • -
-
-
Returns:
-

A vector of responses with corresponding global request ids

-
-
-
- -
-
-std::vector<ResponseWithId> awaitGenerationResponses( - -
-
std::optional<std::chrono::milliseconds> const &timeout,
-
std::optional<int> genIdx = std::nullopt,
-
- -)#
-

Await for generation responses.

-
-
Parameters:
-
    -

  • timeout – The maximum time to wait for new responses.

    • -

  • genIdx – The index of the generation executor to use. If std::nullopt, return ready responses in all generation executors,if hasGenAwaitThreads is true, then this parameter must be std::nullopt.

    • -
-
-
Returns:
-

A vector of responses with corresponding global request ids.

-
-
-
- -
-
-bool canEnqueue() const#
-

Indicates if the current process is allowed to enqueueRequests.

-
- -
-
-std::vector<std::unique_ptr<texec::Executor>> const &getContextExecutors( - -
-
- -) const#
-

Get context executors.

-
- -
-
-std::vector<std::unique_ptr<texec::Executor>> const &getGenExecutors( - -
-
- -) const#
-

Get generation executors.

-
- -
-
-~DisaggExecutorOrchestrator()#
-
- -
-
-

Private Members

-
-
-std::unique_ptr<Impl> mImpl#
-
- -
-
- -
-
-struct ResponseWithId#
-
-

Public Functions

-
-
-inline ResponseWithId( - -
-
tensorrt_llm::executor::Response &&response,
-
IdType gid,
-
- -)#
-
- -
-
-inline ResponseWithId( - -
-
tensorrt_llm::executor::Response const &response,
-
IdType gid,
-
- -)#
-
- -
-
-inline ResponseWithId(ResponseWithId &&other) noexcept#
-
- -
-
-ResponseWithId(ResponseWithId const &other) = default#
-
- -
-
-inline ResponseWithId &operator=(ResponseWithId &&other) noexcept#
-
- -
-
-inline ResponseWithId &operator=(ResponseWithId const &other)#
-
- -
-
-~ResponseWithId() = default#
-
- -
-
-

Public Members

-
-
-tensorrt_llm::executor::Response response#
-
- -
-
-IdType gid#
-
- -
-
- -
- -
- -
- -
-
-

tensor.h#

-
-
-namespace tensorrt_llm
-
-
-namespace executor
-
-
-class Shape : public tensorrt_llm::common::ArrayView<detail::DimType64 const>#
-
-

Public Types

-
-
-using Base = tensorrt_llm::common::ArrayView<detail::DimType64 const>#
-
- -
-
-using DimType64 = typename std::remove_cv_t<Base::value_type>#
-
- -
-
-

Public Functions

-
-
-inline Shape()#
-
- -
-
-inline Shape(DimType64 const *data, Base::size_type size)#
-
- -
-
-inline Shape(std::initializer_list<DimType64> dims)#
-
- -
-
- -
-
-class Tensor#
-
-

Public Types

-
-
-using CudaStreamPtr = std::shared_ptr<runtime::CudaStream>#
-
- -
-
-

Public Functions

-
-
-Tensor copyToCpu(Tensor::CudaStreamPtr stream = nullptr) const#
-
- -
-
-Tensor copyToPinned(Tensor::CudaStreamPtr stream = nullptr) const#
-
- -
-
-Tensor copyToPooledPinned(Tensor::CudaStreamPtr stream = nullptr) const#
-
- -
-
-Tensor copyToManaged(Tensor::CudaStreamPtr stream = nullptr) const#
-
- -
-
-Tensor copyToGpu(Tensor::CudaStreamPtr stream) const#
-
- -
-
-Tensor() noexcept = default#
-
- -
-
-~Tensor() = default#
-
- -
-
-Tensor(Tensor const &other) noexcept = default#
-
- -
-
-Tensor(Tensor &&other) noexcept = default#
-
- -
-
-Tensor &operator=(Tensor const &other) noexcept = default#
-
- -
-
-Tensor &operator=(Tensor &&other) noexcept = default#
-
- -
-
-void *getData()#
-

Returns a pointer to underlying array.

-
- -
-
-void const *getData() const#
-

Returns a pointer to underlying array.

-
- -
-
-DataType getDataType() const#
-

Returns the data type of the buffer.

-
- -
-
-MemoryType getMemoryType() const#
-

Returns the memory type of the buffer.

-
- -
-
-Shape getShape() const#
-

Returns the tensor dimensions.

-
- -
-
-std::size_t getSize() const#
-

Returns the number of elements in the tensor.

-
- -
-
-std::size_t getSizeInBytes() const#
-

Returns the size of the tensor in bytes.

-
- -
-
-void setZero(CudaStreamPtr stream = nullptr)#
-

Set the entire memory to zero.

-
-
Parameters:
-

stream – Must be a valid CUDA stream if the memory type is GPU.

-
-
-
- -
-
-void setFrom(Tensor const &other, CudaStreamPtr stream = nullptr)#
-

Copy the data and shape from another tensor.

-
-
Parameters:
-
    -

  • other – A tensor to copy from.

    • -

  • stream – Must be a valid CUDA stream if the memory type is GPU.

    • -
-
-
-
- -
-
-inline explicit operator bool() const#
-
- -
-
-inline bool operator==(Tensor const &rhs) const#
-
- -
-
-inline bool operator!=(Tensor const &rhs) const#
-
- -
-
-

Public Static Functions

-
-
-static Tensor cpu(DataType dataType, Shape shape = {})#
-

Allocate a cpu tensor with the given shape and data type.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • dataType – The data type of the tensor.

    • -
-
-
-
- -
-
-template<typename T>
static inline Tensor cpu(Shape shape = {})#
-
- -
-
-static Tensor pinned(DataType dataType, Shape shape = {})#
-

Allocate a cpu tensor in pinned memory with the given shape and data type.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • dataType – The data type of the tensor.

    • -
-
-
-
- -
-
-template<typename T>
static inline Tensor pinned(Shape shape = {})#
-
- -
-
-static Tensor pooledPinned(DataType dataType, Shape shape = {})#
-

Allocate a cpu tensor in pooled pinned memory with the given shape and data type.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • dataType – The data type of the tensor.

    • -
-
-
-
- -
-
-template<typename T>
static inline Tensor pooledPinned( - -
-
Shape shape = {},
-
- -)#
-
- -
-
-static Tensor managed(DataType dataType, Shape shape = {})#
-

Allocate a tensor in managed memory (UVM) with the given shape and data type.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • dataType – The data type of the tensor.

    • -
-
-
-
- -
-
-template<typename T>
static inline Tensor managed(Shape shape = {})#
-
- -
-
-static Tensor gpu( - -
-
DataType dataType,
-
CudaStreamPtr stream,
-
Shape shape = {},
-
- -)#
-

Allocate a gpu tensor with the given shape and data type on a particular cuda stream.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • -

  • dataType – The data type of the tensor.

    • -
-
-
-
- -
-
-template<typename T>
static inline Tensor gpu( - -
-
CudaStreamPtr stream,
-
Shape shape = {},
-
- -)#
-
- -
-
-static Tensor of(DataType dataType, void *data, Shape shape)#
-

Wrap a data pointer into a tensor without taking ownership.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • dataType – The data type of the tensor.

    • -

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • -
-
-
-
- -
-
-template<typename T>
static inline Tensor of(T *data, Shape shape)#
-

Wrap a data pointer into a tensor without taking ownership.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • dataType – The data type of the tensor.

    • -

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • -
-
-
-
- -
-
-template<typename T>
static inline Tensor of(T &data)#
-

Wrap any container into a tensor without taking ownership.

-
-
Parameters:
-
    -

  • shape – The shape of the tensor.

    • -

  • dataType – The data type of the tensor.

    • -

  • stream – Specifies the CUDA stream on which to allocate the tensor for GPU memory.

    • -
-
-
-
- -
-
-

Private Types

-
-
-using Impl = runtime::ITensor#
-
- -
-
-

Private Functions

-
-
-explicit Tensor(std::shared_ptr<runtime::ITensor> tensor)#
-
- -
-
-Tensor copyTo( - -
-
std::shared_ptr<Impl> tensor,
-
CudaStreamPtr stream,
-
- -) const#
-
- -
-
-

Private Members

-
-
-std::shared_ptr<Impl> mTensor#
-
- -
-
-

Private Static Functions

-
-
-template<typename T>
static inline DataType getRuntimeType()#
-
- -
-
-

Friends

-
-
-friend class Serialization
-
- -
-
-friend std::shared_ptr<runtime::ITensor> const &toITensor( - -
-
Tensor const &tensor,
-
- -)#
-
- -
-
-friend Tensor ofITensor( - -
-
std::shared_ptr<runtime::ITensor> tensor,
-
- -)#
-
- -
-
- -
-
-namespace detail#
-
-

Typedefs

-
-
-using DimType64 = int64_t#
-
- -
-
-

Functions

-
-
-std::shared_ptr<runtime::ITensor> const &toITensor( - -
-
Tensor const &tensor,
-
- -)#
-
- -
-
-Tensor ofITensor(std::shared_ptr<runtime::ITensor> tensor)#
-
- -
-
- -
-
namespace runtime
@@ -4084,16 +5697,17 @@

Public Functions

-
-explicit CacheTransceiverConfig( +
+explicit CacheTransceiverConfig(
std::optional<BackendType> backendType = std::nullopt,
std::optional<size_t> maxNumTokens = std::nullopt,
std::optional<int> kvTransferTimeoutMs = std::nullopt,
+
std::optional<int> kvTransferSenderFutureTimeoutMs = std::nullopt,
-)#
+)#
@@ -4117,8 +5731,14 @@
-
-std::optional<int> getKvTransferTimeoutMs() const#
+
+void setKvTransferSenderFutureTimeoutMs( + +
+
std::optional<int> kvTransferSenderFutureTimeoutMs,
+
+ +)#
@@ -4131,6 +5751,16 @@ std::optional<BackendType> getBackendType() const#
+
+
+std::optional<int> getKvTransferTimeoutMs() const#
+
+ +
+
+std::optional<int> getKvTransferSenderFutureTimeoutMs() const#
+
+

Private Members

@@ -4150,6 +5780,11 @@ std::optional<int> mKvTransferTimeoutMs#
+
+
+std::optional<int> mKvTransferSenderFutureTimeoutMs#
+
+
@@ -9821,8 +11456,8 @@

Public Functions

-
-inline CacheState( +
+inline CacheState(
ModelConfig modelConfig,
@@ -9832,14 +11467,17 @@
AttentionType attentionType = AttentionType::kDEFAULT,
int kvFactor = 2,
bool enableBlockReuse = false,
+
bool hasIndexerKCache = false,
+
SizeType32 indexerDimPerHead = 0,
+
SizeType32 indexerKCacheQuantBlockSize = 128,
-)#
+)#
-
-inline CacheState( +
+inline CacheState(
std::vector<SizeType32> nbKvHeadPerLayer,
@@ -9856,14 +11494,17 @@
int DPrank = 0,
int DPsize = 0,
bool enableBlockReuse = false,
+
bool hasIndexerKCache = false,
+
SizeType32 indexerDimPerHead = 0,
+
SizeType32 indexerKCacheQuantBlockSize = 128,
-)#
+)#
-
-inline CacheState( +
+inline CacheState(
SizeType32 nbAttentionLayers,
@@ -9881,9 +11522,12 @@
int DPrank = 0,
int DPsize = 0,
bool enableBlockReuse = false,
+
bool hasIndexerKCache = false,
+
SizeType32 indexerDimPerHead = 0,
+
SizeType32 indexerKCacheQuantBlockSize = 128,
-)#
+)#
@@ -9922,6 +11566,21 @@ inline bool getEnableBlockReuse() const#
+
+
+inline bool getHasIndexerKCache() const#
+
+ +
+
+inline SizeType32 getIndexerDimPerHead() const#
+
+ +
+
+inline SizeType32 getIndexerKCacheQuantBlockSize() const#
+
+
inline std::string toString() const#
@@ -9955,6 +11614,21 @@ bool mEnableBlockReuse = {false}#
+
+
+bool mHasIndexerKCache = {false}#
+
+ +
+
+SizeType32 mIndexerDimPerHead = {0}#
+
+ +
+
+SizeType32 mIndexerKCacheQuantBlockSize = {128}#
+
+

Friends

@@ -10263,1736 +11937,131 @@
-
-

serialization.h#

+
+

cacheCommunicator.h#

namespace tensorrt_llm
namespace executor
+
+
+namespace kv_cache
-
-class Serialization#
+
+class Connection#
-

Public Static Functions

+

Public Functions

-
-static size_t serializedSize( - -
-
tensorrt_llm::batch_manager::kv_cache_manager::BlockKey const &key,
-
- -)#
-
- -
-
-static void serialize( - -
-
tensorrt_llm::batch_manager::kv_cache_manager::BlockKey const &key,
-
std::ostream &os,
-
- -)#
-
- -
-
-static tensorrt_llm::batch_manager::kv_cache_manager::BlockKey deserializeBlockKey( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static RequestPerfMetrics::TimePoint deserializeTimePoint( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
RequestPerfMetrics::TimePoint const &tp,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(RequestPerfMetrics::TimePoint const&)#
-
- -
-
-static RequestPerfMetrics deserializeRequestPerfMetrics( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
RequestPerfMetrics const &metrics,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(RequestPerfMetrics const &metrics)#
-
- -
-
-static SamplingConfig deserializeSamplingConfig(std::istream &is)#
-
- -
-
-static void serialize(SamplingConfig const &config, std::ostream &os)#
-
- -
-
-static size_t serializedSize(SamplingConfig const &config)#
-
- -
-
-static OutputConfig deserializeOutputConfig(std::istream &is)#
-
- -
-
-static void serialize(OutputConfig const &config, std::ostream &os)#
-
- -
-
-static size_t serializedSize(OutputConfig const &config)#
-
- -
-
-static AdditionalModelOutput deserializeAdditionalModelOutput( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
AdditionalModelOutput const &additionalModelOutput,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
AdditionalModelOutput const &additionalModelOutput,
-
- -)#
-
- -
-
-static ExternalDraftTokensConfig deserializeExternalDraftTokensConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
ExternalDraftTokensConfig const &config,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(ExternalDraftTokensConfig const &config)#
-
- -
-
-static PromptTuningConfig deserializePromptTuningConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
PromptTuningConfig const &config,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(PromptTuningConfig const &config)#
-
- -
-
-static MultimodalInput deserializeMultimodalInput(std::istream &is)#
-
- -
-
-static void serialize( - -
-
MultimodalInput const &multimodalInput,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(MultimodalInput const &multimodalInput)#
-
- -
-
-static MropeConfig deserializeMropeConfig(std::istream &is)#
-
- -
-
-static void serialize(MropeConfig const &config, std::ostream &os)#
-
- -
-
-static size_t serializedSize(MropeConfig const &config)#
-
- -
-
-static LoraConfig deserializeLoraConfig(std::istream &is)#
-
- -
-
-static void serialize(LoraConfig const &config, std::ostream &os)#
-
- -
-
-static size_t serializedSize(LoraConfig const &config)#
-
- -
-
-static kv_cache::CommState deserializeCommState(std::istream &is)#
-
- -
-
-static void serialize( - -
-
kv_cache::CommState const &state,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(kv_cache::CommState const &state)#
-
- -
-
-static kv_cache::SocketState deserializeSocketState(std::istream &is)#
-
- -
-
-static void serialize( - -
-
kv_cache::SocketState const &state,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(kv_cache::SocketState const &state)#
-
- -
-
-static kv_cache::AgentState deserializeAgentState(std::istream &is)#
-
- -
-
-static void serialize( - -
-
kv_cache::AgentState const &state,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(kv_cache::AgentState const &state)#
-
- -
-
-static kv_cache::CacheState deserializeCacheState(std::istream &is)#
-
- -
-
-static void serialize( - -
-
kv_cache::CacheState const &state,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(kv_cache::CacheState const &state)#
-
- -
-
-static DataTransceiverState deserializeDataTransceiverState( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static DataTransceiverState deserializeDataTransceiverState( - -
-
std::vector<char> &buffer,
-
- -)#
-
- -
-
-static void serialize( - -
-
DataTransceiverState const &dataTransceiverState,
-
std::ostream &os,
-
- -)#
-
- -
-
-static std::vector<char> serialize( - -
-
DataTransceiverState const &dataTransceiverState,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
DataTransceiverState const &dataTransceiverState,
-
- -)#
-
- -
-
-static ContextPhaseParams deserializeContextPhaseParams( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
ContextPhaseParams const &contextPhaseParams,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
ContextPhaseParams const &contextPhaseParams,
-
- -)#
-
- -
-
-static Request deserializeRequest(std::istream &is)#
-
- -
-
-static void serialize(Request const &request, std::ostream &os)#
-
- -
-
-static size_t serializedSize(Request const &request)#
-
- -
-
-static Tensor deserializeTensor(std::istream &is)#
-
- -
-
-static void serialize(Tensor const &tensor, std::ostream &os)#
-
- -
-
-static size_t serializedSize(Tensor const &tensor)#
-
- -
-
-static SpeculativeDecodingFastLogitsInfo deserializeSpecDecFastLogitsInfo( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
SpeculativeDecodingFastLogitsInfo const &info,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
SpeculativeDecodingFastLogitsInfo const &info,
-
- -)#
-
- -
-
-static Result deserializeResult(std::istream &is)#
-
- -
-
-static void serialize(Result const &result, std::ostream &os)#
-
- -
-
-static size_t serializedSize(Result const &result)#
-
- -
-
-static AdditionalOutput deserializeAdditionalOutput(std::istream &is)#
-
- -
-
-static void serialize( - -
-
AdditionalOutput const &additionalOutput,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
AdditionalOutput const &additionalOutput,
-
- -)#
-
- -
-
-static Response deserializeResponse(std::istream &is)#
-
- -
-
-static void serialize(Response const &response, std::ostream &os)#
-
- -
-
-static size_t serializedSize(Response const &response)#
-
- -
-
-static std::vector<Response> deserializeResponses( - -
-
std::vector<char> &buffer,
-
- -)#
-
- -
-
-static std::vector<char> serialize( - -
-
std::vector<Response> const &responses,
-
- -)#
-
- -
-
-static KvCacheConfig deserializeKvCacheConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
KvCacheConfig const &kvCacheConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(KvCacheConfig const &kvCacheConfig)#
-
- -
-
-static DynamicBatchConfig deserializeDynamicBatchConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
DynamicBatchConfig const &dynamicBatchConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
DynamicBatchConfig const &dynamicBatchConfig,
-
- -)#
-
- -
-
-static SchedulerConfig deserializeSchedulerConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
SchedulerConfig const &schedulerConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(SchedulerConfig const &schedulerConfig)#
-
- -
-
-static ExtendedRuntimePerfKnobConfig deserializeExtendedRuntimePerfKnobConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
ExtendedRuntimePerfKnobConfig const &extendedRuntimePerfKnobConfig,
-
- -)#
-
- -
-
-static ParallelConfig deserializeParallelConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
ParallelConfig const &parallelConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(ParallelConfig const &parallelConfig)#
-
- -
-
-static PeftCacheConfig deserializePeftCacheConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
PeftCacheConfig const &peftCacheConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(PeftCacheConfig const &peftCacheConfig)#
-
- -
-
-static OrchestratorConfig deserializeOrchestratorConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
OrchestratorConfig const &orchestratorConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
OrchestratorConfig const &orchestratorConfig,
-
- -)#
-
- -
-
-static DecodingMode deserializeDecodingMode(std::istream &is)#
-
- -
-
-static void serialize( - -
-
DecodingMode const &decodingMode,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(DecodingMode const &decodingMode)#
-
- -
-
-static LookaheadDecodingConfig deserializeLookaheadDecodingConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
LookaheadDecodingConfig const &lookaheadDecodingConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
LookaheadDecodingConfig const &lookaheadDecodingConfig,
-
- -)#
-
- -
-
-static EagleConfig deserializeEagleConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
EagleConfig const &eagleConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(EagleConfig const &eagleConfig)#
-
- -
-
-static SpeculativeDecodingConfig deserializeSpeculativeDecodingConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
SpeculativeDecodingConfig const &specDecConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
SpeculativeDecodingConfig const &specDecConfig,
-
- -)#
-
- -
-
-static GuidedDecodingConfig deserializeGuidedDecodingConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
GuidedDecodingConfig const &guidedDecodingConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
GuidedDecodingConfig const &guidedDecodingConfig,
-
- -)#
-
- -
-
-static GuidedDecodingParams deserializeGuidedDecodingParams( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
GuidedDecodingParams const &guidedDecodingParams,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
GuidedDecodingParams const &guidedDecodingParams,
-
- -)#
-
- -
-
-static KvCacheRetentionConfig deserializeKvCacheRetentionConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
KvCacheRetentionConfig const &kvCacheRetentionConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
KvCacheRetentionConfig const &kvCacheRetentionConfig,
-
- -)#
-
- -
-
-static KvCacheRetentionConfig::TokenRangeRetentionConfig deserializeTokenRangeRetentionConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
KvCacheRetentionConfig::TokenRangeRetentionConfig const &tokenRangeRetentionConfig,
-
- -)#
-
- -
-
-static DecodingConfig deserializeDecodingConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
DecodingConfig const &decodingConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(DecodingConfig const &decodingConfig)#
-
- -
-
-static DebugConfig deserializeDebugConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
DebugConfig const &debugConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(DebugConfig const &debugConfig)#
-
- -
-
-static CacheTransceiverConfig deserializeCacheTransceiverConfig( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
CacheTransceiverConfig const &cacheTransceiverConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
CacheTransceiverConfig const &cacheTransceiverConfig,
-
- -)#
-
- -
-
-static ExecutorConfig deserializeExecutorConfig(std::istream &is)#
-
- -
-
-static void serialize( - -
-
ExecutorConfig const &executorConfig,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(ExecutorConfig const &executorConfig)#
-
- -
-
-static KvCacheStats deserializeKvCacheStats(std::istream &is)#
-
- -
-
-static void serialize( - -
-
KvCacheStats const &kvCacheStats,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(KvCacheStats const &kvCacheStats)#
-
- -
-
-static StaticBatchingStats deserializeStaticBatchingStats( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
StaticBatchingStats const &staticBatchingStats,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
StaticBatchingStats const &staticBatchingStats,
-
- -)#
-
- -
-
-static InflightBatchingStats deserializeInflightBatchingStats( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
InflightBatchingStats const &inflightBatchingStats,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
InflightBatchingStats const &inflightBatchingStats,
-
- -)#
-
- -
-
-static SpecDecodingStats deserializeSpecDecodingStats( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
SpecDecodingStats const &specDecodingStats,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
SpecDecodingStats const &specDecodingStats,
-
- -)#
-
- -
-
-static IterationStats deserializeIterationStats( - -
-
std::vector<char> &buffer,
-
- -)#
-
- -
-
-static IterationStats deserializeIterationStats(std::istream &is)#
-
- -
-
-static void serialize( - -
-
IterationStats const &iterStats,
-
std::ostream &os,
-
- -)#
-
- -
-
-static std::vector<char> serialize(IterationStats const &iterStats)#
-
- -
-
-static size_t serializedSize(IterationStats const &iterStats)#
-
- -
-
-static std::vector<char> serialize( - -
-
std::vector<IterationStats> const &iterStatsVec,
-
- -)#
-
- -
-
-static std::vector<IterationStats> deserializeIterationStatsVec( - -
-
std::vector<char> &buffer,
-
- -)#
-
- -
-
-static DisServingRequestStats deserializeDisServingRequestStats( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static void serialize( - -
-
DisServingRequestStats const &stats,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
DisServingRequestStats const &disServingRequestStats,
-
- -)#
-
- -
-
-static RequestStage deserializeRequestStage(std::istream &is)#
-
- -
-
-static void serialize( - -
-
RequestStage const &requestStage,
-
std::ostream &os,
-
- -)#
-
- -
-
-static size_t serializedSize(RequestStage const &requestStage)#
-
- -
-
-static RequestStats deserializeRequestStats(std::istream &is)#
-
- -
-
-static void serialize(RequestStats const &state, std::ostream &os)#
-
- -
-
-static size_t serializedSize(RequestStats const &state)#
-
- -
-
-static RequestStatsPerIteration deserializeRequestStatsPerIteration( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static RequestStatsPerIteration deserializeRequestStatsPerIteration( - -
-
std::vector<char> &buffer,
-
- -)#
-
- -
-
-static void serialize( - -
-
RequestStatsPerIteration const &state,
-
std::ostream &os,
-
- -)#
-
- -
-
-static std::vector<char> serialize( - -
-
RequestStatsPerIteration const &state,
-
- -)#
-
- -
-
-static size_t serializedSize(RequestStatsPerIteration const &state)#
-
- -
-
-static std::vector<char> serialize( - -
-
std::vector<RequestStatsPerIteration> const &requestStatsVec,
-
- -)#
-
- -
-
-static std::vector<RequestStatsPerIteration> deserializeRequestStatsPerIterationVec( - -
-
std::vector<char> &buffer,
-
- -)#
-
- -
-
-static std::vector<char> serialize( - -
-
std::deque<KVCacheEvent> const &kvCacheEvents,
-
- -)#
-
- -
-
-static std::deque<KVCacheEvent> deserializeKVCacheEvents( - -
-
std::vector<char> &buffer,
-
- -)#
+
+virtual ~Connection() = default#
-
-static size_t serializedSize(KVCacheEvent const &event)#
-
- -
-
-static void serialize(KVCacheEvent const &event, std::ostream &os)#
-
- -
-
-static KVCacheEvent deserializeKVCacheEvent(std::istream &is)#
-
- -
-
-static size_t serializedSize(KVCacheCreatedData const &data)#
-
- -
-
-static void serialize( - -
-
KVCacheCreatedData const &data,
-
std::ostream &os,
-
- -)#
-
- -
-
-static KVCacheCreatedData deserializeKVCacheCreatedData( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static size_t serializedSize(KVCacheStoredData const &data)#
-
- -
-
-static void serialize( - -
-
KVCacheStoredData const &data,
-
std::ostream &os,
-
- -)#
-
- -
-
-static KVCacheStoredData deserializeKVCacheStoredData( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static size_t serializedSize(KVCacheStoredBlockData const &data)#
-
- -
-
-static void serialize( - -
-
KVCacheStoredBlockData const &data,
-
std::ostream &os,
-
- -)#
-
- -
-
-static KVCacheStoredBlockData deserializeKVCacheStoredBlockData( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static size_t serializedSize(KVCacheRemovedData const &data)#
-
- -
-
-static void serialize( - -
-
KVCacheRemovedData const &data,
-
std::ostream &os,
-
- -)#
-
- -
-
-static KVCacheRemovedData deserializeKVCacheRemovedData( - -
-
std::istream &is,
-
- -)#
-
- -
-
-template<typename T>
static size_t serializedSize( - -
-
KVCacheEventDiff<T> const &data,
-
- -)#
-
- -
-
-template<typename T>
static void serialize( - -
-
KVCacheEventDiff<T> const &data,
-
std::ostream &os,
-
- -)#
-
- -
-
-template<typename T>
static KVCacheEventDiff<T> deserializeKVCacheEventDiff( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static size_t serializedSize(KVCacheUpdatedData const &data)#
-
- -
-
-static void serialize( - -
-
KVCacheUpdatedData const &data,
-
std::ostream &os,
-
- -)#
-
- -
-
-static KVCacheUpdatedData deserializeKVCacheUpdatedData( - -
-
std::istream &is,
-
- -)#
-
- -
-
-static size_t serializedSize( - -
-
tensorrt_llm::runtime::UniqueToken const &token,
-
- -)#
-
- -
-
-static void serialize( +
+virtual void send(
-
tensorrt_llm::runtime::UniqueToken const &token,
-
std::ostream &os,
+
DataContext const &ctx,
+
void const *data,
+
size_t size,
-)#
+) const = 0#
-
-static tensorrt_llm::runtime::UniqueToken deserializeUniqueToken( +
+virtual void recv(
-
std::istream &is,
+
DataContext const &ctx,
+
void *data,
+
size_t size,
-)#
-
- -
-
-static std::string deserializeString(std::istream &is)#
-
- -
-
-static bool deserializeBool(std::istream &is)#
+) const = 0#
-
-static ModelType deserializeModelType(std::istream &is)#
+
+inline virtual bool isThreadSafe() const noexcept#
-
-
-namespace kv_cache
+
+
+class ConnectionManager#
+
+

Public Functions

+
+
+virtual ~ConnectionManager() = default#
+
+
+virtual Connection const *recvConnect( + +
+
DataContext const &ctx,
+
void *data,
+
size_t size,
+
+ +) = 0#
+
+ +
+
+virtual std::vector<Connection const*> getConnections( + +
+
CommState const &state,
+
+ +) = 0#
+
+ +
+
+virtual CommState const &getCommState() const = 0#
+
+ +
+
+ +
+
+struct DataContext#
+
+

Public Functions

+
+
+inline explicit DataContext(int tag)#
+
+ +
+
+inline int getTag() const noexcept#
+
+ +
+
+

Private Members

+
+
+int const mTag#
+
+ +
+
+ +
+
@@ -12031,134 +12100,33 @@
+
+

bufferManager.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class BufferManager#
+
+#include <bufferManager.h>
+

A helper class for managing memory on host and device.

+
+

Public Types

+
+
+using IBufferPtr = IBuffer::UniquePtr#
+
+ +
+
+using ITensorPtr = ITensor::UniquePtr#
+
+ +
+
+using CudaStreamPtr = std::shared_ptr<CudaStream>#
+
+ +
+
+using CudaMemPoolPtr = std::shared_ptr<CudaMemPool>#
+
+ +
+
+

Public Functions

+
+
+explicit BufferManager(CudaStreamPtr stream, bool trimPool = false)#
+

Construct a BufferManager.

+
+
Parameters:
+

cudaStream – [in] The cuda stream to use for all operations on GPU (allocation, de-allocation, copying, etc.).

+
+
+
+ +
+
+inline ~BufferManager()#
+

Destructor.

+
+ +
+
+IBufferPtr gpu( + +
+
std::size_t size,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +) const#
+

Allocates an IBuffer of the given size on the GPU, using cudaMallocAsync.

+
+ +
+
+ITensorPtr gpu( + +
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +) const#
+

Allocates an ITensor of the given dimensions on the GPU, using cudaMallocAsync.

+
+ +
+
+IBufferPtr allocate( + +
+
MemoryType memoryType,
+
std::size_t size,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +) const#
+

Allocates an IBuffer of the given size and memory type.

+
+ +
+
+ITensorPtr allocate( + +
+
MemoryType memoryType,
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +) const#
+

Allocates an ITensor of the given dimensions and memory type.

+
+ +
+
+inline IBufferPtr emptyBuffer( + +
+
MemoryType memoryType,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +) const#
+

Create an empty IBuffer of the given memory type. It may be resized later.

+
+ +
+
+inline ITensorPtr emptyTensor( + +
+
MemoryType memoryType,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +) const#
+

Create an empty ITensor of the given memory type. It may be reshaped later.

+
+ +
+
+void setMem(IBuffer &buffer, int32_t value) const#
+

Set the contents of the given buffer to value.

+
+ +
+
+void setZero(IBuffer &buffer) const#
+

Set the contents of the given buffer to zero.

+
+ +
+
+void copy(void const *src, IBuffer &dst, MemoryType srcType) const#
+

Copy src to dst.

+
+ +
+
+void copy(IBuffer const &src, void *dst, MemoryType dstType) const#
+

Copy src to dst.

+
+ +
+
+inline void copy(void const *src, IBuffer &dst) const#
+

Copy src to dst.

+
+ +
+
+inline void copy(IBuffer const &src, void *dst) const#
+

Copy src to dst.

+
+ +
+
+void copy(IBuffer const &src, IBuffer &dst) const#
+

Copy src to dst.

+
+ +
+
+IBufferPtr copyFrom(IBuffer const &src, MemoryType memoryType) const#
+

Copy src into a new IBuffer with a potentially different memory type.

+
+ +
+
+ITensorPtr copyFrom(ITensor const &src, MemoryType memoryType) const#
+

Copy src into a new ITensor with a potentially different memory type.

+
+ +
+
+template<typename T>
inline IBufferPtr copyFrom( + +
+
std::vector<T> const &src,
+
MemoryType memoryType,
+
+ +) const#
+

Copy src into a new IBuffer with a potentially different memory type.

+
+ +
+
+template<typename T>
inline ITensorPtr copyFrom( + +
+
T *src,
+
nvinfer1::Dims dims,
+
MemoryType memoryType,
+
+ +) const#
+

Copy src into a new ITensor with a potentially different memory type.

+
+ +
+
+template<typename T>
inline ITensorPtr copyFrom( + +
+
std::vector<T> const &src,
+
nvinfer1::Dims dims,
+
MemoryType memoryType,
+
+ +) const#
+

Copy src into a new ITensor with a potentially different memory type.

+
+ +
+
+CudaStream const &getStream() const#
+

Get the underlying cuda stream.

+
+ +
+
+std::size_t memoryPoolReserved() const#
+

The current size of the memory reserved by the memory pool.

+
+ +
+
+std::size_t memoryPoolUsed() const#
+

The current size of the memory used by the memory pool.

+
+ +
+
+std::size_t memoryPoolFree() const#
+

The current size of the memory free in the memory pool.

+
+ +
+
+void memoryPoolTrimTo(std::size_t size)#
+

Try to trim the memory reserved by the pool to size bytes. This synchronizes implicitly with the stream.

+
+ +
+
+

Public Static Functions

+
+
+static IBufferPtr gpuSync( + +
+
std::size_t size,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates an IBuffer of the given size on the GPU, using cudaMalloc.

+
+ +
+
+static ITensorPtr gpuSync( + +
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates an ITensor of the given dimensions on the GPU, using cudaMalloc.

+
+ +
+
+static IBufferPtr cpu( + +
+
std::size_t size,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates an IBuffer of the given size on the CPU.

+
+ +
+
+static ITensorPtr cpu( + +
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates an ITensor of the given dimensions on the CPU.

+
+ +
+
+static IBufferPtr pinned( + +
+
std::size_t size,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates a pinned IBuffer of the given size on the CPU.

+
+ +
+
+static ITensorPtr pinned( + +
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates a pinned ITensor of the given dimensions on the CPU.

+
+ +
+
+static IBufferPtr pinnedPool( + +
+
std::size_t size,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates a pinned IBuffer of the given size on the CPU in the default memory pool.

+
+ +
+
+static ITensorPtr pinnedPool( + +
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates a pinned ITensor of the given dimensions on the CPU in the default memory pool.

+
+ +
+
+static IBufferPtr managed( + +
+
std::size_t size,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates an IBuffer of the given size in UVM.

+
+ +
+
+static ITensorPtr managed( + +
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type = kBYTE_TYPE,
+
+ +)#
+

Allocates an ITensor of the given dimensions in UVM.

+
+ +
+
+static ITensorPtr ipcNvls( + +
+
std::set<int> ranks,
+
nvinfer1::Dims dims,
+
nvinfer1::DataType type,
+
+ +)#
+

Allocates an ITensor of the given dimensions for NVLS.

+
+ +
+
+

Public Static Attributes

+
+
+static auto constexpr kBYTE_TYPE = nvinfer1::DataType::kUINT8#
+
+ +
+
+

Private Members

+
+
+CudaStreamPtr mStream#
+
+ +
+
+CudaMemPoolPtr mPool#
+
+ +
+
+bool const mTrimPool#
+
+ +
+
+

Friends

+
+
+friend class ::BufferManagerTest
+
+ +
+
+ +
+ +
+ +
+
+

gptJsonConfig.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class GptJsonConfig#
+
+

Public Functions

+
+
+inline GptJsonConfig( + +
+
std::string name,
+
std::string version,
+
std::string precision,
+
SizeType32 tensorParallelism,
+
SizeType32 pipelineParallelism,
+
SizeType32 contextParallelism,
+
SizeType32 gpusPerNode,
+
ModelConfig modelConfig,
+
std::optional<RuntimeDefaults> runtimeDefaults = std::nullopt,
+
+ +)#
+
+ +
+
+inline ModelConfig const &getModelConfig() const#
+
+ +
+
+inline ModelConfig &getModelConfigMutable()#
+
+ +
+
+inline std::string const &getName() const#
+
+ +
+
+inline std::string const &getVersion() const#
+
+ +
+
+inline std::string const &getPrecision() const#
+
+ +
+
+inline SizeType32 constexpr getTensorParallelism() const#
+
+ +
+
+inline SizeType32 constexpr getPipelineParallelism() const#
+
+ +
+
+inline SizeType32 constexpr getContextParallelism() const#
+
+ +
+
+inline SizeType32 constexpr getGpusPerNode() const#
+
+ +
+
+inline SizeType32 constexpr getWorldSize() const#
+
+ +
+
+inline std::optional<RuntimeDefaults> getRuntimeDefaults() const#
+
+ +
+
+std::string engineFilename( + +
+
WorldConfig const &worldConfig,
+
std::string const &model,
+
+ +) const#
+
+ +
+
+inline std::string engineFilename( + +
+
WorldConfig const &worldConfig,
+
+ +) const#
+
+ +
+
+

Public Static Functions

+
+
+static GptJsonConfig parse(std::string const &json)#
+
+ +
+
+static GptJsonConfig parse(std::istream &json)#
+
+ +
+
+static GptJsonConfig parse(std::filesystem::path const &path)#
+
+ +
+
+

Private Members

+
+
+std::string const mName#
+
+ +
+
+std::string const mVersion#
+
+ +
+
+std::string const mPrecision#
+
+ +
+
+SizeType32 const mTensorParallelism#
+
+ +
+
+SizeType32 const mPipelineParallelism#
+
+ +
+
+SizeType32 const mContextParallelism#
+
+ +
+
+SizeType32 const mGpusPerNode#
+
+ +
+
+ModelConfig mModelConfig#
+
+ +
+
+std::optional<RuntimeDefaults> mRuntimeDefaults#
+
+ +
+
+ +
+ +
+ +
+
+

runtimeDefaults.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+struct RuntimeDefaults#
+
+

Public Functions

+
+
+inline RuntimeDefaults( + +
+
std::optional<std::vector<SizeType32>> maxAttentionWindowVec,
+
std::optional<SizeType32> sinkTokenLength,
+
+ +)#
+
+ +
+
+RuntimeDefaults() = default#
+
+ +
+
+

Public Members

+
+
+std::optional<std::vector<SizeType32>> maxAttentionWindowVec#
+
+ +
+
+std::optional<SizeType32> sinkTokenLength#
+
+ +
@@ -5795,6 +5438,925 @@ one more than decoding draft tokens for prediction from primary head

+
+
+

rawEngine.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class RawEngine#
+
+

Public Types

+
+
+enum Type#
+

Values:

+
+
+enumerator FilePath#
+
+ +
+
+enumerator AddressWithSize#
+
+ +
+
+enumerator HostMemory#
+
+ +
+ +
+
+

Public Functions

+
+
+inline explicit RawEngine(std::filesystem::path enginePath) noexcept#
+
+ +
+
+inline explicit RawEngine( + +
+
void const *engineAddr,
+
std::size_t engineSize,
+
+ +) noexcept#
+
+ +
+
+inline explicit RawEngine( + +
+
nvinfer1::IHostMemory const *engineBuffer,
+
+ +) noexcept#
+
+ +
+
+inline Type getType() const#
+
+ +
+
+inline std::filesystem::path getPath() const#
+
+ +
+
+inline std::optional<std::filesystem::path> getPathOpt() const#
+
+ +
+
+inline void setPath(std::filesystem::path enginePath)#
+
+ +
+
+inline std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> const &getManagedWeightsMapOpt( + +
+
+ +) const#
+
+ +
+
+inline void setManagedWeightsMap( + +
+
std::map<std::string, tensorrt_llm::executor::Tensor> managedWeightsMap,
+
+ +)#
+
+ +
+
+inline void const *getAddress() const#
+
+ +
+
+inline std::size_t getSize() const#
+
+ +
+
+inline nvinfer1::IHostMemory const *getHostMemory() const#
+
+ +
+
+

Public Members

+
+
+void const *mEngineAddr = {}#
+
+ +
+
+std::size_t mEngineSize = {}#
+
+ +
+
+

Private Members

+
+
+Type mType#
+
+ +
+
+std::optional<std::filesystem::path> mEnginePath#
+
+ +
+
+struct tensorrt_llm::runtime::RawEngine
+
+ +
+
+nvinfer1::IHostMemory const *mEngineBuffer = {}#
+
+ +
+
+std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> mManagedWeightsMap#
+
+ +
+
+ +
+ +
+ +
+
+

gptDecoder.h#

+
+
+namespace tensorrt_llm
+
+
+namespace layers#
+
+ +
+
+namespace runtime
+
+

Functions

+
+
+inline runtime::ITensor::SharedConstPtr getDefaultBatchSlots( + +
+
runtime::SizeType32 batchSize,
+
+ +)#
+

Helper function to produce batch slots [0, 1, …, batchSize - 1] for paths that do not explicitly provide batch slots to the decoder.

+
+ +
+
+
+template<typename T>
class GptDecoder : public virtual tensorrt_llm::runtime::IGptDecoder#
+
+

Public Types

+
+
+using CudaStreamPtr = BufferManager::CudaStreamPtr#
+
+ +
+
+using TensorPtr = std::shared_ptr<ITensor>#
+
+ +
+
+

Public Functions

+
+
+GptDecoder( + +
+
executor::DecodingMode const &mode,
+
size_t maxNumSequences,
+
size_t maxBeamWidth,
+
size_t vocabSize,
+
size_t vocabSizePadded,
+
CudaStreamPtr const &stream,
+
std::shared_ptr<SpeculativeDecodingModule const> speculativeDecodingModule = nullptr,
+
+ +)#
+
+ +
+
+virtual void setup( + +
+
SamplingConfig const &samplingConfig,
+
size_t batchSize,
+
TensorConstPtr const &batchSlots,
+
std::optional<DecodingOutput> const &output = std::nullopt,
+
std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt,
+
std::optional<std::vector<TensorConstPtr>> const &lookaheadPrompt = std::nullopt,
+
std::optional<std::vector<executor::LookaheadDecodingConfig>> const &lookaheadAlgoConfigs = std::nullopt,
+
+ +) override#
+
+
Parameters:
+

explicitDraftTokensDType – is only used by ExplicitDraftTokens model to WAR the lack of bf16 decoder.

+
+
+
+ +
+
+virtual void forwardAsync( + +
+
DecodingOutput &output,
+
DecodingInput const &input,
+
+ +) override#
+
+ +
+
+virtual void forwardSync( + +
+
DecodingOutput &output,
+
DecodingInput const &input,
+
+ +) override#
+
+ +
+
+inline virtual SamplingConfig const &getSamplingConfig() override#
+
+ +
+
+virtual void disableLookahead( + +
+
std::optional<SamplingConfig> const &samplingConfig,
+
SizeType32 batchSize,
+
TensorConstPtr batchSlots,
+
+ +) override#
+
+ +
+
+

Private Members

+
+
+std::shared_ptr<BufferManager> mManager#
+
+ +
+
+std::shared_ptr<tensorrt_llm::layers::DynamicDecodeLayer<T>> mDynamicDecodeLayer#
+
+ +
+
+std::shared_ptr<tensorrt_llm::runtime::DecodingLayerWorkspace> mDecodingLayerWorkspace#
+
+ +
+
+SamplingConfig mSamplingConfig#
+
+ +
+
+size_t mMaxNumSequences#
+
+ +
+
+size_t mVocabSize#
+
+ +
+
+size_t mVocabSizePadded#
+
+ +
+
+executor::DecodingMode mDecodingMode#
+
+ +
+
+ +
+
+class IGptDecoder#
+

Subclassed by tensorrt_llm::runtime::GptDecoder< T >

+
+

Public Types

+
+
+using TensorPtr = runtime::ITensor::SharedPtr#
+
+ +
+
+using TensorConstPtr = runtime::ITensor::SharedConstPtr#
+
+ +
+
+

Public Functions

+
+
+virtual ~IGptDecoder() = default#
+
+ +
+
+virtual void setup( + +
+
SamplingConfig const &samplingConfig,
+
size_t batchSize,
+
TensorConstPtr const &batchSlots,
+
std::optional<DecodingOutput> const &output = std::nullopt,
+
std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt,
+
std::optional<std::vector<TensorConstPtr>> const &lookaheadPrompt = std::nullopt,
+
std::optional<std::vector<executor::LookaheadDecodingConfig>> const &lookaheadAlgoConfigs = std::nullopt,
+
+ +) = 0#
+
+
Parameters:
+

explicitDraftTokensDType – is only used by ExplicitDraftTokens model to WAR the lack of bf16 decoder.

+
+
+
+ +
+
+virtual void forwardAsync( + +
+
DecodingOutput &output,
+
DecodingInput const &input,
+
+ +) = 0#
+
+ +
+
+virtual void forwardSync( + +
+
DecodingOutput &output,
+
DecodingInput const &input,
+
+ +) = 0#
+
+ +
+
+virtual SamplingConfig const &getSamplingConfig() = 0#
+
+ +
+
+virtual void disableLookahead( + +
+
std::optional<SamplingConfig> const &samplingConfig,
+
SizeType32 batchSize,
+
TensorConstPtr batchSlots,
+
+ +) = 0#
+
+ +
+
+

Public Static Functions

+
+
+static inline std::unique_ptr<IGptDecoder> create( + +
+
executor::DecodingMode const &mode,
+
nvinfer1::DataType dtype,
+
size_t maxNumSequences,
+
size_t maxBeamWidth,
+
size_t vocabSize,
+
size_t vocabSizePadded,
+
BufferManager::CudaStreamPtr const &stream,
+
std::shared_ptr<SpeculativeDecodingModule const> const &speculativeDecodingModule = nullptr,
+
+ +)#
+
+ +
+
+ +
+ +
+ +
+
+

eagleBuffers.h#

+
+
+namespace tensorrt_llm
+
+
+namespace batch_manager
+
+ +
+
+namespace runtime
+
+
+class EagleBuffers#
+
+

Public Types

+
+
+using LlmRequestPtr = std::shared_ptr<tensorrt_llm::batch_manager::LlmRequest>#
+
+ +
+
+using RequestVector = std::vector<LlmRequestPtr>#
+
+ +
+
+using SizeType32 = runtime::SizeType32#
+
+ +
+
+using ITensor = runtime::ITensor#
+
+ +
+
+using BufferPtr = runtime::IBuffer::SharedPtr#
+
+ +
+
+using TensorPtr = runtime::ITensor::SharedPtr#
+
+ +
+
+using TensorMap = runtime::StringPtrMap<runtime::ITensor>#
+
+ +
+
+

Public Functions

+
+
+EagleBuffers( + +
+
SizeType32 maxBatchSize,
+
SizeType32 maxBeamWidth,
+
runtime::BufferManager const &manager,
+
runtime::ModelConfig const &modelConfig,
+
runtime::WorldConfig const &worldConfig,
+
executor::DecodingConfig const &decodingConfig,
+
+ +)#
+
+ +
+
+void reshape( + +
+
SizeType32 numCtxSequences,
+
SizeType32 numGenSequences,
+
runtime::ModelConfig const &modelConfig,
+
+ +)#
+
+ +
+
+void setFromInputs( + +
+
RequestVector const &contextRequests,
+
RequestVector const &genRequests,
+
runtime::ITensor const &requestTypes,
+
ITensor const &seqSlots,
+
EagleBuffers::Inputs const &decoderBuffers,
+
runtime::BufferManager const &manager,
+
runtime::ModelConfig const &modelConfig,
+
runtime::WorldConfig const &worldConfig,
+
+ +) const#
+
+ +
+
+void insertInputTensors( + +
+
TensorMap &inputBuffers,
+
TensorMap &outputBuffers,
+
runtime::WorldConfig const &worldConfig,
+
+ +) const#
+
+ +
+
+

Public Members

+
+
+Inputs engineInputs#
+
+ +
+
+class tensorrt_llm::runtime::EagleBuffers::EngineOutputs engineOutputs#
+
+ +
+
+

Private Functions

+
+
+template<typename T>
void setFromInputs( + +
+
RequestVector const &contextRequests,
+
RequestVector const &genRequests,
+
SizeType32 vocabSizePadded,
+
ITensor const &seqSlots,
+
EagleBuffers::Inputs const &draftBuffers,
+
runtime::EagleModule const &eagleModule,
+
runtime::BufferManager const &manager,
+
+ +) const#
+
+ +
+
+

Private Members

+
+
+std::size_t scanReduceTempStorageBytes = {0}#
+
+ +
+
+float mDefaultPosteriorThreshold = {0.09f}#
+
+ +
+
+bool mDoGreedySampling = {true}#
+
+ +
+
+BufferPtr scanReduceTempStorage#
+
+ +
+
+TensorPtr cumSumGenerationLengths#
+
+ +
+
+TensorPtr maxGenerationLength#
+
+ +
+
+TensorPtr chunkedContextNextTokensHost#
+
+ +
+
+TensorPtr greedySamplingHost#
+
+ +
+
+TensorPtr posteriorAlphaHost#
+
+ +
+
+TensorPtr posteriorThresholdHost#
+
+ +
+
+
+class EngineOutputs#
+
+

Public Members

+
+
+TensorPtr nextDraftTokens#
+

[batchSize, maxDecodingDraftTokens]

+
+ +
+
+TensorPtr nextDraftLens#
+

[batchSize]

+
+ +
+
+TensorPtr nextDraftPaths#
+

[batchSize, maxNumPaths, maxPathLen]

+
+ +
+
+TensorPtr acceptedTokens#
+

[batchSize, maxPathLen]

+
+ +
+
+TensorPtr acceptedLens#
+

[batchSize]

+
+ +
+
+TensorPtr acceptedPaths#
+

[batchSize]

+
+ +
+
+TensorPtr chunkedContextNextTokens#
+

[batchSize]

+
+ +
+
+ +
+
+class Inputs#
+
+

Public Functions

+
+
+void create( + +
+
SizeType32 maxNumSequences,
+
BufferManager const &manager,
+
ModelConfig const &modelConfig,
+
WorldConfig const &worldConfig,
+
+ +)#
+
+ +
+
+

Public Members

+
+
+TensorPtr temperatures#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr posteriorAlpha#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr posteriorThreshold#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr randomDataSample#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr randomDataValidation#
+

[maxBatchSize, maxDecodingTokens] or [numSequences, maxDecodingTokens]

+
+ +
+
+TensorPtr draftTokens#
+

[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens]

+
+ +
+
+TensorPtr draftLens#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr draftPaths#
+

[maxBatchSize, maxNumPaths, maxPathLen] or [numSequences, maxNumPaths, maxPathLen]

+
+ +
+
+TensorPtr draftPathsHost#
+

[maxBatchSize, maxNumPaths, maxPathLen] or [numSequences, maxNumPaths, maxPathLen]

+
+ +
+
+TensorPtr specDecodingGenerationLengths#
+

[maxBatchSize] or [numGenSequences]

+
+ +
+
+TensorPtr specDecodingGenerationLengthsHost#
+

[maxBatchSize] or [numGenSequences]

+
+ +
+
+TensorPtr specDecodingPackedMasks#
+

[maxBatchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] or [numGenSequences, maxDecodingTokens, ceil(maxDecodingTokens / 32)]

+
+ +
+
+TensorPtr specDecodingPositionOffsets#
+

[maxBatchSize] or [numGenSequences]

+
+ +
+
+TensorPtr eagleNetCtxRequestTypesHost#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr eagleNetCtxContextLengthsHost#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr eagleNetCtxPastKeyValueLengthsHost#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr eagleNetGenRequestTypesHost#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr eagleNetGenContextLengthsHost#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr eagleNetGenPastKeyValueLengthsHost#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr inputGenTokensHost#
+

[maxBatchSize * maxDecodingTokens] or [numSequences * maxDecodingTokens]

+
+ +
+
+TensorPtr chunkedContextNextTokens#
+

[maxBatchSize] or [numSequences]

+
+ +
+
+TensorPtr useSpecDecoding#
+

[1]

+
+ +
+
+TensorPtr useDynamicTreeHost#
+

[1]

+
+ +
+
+TensorPtr dynamicTreeMaxTopKHost#
+

[1]

+
+ +
+
+TensorPtr prevScores#
+

[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens]

+
+ +
+
+TensorPtr currentExpandIndices#
+

[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens]

+
+ +
+
+TensorPtr allLayersScores#
+

[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens]

+
+ +
+
+TensorPtr allLayersDraftTokenIds#
+

[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens]

+
+ +
+
+TensorPtr allLayersDraftTokenIdsPredecessor#
+

[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens]

+
+ +
+
+ +
+ +
+ +
+

medusaModule.h#

@@ -5859,6 +6421,5223 @@ one more than decoding draft tokens for prediction from primary head

+
+
+

virtualMemory.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+

Functions

+
+
+CudaVirtualMemoryManager &getVirtualMemoryManager()#
+
+ +
+
+CudaVirtualMemoryAllocator getVirtualMemoryAllocator()#
+
+ +
+
+void setVirtualMemoryAllocator( + +
+
std::string const &tag,
+
CudaVirtualMemoryAllocator::RestoreMode mode,
+
std::shared_ptr<CudaStream> backStream,
+
+ +)#
+
+ +
+
+void clearVirtualMemoryAllocator()#
+
+ +
+
+
+class CudaVirtualMemoryAllocator#
+
+

Public Types

+
+
+enum RestoreMode#
+

Values:

+
+
+enumerator NONE#
+
+ +
+
+enumerator MEMSET#
+
+ +
+
+enumerator CPU#
+
+ +
+
+enumerator PINNED#
+
+ +
+ +
+
+

Public Functions

+
+
+inline explicit CudaVirtualMemoryAllocator( + +
+
std::shared_ptr<Configuration> config,
+
+ +)#
+
+ +
+
+inline explicit operator bool() const noexcept#
+
+ +
+
+void allocate(Pointer *ptr, std::size_t n, int device) const#
+
+ +
+
+void deallocate(Pointer ptr, std::size_t n) const#
+
+ +
+
+

Private Types

+
+
+using CudaStreamPtr = std::shared_ptr<CudaStream>#
+
+ +
+
+using Pointer = void*#
+
+ +
+
+

Private Members

+
+
+std::shared_ptr<Configuration> mConfig#
+
+ +
+
+
+class Configuration#
+
+

Public Functions

+
+
+inline Configuration( + +
+
CudaVirtualMemoryManager &manager,
+
std::string tag,
+
RestoreMode mode,
+
CudaStreamPtr backStream,
+
+ +)#
+

CudaVirtualMemoryAllocator::Configuration

+
+
Parameters:
+
    +

  • manager – Manager used to track and manage virtual memories

    • +

  • tag – The tag for allocated memories

    • +

  • mode – Backed storage mode

    • +

  • backStream – The CUDA stream used for restoring memory content Note: Virtual Address Allocation is not async. The stream is not used in allocation.

    • +
+
+
+
+ +
+
+inline std::size_t pageAligned(std::size_t n) const noexcept#
+
+ +
+
+

Public Static Attributes

+
+
+static Configuration backgroundConfiguration#
+
+ +
+
+

Private Functions

+
+
+inline Configuration( + +
+
CudaVirtualMemoryManager &manager,
+
std::string tag,
+
RestoreMode mode,
+
CudaStreamPtr backStream,
+
bool background,
+
+ +)#
+
+ +
+
+

Private Members

+
+
+CudaVirtualMemoryManager &mManager#
+
+ +
+
+std::string mTag#
+
+ +
+
+CudaStreamPtr mBackStream#
+
+ +
+
+std::size_t mPageSize#
+
+ +
+
+RestoreMode mMode#
+
+ +
+
+bool mBackground = {}#
+
+ +
+
+

Friends

+
+
+friend class CudaVirtualMemoryAllocator
+
+ +
+
+friend void setVirtualMemoryAllocator( + +
+
std::string const &tag,
+
RestoreMode mode,
+
std::shared_ptr<CudaStream> backStream,
+
+ +)#
+
+ +
+
+ +
+ +
+
+class CUDAVirtualMemoryChunk#
+
+#include <virtualMemory.h>
+

CUDAVirtualMemoryChunk is a handle to a piece of CUDA memory allocation, providing the ability to release and rematerialize the allocation.

+
+

Public Types

+
+
+enum Status#
+

Values:

+
+
+enumerator INVALID#
+
+ +
+
+enumerator RELEASED#
+
+ +
+
+enumerator MATERIALIZED#
+
+ +
+
+enumerator ERRORED#
+
+ +
+ +
+
+using CreatorPtr = std::unique_ptr<Creator>#
+
+ +
+
+using ConfiguratorPtr = std::unique_ptr<Configurator>#
+
+ +
+
+using Configurators = std::vector<ConfiguratorPtr>#
+
+ +
+
+

Public Functions

+
+
+inline Status status() const noexcept#
+
+ +
+
+void materialize()#
+

Materialize this CUDAVirtualMemoryChunk. Shall be called only when status() == RELEASED.

+

Calls creator.create(), and then configurator.setup() for each configurator in order.

+

Stop at the first thrown exception and propagates it.

+
+ +
+
+inline void release()#
+

Release this CUDAVirtualMemoryChunk. Shall be called only when status() == MATERIALIZED, or materialize() throws. Will be called automatically by destructor if necessary.

+

Calls configurator.teardown() for each configurator that setup() succeed in materialize() in reversed order, and then creator.release().

+

Never stops early upon exception. The last thrown exception will be propagated, and others logged.

+
+ +
+
+CUDAVirtualMemoryChunk(CUDAVirtualMemoryChunk const&) = delete#
+
+ +
+
+CUDAVirtualMemoryChunk &operator=( + +
+
CUDAVirtualMemoryChunk const&,
+
+ +) = delete#
+
+ +
+
+inline CUDAVirtualMemoryChunk( + +
+
CUDAVirtualMemoryChunk &&other,
+
+ +) noexcept#
+
+ +
+
+inline CUDAVirtualMemoryChunk &operator=( + +
+
CUDAVirtualMemoryChunk &&other,
+
+ +)#
+
+ +
+
+CUDAVirtualMemoryChunk() noexcept = default#
+
+ +
+
+inline CUDAVirtualMemoryChunk( + +
+
CreatorPtr &&creator,
+
Configurators &&configurators,
+
+ +)#
+
+ +
+
+inline virtual ~CUDAVirtualMemoryChunk()#
+
+ +
+
+inline explicit operator bool() const noexcept#
+

Test if this CUDAVirtualMemoryChunk is managing a memory block.

+
+ +
+
+

Private Functions

+
+
+void _release(bool destructing)#
+
+ +
+
+

Private Members

+
+
+size_t mState = 0#
+
+ +
+
+CUmemGenericAllocationHandle mHandle = {}#
+
+ +
+
+std::unique_ptr<Creator> mCreator#
+
+ +
+
+std::vector<std::unique_ptr<Configurator>> mConfigurators#
+
+ +
+
+

Private Static Attributes

+
+
+static constexpr size_t INVALID_STATE = static_cast<size_t>(-1)#
+
+ +
+
+
+struct Configurator#
+
+#include <virtualMemory.h>
+

CUDAVirtualMemoryChunk::Configurator is the interface to configure a CUmemGenericAllocationHandle:

    +

  • Map into virtual address

    • +

  • Bind to multicast object

    • +

  • Backup and restore memory content

    • +
+

+

Subclassed by tensorrt_llm::runtime::MemsetConfigurator, tensorrt_llm::runtime::MulticastConfigurator, tensorrt_llm::runtime::OffloadConfigurator, tensorrt_llm::runtime::UnicastConfigurator

+
+

Public Functions

+
+
+Configurator() = default#
+
+ +
+
+virtual ~Configurator() = default#
+
+ +
+
+Configurator(Configurator const&) = default#
+
+ +
+
+Configurator &operator=(Configurator const&) = default#
+
+ +
+
+Configurator(Configurator&&) = default#
+
+ +
+
+Configurator &operator=(Configurator&&) = default#
+
+ +
+
+virtual void setup(CUmemGenericAllocationHandle handle) = 0#
+
+ +
+
+virtual void teardown( + +
+
CUmemGenericAllocationHandle handle,
+
bool destructing,
+
+ +) = 0#
+
+ +
+
+ +
+
+struct Creator#
+
+#include <virtualMemory.h>
+

CUDAVirtualMemoryChunk::Creator is the interface to obtain a CUmemGenericAllocationHandle, either by creating one locally, or importing one from remote.

+

Subclassed by tensorrt_llm::runtime::LocalCreator< count >

+
+

Public Functions

+
+
+Creator() = default#
+
+ +
+
+virtual ~Creator() = default#
+
+ +
+
+Creator(Creator const&) = default#
+
+ +
+
+Creator &operator=(Creator const&) = default#
+
+ +
+
+Creator(Creator&&) = default#
+
+ +
+
+Creator &operator=(Creator&&) = default#
+
+ +
+
+virtual CUmemGenericAllocationHandle create() = 0#
+
+ +
+
+virtual void release( + +
+
CUmemGenericAllocationHandle handle,
+
bool destructing,
+
+ +) = 0#
+
+ +
+
+ +
+ +
+
+class CudaVirtualMemoryManager#
+
+

Public Functions

+
+
+void add( + +
+
uintptr_t handle,
+
std::string tag,
+
CUDAVirtualMemoryChunk &&memory,
+
+ +)#
+

Add memory to be managed by this manager.

+

The memory and internal state will remain valid if any exception is thrown.

+

+
+
Parameters:
+
    +

  • handle – Unique handle provided to reference this memory in remove.

    • +

  • tag – Tag the memory, so this memory can be targeted in releaseWithTag and materializeWithTag.

    • +

  • memory – The CUDAVirtualMemory object.

    • +
+
+
+
+ +
+
+void add( + +
+
uintptr_t handle,
+
std::string tag,
+
CUDAVirtualMemoryChunk::CreatorPtr &&creator,
+
CUDAVirtualMemoryChunk::Configurators &&configurators,
+
+ +)#
+

Creates and adds memory to be managed by this manager. The created memory is automatically materialized.

+

The internal state will remain valid if any exception is thrown.

+

+
+
Parameters:
+
    +

  • handle – Unique handle provided to reference this memory in remove.

    • +

  • tag – Tag the memory, so this memory can be targeted in releaseWithTag and materializeWithTag.

    • +

  • creator – The creator for the memory.

    • +

  • configurators – The configurators for the memory.

    • +
+
+
+
+ +
+
+template<typename ...Configurators>
inline void add( + +
+
uintptr_t handle,
+
std::string tag,
+
CUDAVirtualMemoryChunk::CreatorPtr &&creator,
+
Configurators&&... configurators,
+
+ +)#
+
+ +
+
+CUDAVirtualMemoryChunk remove(uintptr_t handle) noexcept#
+

Remove the memory from the manager.

+
+
Parameters:
+

handle – The handle provided to add.

+
+
Returns:
+

The CUDAVirtualMemory object. If the handle is unknown, an empty CUDAVirtualMemory will be returned.

+
+
+
+ +
+
+size_t releaseWithTag(std::string const &tag)#
+

Call release for CUDAVirtualMemoryChunk

objects with a given tag.

+

This function will always call

+CUDAVirtualMemoryChunk::release on all selected objects. The last exception thrown by CUDAVirtualMemoryChunk::release will be rethrown, and others will be logged.

+

If any CUDAVirtualMemoryChunk threw an exception during release, it will be removed from the manager. Call retrieveBadHandles to retrieve handles of all CUDAVirtualMemoryChunk that got removed due to exception.

+
+
Parameters:
+

tag – the tag to select target memories.

+
+
Returns:
+

Number of objects selected.

+
+
+
+ +
+
+size_t materializeWithTag(std::string const &tag)#
+

Call materialize for CUDAVirtualMemoryChunk

objects with a given tag.

+

This function will stop at the first

+CUDAVirtualMemoryChunk::materialize that throws exception, and attempt to roll back previous successful materialize by calling release. The exception thrown by CUDAVirtualMemoryChunk::materialize will be rethrown, and any exception thrown by release will be logged.

+

If any CUDAVirtualMemoryChunk threw an exception during materialize or release, it will be removed from the manager. Successfully roll backed CUDAVirtualMemoryChunk will not be removed. Call retrieveBadHandles to retrieve handles of all CUDAVirtualMemoryChunk that got removed due to exception.

+
+
Parameters:
+

tag – the tag to select target memories.

+
+
Returns:
+

Number of objects selected.

+
+
+
+ +
+
+std::vector<uintptr_t> retrieveBadHandles() noexcept#
+

Retrieve handles of all CUDAVirtualMemoryChunk that got removed due to exception and reset the list. The returned list may not include all removed CUDAVirtualMemoryChunk handles if OOM happened. This method is only for diagnostic purpose, and should not be called concurrently with other methods.

+
+
Returns:
+

The handle list.

+
+
+
+ +
+
+

Private Types

+
+
+using PointerMemoryMap = std::map<uintptr_t, Entry>#
+
+ +
+
+using TagEntryMap = std::multimap<std::string, PointerMemoryMap::iterator>#
+
+ +
+
+

Private Functions

+
+
+CUDAVirtualMemoryChunk unsafeRemove(uintptr_t handle) noexcept#
+
+ +
+
+void addBadHandle(uintptr_t handle) noexcept#
+
+ +
+
+

Private Members

+
+
+std::mutex mMutex#
+
+ +
+
+PointerMemoryMap mMemories#
+
+ +
+
+TagEntryMap mEntries#
+
+ +
+
+std::vector<uintptr_t> mBadHandles#
+
+ +
+
+friend VirtualMemoryManagerTest
+
+ +
+
+
+struct Entry#
+
+

Public Members

+
+
+CUDAVirtualMemoryChunk mMemory#
+
+ +
+
+TagEntryMap::iterator mEntryIt#
+
+ +
+
+ +
+ +
+
+template<bool count = true>
struct LocalCreator : public tensorrt_llm::runtime::CUDAVirtualMemoryChunk::Creator#
+
+#include <virtualMemory.h>
+

LocalCreator creates memory allocation locally through cuMemCreate.

+
+

Public Functions

+
+
+inline LocalCreator(CUmemAllocationProp const &prop, size_t size)#
+
+ +
+
+inline virtual CUmemGenericAllocationHandle create() override#
+
+ +
+
+inline virtual void release( + +
+
CUmemGenericAllocationHandle handle,
+
bool destructing,
+
+ +) override#
+
+ +
+
+

Public Members

+
+
+CUmemAllocationProp mProp = {}#
+
+ +
+
+size_t mSize = {}#
+
+ +
+
+ +
+
+struct MemsetConfigurator : public tensorrt_llm::runtime::CUDAVirtualMemoryChunk::Configurator#
+
+#include <virtualMemory.h>
+

MemsetConfigurator fills the memory with given value.

+
+

Public Functions

+
+
+inline MemsetConfigurator( + +
+
CUdeviceptr address,
+
size_t size,
+
uint8_t value,
+
CUstream stream,
+
+ +)#
+
+ +
+
+inline virtual void setup(CUmemGenericAllocationHandle) override#
+
+ +
+
+inline virtual void teardown( + +
+
CUmemGenericAllocationHandle,
+
bool,
+
+ +) noexcept override#
+
+ +
+
+

Public Members

+
+
+CUdeviceptr mAddress#
+
+ +
+
+size_t mSize#
+
+ +
+
+CUstream mStream = {}#
+
+ +
+
+uint8_t mValue#
+
+ +
+
+bool mFirstTime = true#
+
+ +
+
+ +
+
+struct MulticastConfigurator : public tensorrt_llm::runtime::CUDAVirtualMemoryChunk::Configurator#
+
+#include <virtualMemory.h>
+

MulticastConfigurator binds the allocation handle to the given multicast object and offset.

+
+

Public Functions

+
+
+inline virtual void setup( + +
+
CUmemGenericAllocationHandle handle,
+
+ +) override#
+
+ +
+
+inline virtual void teardown( + +
+
CUmemGenericAllocationHandle,
+
bool,
+
+ +) override#
+
+ +
+
+

Public Members

+
+
+CUmemGenericAllocationHandle mMulticast#
+
+ +
+
+size_t mBindOffset#
+
+ +
+
+CUdevice mDevice#
+
+ +
+
+size_t mSize#
+
+ +
+
+ +
+
+struct OffloadConfigurator : public tensorrt_llm::runtime::CUDAVirtualMemoryChunk::Configurator#
+
+#include <virtualMemory.h>
+

OffloadConfigurator offload the content of the allocation to the backup storage when teardown, and restore the content on the following setup.

+
+

Public Functions

+
+
+inline OffloadConfigurator( + +
+
CUdeviceptr address,
+
size_t size,
+
MemoryType backType,
+
CUstream stream,
+
bool ondemand = false,
+
+ +)#
+
+ +
+
+virtual void setup(CUmemGenericAllocationHandle handle) override#
+
+ +
+
+virtual void teardown( + +
+
CUmemGenericAllocationHandle handle,
+
bool destructing,
+
+ +) override#
+
+ +
+
+

Public Members

+
+
+CUdeviceptr mAddress#
+
+ +
+
+size_t mSize#
+
+ +
+
+MemoryType mBackType#
+
+ +
+
+CUstream mStream#
+
+ +
+
+bool mOndemand#
+
+ +
+
+IBuffer::UniquePtr mBackedStorage#
+
+ +
+
+ +
+
+struct UnicastConfigurator : public tensorrt_llm::runtime::CUDAVirtualMemoryChunk::Configurator#
+
+#include <virtualMemory.h>
+

UnicastConfigurator maps the allocation handle into the specified unicast address range.

+
+

Public Functions

+
+
+inline UnicastConfigurator( + +
+
CUdeviceptr address,
+
size_t size,
+
CUmemAccessDesc const &desc,
+
+ +)#
+
+ +
+
+inline virtual void setup( + +
+
CUmemGenericAllocationHandle handle,
+
+ +) override#
+
+ +
+
+inline virtual void teardown( + +
+
CUmemGenericAllocationHandle,
+
bool,
+
+ +) override#
+
+ +
+
+

Public Members

+
+
+CUdeviceptr mAddress#
+
+ +
+
+size_t mSize#
+
+ +
+
+CUmemAccessDesc mDesc#
+
+ +
+
+ +
+ +
+ +
+
+

explicitDraftTokensBuffers.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class ExplicitDraftTokensBuffers#
+
+

Public Types

+
+
+using SizeType32 = runtime::SizeType32#
+
+ +
+
+using ITensor = runtime::ITensor#
+
+ +
+
+using BufferPtr = runtime::IBuffer::SharedPtr#
+
+ +
+
+using TensorPtr = runtime::ITensor::SharedPtr#
+
+ +
+
+using TensorMap = runtime::StringPtrMap<runtime::ITensor>#
+
+ +
+
+

Public Functions

+
+
+ExplicitDraftTokensBuffers( + +
+
SizeType32 maxBatchSize,
+
SizeType32 maxBeamWidth,
+
runtime::BufferManager const &manager,
+
runtime::ModelConfig const &modelConfig,
+
runtime::WorldConfig const &worldConfig,
+
+ +)#
+
+ +
+
+void reshape( + +
+
SizeType32 numCtxSequences,
+
SizeType32 numGenSequences,
+
runtime::ModelConfig const &modelConfig,
+
+ +)#
+
+ +
+
+void setFromInputs( + +
+
SizeType32 numCtxSequences,
+
SizeType32 numGenSequences,
+
runtime::ITensor const &requestTypes,
+
ITensor const &seqSlots,
+
ExplicitDraftTokensBuffers::Inputs const &decoderBuffers,
+
ITensor const &contextPositionIds,
+
runtime::ModelConfig const &modelConfig,
+
runtime::WorldConfig const &worldConfig,
+
runtime::BufferManager const &manager,
+
runtime::CudaStream const &stream,
+
+ +) const#
+
+ +
+
+void insertInputTensors( + +
+
TensorMap &inputBuffers,
+
TensorMap &outputBuffers,
+
runtime::WorldConfig const &worldConfig,
+
+ +) const#
+
+ +
+
+

Public Members

+
+
+tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs engineInputs#
+
+ +
+
+class tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs engineOutputs#
+
+ +
+
+std::size_t scanTempStorageBytes = {0}#
+
+ +
+
+BufferPtr scanTempStorage#
+
+ +
+
+TensorPtr cumSumGenerationLengths#
+
+ +
+
+

Private Functions

+
+
+template<typename T>
void setFromInputs( + +
+
SizeType32 numCtxSequences,
+
SizeType32 numGenSequences,
+
SizeType32 vocabSizePadded,
+
ITensor const &seqSlots,
+
ExplicitDraftTokensBuffers::Inputs const &draftBuffers,
+
ITensor const &contextPositionIds,
+
runtime::ExplicitDraftTokensModule const &explicitDraftTokensModule,
+
runtime::CudaStream const &stream,
+
+ +) const#
+
+ +
+
+
+class EngineInputs : public tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs#
+
+

Public Members

+
+
+TensorPtr requestTypesDevice#
+

[numSequences], on gpu

+
+ +
+
+TensorPtr positionOffsets#
+

[numGenSequences]

+
+ +
+
+ +
+
+class EngineOutputs#
+
+

Public Members

+
+
+TensorPtr nextGenerationLengths#
+

[batchSize]

+
+ +
+
+TensorPtr nextPositionOffsets#
+

[batchSize]

+
+ +
+
+TensorPtr masks#
+

[batchSize, maxDecodingTokens, maxDecodingTokens], bool

+
+ +
+
+TensorPtr nextDraftTokens#
+

[batchSize, maxNumPaths, maxPathLen]

+
+ +
+
+TensorPtr nextDraftIndices#
+

[batchSize, maxNumPaths, maxPathLen]

+
+ +
+
+TensorPtr nextDraftProbs#
+

[batchSize, maxNumPaths, maxDraftPathLen, vocabSize]

+
+ +
+
+TensorPtr nextFlatTokens#
+

[batchSize * maxDecodingTokens]

+
+ +
+
+TensorPtr bestPathLengths#
+

[batchSize]

+
+ +
+
+TensorPtr bestPathIndices#
+

[batchSize]

+
+ +
+
+TensorPtr maxGenToken#
+

[1]

+
+ +
+
+TensorPtr totalGenToken#
+

[1]

+
+ +
+
+TensorPtr packedPositionIds#
+

[batchSize * maxDecodingTokens]

+
+ +
+
+ +
+
+class Inputs#
+

Subclassed by tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs

+
+

Public Functions

+
+
+void create( + +
+
SizeType32 maxNumSequences,
+
runtime::BufferManager const &manager,
+
runtime::ModelConfig const &modelConfig,
+
runtime::WorldConfig const &worldConfig,
+
+ +)#
+
+ +
+
+

Public Members

+
+
+TensorPtr temperatures#
+

[maxBatchSize]

+
+ +
+
+TensorPtr positionIdsBase#
+

[maxBatchSize]

+
+ +
+
+TensorPtr generationLengths#
+

[maxBatchSize] or [numGenSequences]

+
+ +
+
+TensorPtr randomDataSample#
+

[maxBatchSize]

+
+ +
+
+TensorPtr randomDataValidation#
+

[maxBatchSize, maxNumPaths, maxPathDraftLen] or [numGenSequences, maxNumPaths, maxPathDraftLen]

+
+ +
+
+TensorPtr draftTokens#
+

[maxBatchSize, maxNumPaths, maxPathLen] or [numGenSequences, maxNumPaths, maxPathLen]

+
+ +
+
+TensorPtr draftIndices#
+

[maxBatchSize, maxNumPaths, maxPathLen] or [numGenSequences, maxNumPaths, maxPathLen]

+
+ +
+
+TensorPtr draftProbs#
+

[maxBatchSize, maxNumPaths, maxPathDraftLen, vocabSize] or [numGenSequences, maxNumPaths, maxPathDraftLen, vocabSize]

+
+ +
+
+TensorPtr packedMasks#
+

[maxBatchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] or [numGenSequences, maxDecodingTokens, ceil(maxDecodingTokens / 32)]

+
+ +
+
+TensorPtr positionIds#
+

[maxBatchSize] or [numGenSequences]

+
+ +
+
+TensorPtr maxGenLengthHost#
+
+ +
+
+TensorPtr generationLengthsHost#
+
+ +
+
+TensorPtr useSpecDecoding#
+
+ +
+
+ +
+ +
+ +
+ +
+
+

iTensor.h#

+
+
+namespace nvinfer1#
+
+ +
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+

Functions

+
+
+inline std::ostream &operator<<( + +
+
std::ostream &output,
+
ITensor::Shape const &dims,
+
+ +)#
+

Utility function to print a shape.

+
+ +
+
+std::ostream &operator<<( + +
+
std::ostream &output,
+
ITensor const &tensor,
+
+ +)#
+

Utility function to print a tensor with its shape.

+
+ +
+
+template<typename T>
T const *bufferCastOrNull( + +
+
ITensor::SharedConstPtr const &tensorPtr,
+
+ +)#
+

Retrieves a T const typed pointer to the underlying data of the tensor pointed to by the tensorPtr, or nullptr if the tensorPtr is null.

+

This overload has to be declared to avoid ambiguity when an implicit conversion to IBuffer is involved.

+
+
Template Parameters:
+

T – The type of the underlying data.

+
+
Parameters:
+

tensorPtr – A possibly null shared ptr.

+
+
Returns:
+

A pointer to T const, possibly nullptr.

+
+
+
+ +
+
+template<typename T>
T *bufferCastOrNull( + +
+
ITensor::SharedPtr const &tensorPtr,
+
+ +)#
+

Retrieves a T typed pointer to the underlying data of the buffer pointed to by the tensorPtr, or nullptr if the tensorPtr is null.

+

This overload has to be declared to avoid ambiguity when an implicit conversion to IBuffer is involved.

+
+
Template Parameters:
+

T – The type of the underlying data.

+
+
Parameters:
+

tensorPtr – A possibly null shared ptr.

+
+
Returns:
+

A pointer to T, possibly nullptr.

+
+
+
+ +
+
+template<typename T>
T *bufferCastOrNull( + +
+
std::optional<ITensor::SharedPtr> const &optionalTensorPtr,
+
+ +)#
+

Retrieves a T typed pointer to the underlying data of the tensor pointed to by the tensor pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value.

+

This overload has to be declared to avoid ambiguity when an implicit conversion to IBuffer is involved.

+
+
Template Parameters:
+

T – The type of the underlying data.

+
+
Parameters:
+

optionalBufferPtr – A possibly empty optional.

+
+
Returns:
+

A pointer to T, possibly nullptr.

+
+
+
+ +
+
+template<typename T>
T const *bufferCastOrNull( + +
+
std::optional<ITensor::SharedConstPtr> const &optionalTensorPtr,
+
+ +)#
+

Retrieves a T const typed pointer to the underlying data of the tensor pointed to by the tensor pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value.

+

This overload has to be declared to avoid ambiguity when an implicit conversion to IBuffer is involved.

+
+
Template Parameters:
+

T – The type of the underlying data.

+
+
Parameters:
+

optionalBufferPtr – A possibly empty optional.

+
+
Returns:
+

A pointer to const T, possibly nullptr.

+
+
+
+ +
+
+
+class ITensor : public virtual tensorrt_llm::runtime::IBuffer#
+
+

Public Types

+
+
+using UniquePtr = std::unique_ptr<ITensor>#
+
+ +
+
+using SharedPtr = std::shared_ptr<ITensor>#
+
+ +
+
+using UniqueConstPtr = std::unique_ptr<ITensor const>#
+
+ +
+
+using SharedConstPtr = std::shared_ptr<ITensor const>#
+
+ +
+
+using Shape = nvinfer1::Dims#
+
+ +
+
+using DimType64 = std::remove_reference_t<decltype(Shape::d[0])>#
+
+ +
+
+using TensorMap = runtime::StringPtrMap<runtime::ITensor>#
+
+ +
+
+

Public Functions

+
+
+~ITensor() override = default#
+
+ +
+
+virtual Shape const &getShape() const = 0#
+

Returns the tensor dimensions.

+
+ +
+
+template<SizeType32 n>
inline DimType64 getDimension() const#
+

Returns the tensor n-th dimension. If n is negative, returns the (nbDims - n)th dimension. TODO: replace with constexpr parameter when moving to C++20.

+
+ +
+
+virtual void reshape(Shape const &dims) = 0#
+

Sets the tensor dimensions. The new size of the tensor will be volume(dims)

+
+ +
+
+inline virtual void resize(std::size_t newSize) override#
+

Resizes the buffer. This is a no-op if the new size is smaller than or equal to the current capacity.

+
+ +
+
+ITensor(ITensor const&) = delete#
+

Not allowed to copy.

+
+ +
+
+ITensor &operator=(ITensor const&) = delete#
+

Not allowed to copy.

+
+ +
+
+inline void squeeze(SizeType32 dim)#
+

Removes the given unit dimensions from this tensor.

+
+ +
+
+inline void unsqueeze(SizeType32 dim)#
+

Adds a unit dimension at the specified position.

+
+ +
+
+inline bool shapeEquals(Shape const &other) const#
+
+ +
+
+inline bool shapeEquals( + +
+
std::initializer_list<SizeType32> const &other,
+
+ +) const#
+
+ +
+
+template<typename T>
inline bool shapeEquals( + +
+
T const *dims,
+
SizeType32 count,
+
+ +) const#
+
+ +
+
+

Public Static Functions

+
+
+static inline std::int64_t volume(Shape const &dims)#
+

Returns the volume of the dimensions. Returns -1 if d.nbDims < 0.

+
+ +
+
+static inline std::size_t volumeNonNegative(Shape const &shape)#
+

Returns the volume of the dimensions. Throws if d.nbDims < 0.

+
+ +
+
+static inline Shape strides(Shape const &dims)#
+

Returns the strides of each dimemsion in a Shape.

+
+ +
+
+static Shape squeeze(Shape const &shape, SizeType32 dim)#
+

Removes the given unit dimension from shape.

+
+
Parameters:
+
    +

  • shape – The shape to squeeze.

    • +

  • dim – The dimension that should be removed (“squeezed”).

    • +
+
+
Returns:
+

A new shape without the unit dimension.

+
+
+
+ +
+
+static Shape unsqueeze(Shape const &shape, SizeType32 dim)#
+

Add a unit dimension to shape at the specified position.

+
+
Parameters:
+
    +

  • shape – The shape to unsqueeze.

    • +

  • dim – The dimension where unit dimension should be added.

    • +
+
+
Returns:
+

A new shape with the added unit dimension.

+
+
+
+ +
+
+static UniquePtr slice( + +
+
SharedPtr tensor,
+
std::size_t offset,
+
std::size_t size,
+
+ +)#
+

Creates a sliced view on the underlying tensor. The view will have the same data type as tensor.

+
+
Parameters:
+
    +

  • tensor – The tensor to view.

    • +

  • offset – The offset of the view w.r.t. dimension 0 of the tensor.

    • +

  • size – The size of the view w.r.t. dimension 0 of the tensor.

    • +
+
+
Returns:
+

A view on the buffer.

+
+
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( + +
+
TConstPtr &&tensor,
+
std::size_t offset,
+
std::size_t size,
+
+ +)#
+
+ +
+
+static inline UniquePtr slice(SharedPtr tensor, std::size_t offset)#
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( + +
+
TConstPtr &&tensor,
+
std::size_t offset,
+
+ +)#
+
+ +
+
+static UniquePtr slice( + +
+
SharedPtr tensor,
+
Shape const &offsetDims,
+
DimType64 size,
+
+ +)#
+
+
Parameters:
+
    +

  • offsetDims – The offset in multiple dimensions.

    • +

  • tensor – The tensor to view.

    • +

  • offsetDims – The offset dimensions of the view.

    • +

  • size – The size of the view w.r.t. the last dimension in offsetDims.

    • +

  • offsetDims – specifies all dimensions.

    • +
+
+
Throws:
+

Whenever – offset overflows or the last dimension offset+size overflows.

+
+
Returns:
+

A view of shape [size, the rest dimensions] or [size] when

+
+
+
+ +
+
+static inline UniquePtr slice( + +
+
SharedPtr tensor,
+
std::initializer_list<DimType64> const &offsetDims,
+
DimType64 size,
+
+ +)#
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( + +
+
TConstPtr &&tensor,
+
Shape const &offsetDims,
+
std::size_t size,
+
+ +)#
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( + +
+
TConstPtr &&tensor,
+
std::initializer_list<DimType64> const &offsetDims,
+
std::size_t size,
+
+ +)#
+
+ +
+
+static inline UniquePtr slice( + +
+
SharedPtr tensor,
+
Shape const &offsetDims,
+
+ +)#
+

return the rest slices at the last dimension when size omitted.

+
+ +
+
+static inline UniquePtr slice( + +
+
SharedPtr tensor,
+
std::initializer_list<DimType64> const &offsetDims,
+
+ +)#
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( + +
+
TConstPtr &&tensor,
+
Shape const &offsetDims,
+
+ +)#
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( + +
+
TConstPtr &&tensor,
+
std::initializer_list<DimType64> const &offsetDims,
+
+ +)#
+
+ +
+
+static inline UniquePtr at(SharedPtr tensor, Shape const &offsetDims)#
+
+
Parameters:
+

offsetDims – specifies all dimensions.

+
+
Returns:
+

Just the block at the point, with shape of [the rest dimensions] or [1] when

+
+
+
+ +
+
+static inline UniquePtr at( + +
+
SharedPtr tensor,
+
std::initializer_list<DimType64> const &offsetDims,
+
+ +)#
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr at( + +
+
TConstPtr &&tensor,
+
Shape const &offsetDims,
+
+ +)#
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline ITensor::UniqueConstPtr at( + +
+
TConstPtr &&tensor,
+
std::initializer_list<DimType64> const &offsetDims,
+
+ +)#
+
+ +
+
+static UniquePtr view(IBuffer::SharedPtr buffer, Shape const &dims)#
+

Returns a view on the underlying buffer (or tensor) with the given shape.

+
+
Parameters:
+
    +

  • tensor – The tensor to view.

    • +

  • shape – The shape of the view.

    • +
+
+
Returns:
+

A view on the tensor.

+
+
+
+ +
+
+template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr view( + +
+
TConstPtr &&tensor,
+
Shape const &dims,
+
+ +)#
+
+ +
+
+static inline UniquePtr view(SharedPtr tensor)#
+

Returns a view on the underlying tensor which can be independently reshaped.

+
+
Parameters:
+

tensor – The tensor to view.

+
+
Returns:
+

A view on the tensor.

+
+
+
+ +
+
+static inline UniquePtr flattenN( + +
+
SharedPtr tensor,
+
std::int64_t sliceN = -1,
+
+ +)#
+

Returns a flattened view on the underlying tensor which can be independently reshaped.

+
+
Parameters:
+
    +

  • tensor – The tensor to flatten.

    • +

  • sliceN – Slice the first N elements after flattening. -1 means take the whole flattened tensor.

    • +
+
+
Returns:
+

A flatten view on the tensor.

+
+
+
+ +
+
+static UniquePtr wrap( + +
+
void *data,
+
nvinfer1::DataType type,
+
Shape const &shape,
+
std::size_t capacity,
+
+ +)#
+

Wraps the given data in an ITensor. The ITensor will not own the underlying data and cannot be reshaped beyond capacity.

+
+
Parameters:
+
    +

  • data – The data to wrap.

    • +

  • type – The data type of the data.

    • +

  • shape – The shape of the tensor.

    • +

  • capacity – The capacity of the buffer.

    • +
+
+
Returns:
+

An ITensor.

+
+
+
+ +
+
+static inline UniquePtr wrap( + +
+
void *data,
+
nvinfer1::DataType type,
+
Shape const &shape,
+
+ +)#
+
+ +
+
+template<typename T>
static inline UniquePtr wrap( + +
+
T *data,
+
Shape const &shape,
+
std::size_t capacity,
+
+ +)#
+
+ +
+
+template<typename T>
static inline UniquePtr wrap( + +
+
T *data,
+
Shape const &shape,
+
+ +)#
+
+ +
+
+template<typename T>
static inline UniquePtr wrap( + +
+
std::vector<T> &v,
+
Shape const &shape,
+
+ +)#
+
+ +
+
+static Shape makeShape( + +
+
std::initializer_list<DimType64> const &dims,
+
+ +)#
+

A convenience function to create a tensor shape with the given dimensions.

+
+ +
+
+static std::string toString(Shape const &dims)#
+

A convenience function for converting a tensor shape to a string.

+
+ +
+
+static inline bool shapeEquals(Shape const &lhs, Shape const &rhs)#
+

A convenience function to compare shapes.

+
+ +
+
+template<typename T>
static inline bool shapeEquals( + +
+
Shape const &lhs,
+
T const *dims,
+
SizeType32 count,
+
+ +)#
+

A convenience function to compare shapes.

+
+ +
+
+

Protected Functions

+
+
+ITensor() = default#
+
+ +
+
+

Protected Static Functions

+
+
+static inline DimType64 castSize(size_t newSize)#
+
+ +
+
+

Friends

+
+
+friend class ITensorBindings
+
+ +
+
+ +
+ +
+ +
+
+

common.h#

+
+

Defines

+
+
+FMT_DIM#
+
+ +
+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+

Typedefs

+
+
+using SizeType32 = std::int32_t#
+
+ +
+
+using SizeType64 = std::int64_t#
+
+ +
+
+using TokenIdType = std::int32_t#
+
+ +
+
+using LoraTaskIdType = std::uint64_t#
+
+ +
+
+using TokenExtraIdType = std::uint64_t#
+
+ +
+
+using VecTokenExtraIds = std::vector<TokenExtraIdType>#
+
+ +
+
+using CacheSaltIDType = std::uint64_t#
+
+ +
+
+using VecUniqueTokens = std::vector<UniqueToken>#
+
+ +
+
+template<typename T>
using StringPtrMap = std::unordered_map<std::string, std::shared_ptr<T>>#
+
+ +
+
+

Enums

+
+
+enum class RequestType : std::int32_t#
+

Values:

+
+
+enumerator kCONTEXT#
+
+ +
+
+enumerator kGENERATION#
+
+ +
+ +
+
+
+struct UniqueToken#
+
+

Public Functions

+
+
+inline bool operator==(UniqueToken const &other) const noexcept#
+
+ +
+
+

Public Members

+
+
+TokenIdType tokenId#
+
+ +
+
+TokenExtraIdType tokenExtraId#
+
+ +
+
+ +
+ +
+ +
+
+

loraCachePageManagerConfig.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+

Functions

+
+
+inline std::ostream &operator<<( + +
+
std::ostream &os,
+
LoraCachePageManagerConfig const &c,
+
+ +)#
+
+ +
+
+inline std::string to_string(LoraCachePageManagerConfig const &c)#
+
+ +
+
+
+class LoraCachePageManagerConfig#
+
+#include <loraCachePageManagerConfig.h>
+

Configuration for LoraCachePageManager

+

See LoraCache docs for description of pages, slots, and page blocks.

+
+

Public Functions

+
+
+inline explicit constexpr LoraCachePageManagerConfig( + +
+
runtime::MemoryType memType,
+
nvinfer1::DataType dType,
+
SizeType32 totalNumPages,
+
SizeType32 maxPagesPerBlock,
+
SizeType32 slotsPerPage,
+
SizeType32 pageWidth,
+
SizeType32 numCopyStreams,
+
+ +)#
+
+ +
+
+inline runtime::MemoryType constexpr getMemoryType() const noexcept#
+
+ +
+
+inline void constexpr setMemoryType( + +
+
runtime::MemoryType const &memoryType,
+
+ +) noexcept#
+
+ +
+
+inline nvinfer1::DataType constexpr getDataType() const noexcept#
+
+ +
+
+inline void constexpr setDataType( + +
+
nvinfer1::DataType const &dtype,
+
+ +) noexcept#
+
+ +
+
+inline SizeType32 constexpr getTotalNumPages() const noexcept#
+
+ +
+
+inline void constexpr setTotalNumPage( + +
+
SizeType32 const &totalNumPages,
+
+ +) noexcept#
+
+ +
+
+inline SizeType32 constexpr getMaxPagesPerBlock() const noexcept#
+
+ +
+
+inline void constexpr setMaxPagesPerBlock( + +
+
SizeType32 const &maxPagesPerBlock,
+
+ +) noexcept#
+
+ +
+
+inline SizeType32 constexpr getSlotsPerPage() const noexcept#
+
+ +
+
+inline void constexpr setSlotsPerPage( + +
+
SizeType32 const &slotsPerPage,
+
+ +) noexcept#
+
+ +
+
+inline SizeType32 constexpr getPageWidth() const noexcept#
+
+ +
+
+inline void constexpr setPageWidth( + +
+
SizeType32 const &pageWidth,
+
+ +) noexcept#
+
+ +
+
+inline bool constexpr getInitToZero() const noexcept#
+
+ +
+
+inline void constexpr setInitToZero(bool initToZero) noexcept#
+
+ +
+
+inline SizeType32 constexpr getNumCopyStreams() const noexcept#
+
+ +
+
+inline void constexpr setNumCopyStreams( + +
+
SizeType32 numCopyStreams,
+
+ +) noexcept#
+
+ +
+
+

Private Members

+
+
+runtime::MemoryType mMemoryType#
+
+ +
+
+nvinfer1::DataType mDataType#
+
+ +
+
+SizeType32 mTotalNumPages#
+
+ +
+
+SizeType32 mMaxPagesPerBlock#
+
+ +
+
+SizeType32 mSlotsPerPage#
+
+ +
+
+SizeType32 mPageWidth#
+
+ +
+
+SizeType32 mNumCopyStreams = 1#
+
+ +
+
+bool mInitToZero#
+
+ +
+
+ +
+ +
+ +
+
+

worldConfig.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class WorldConfig#
+
+

Public Functions

+
+
+explicit WorldConfig( + +
+
SizeType32 tensorParallelism = 1,
+
SizeType32 pipelineParallelism = 1,
+
SizeType32 contextParallelism = 1,
+
SizeType32 rank = 0,
+
SizeType32 gpusPerNode = kDefaultGpusPerNode,
+
std::optional<std::vector<SizeType32>> const &deviceIds = std::nullopt,
+
bool enableAttentionDP = false,
+
+ +)#
+
+ +
+
+inline SizeType32 constexpr getSize() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getTensorParallelism() const noexcept#
+
+ +
+
+inline bool constexpr isTensorParallel() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getPipelineParallelism() const noexcept#
+
+ +
+
+inline bool constexpr isPipelineParallel() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getContextParallelism() const noexcept#
+
+ +
+
+inline bool constexpr isContextParallel() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getRank() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getGpusPerNode() const noexcept#
+
+ +
+
+inline SizeType32 getGpusPerGroup() const noexcept#
+
+ +
+
+inline SizeType32 getDevice() const noexcept#
+
+ +
+
+inline SizeType32 getDeviceOf(SizeType32 rank) const noexcept#
+
+ +
+
+inline SizeType32 constexpr getPipelineParallelRank() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getTensorParallelRank() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getContextParallelRank() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getLocalRank() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getNodeRank() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getNodeRankOf( + +
+
SizeType32 rank,
+
+ +) const noexcept#
+
+ +
+
+inline bool constexpr isFirstPipelineParallelRank() const noexcept#
+
+ +
+
+inline bool constexpr isLastPipelineParallelRank() const noexcept#
+

Is my rank the last rank in its pipeline?

+
+ +
+
+inline bool constexpr isFirstTensorParallelRank() const noexcept#
+
+ +
+
+inline bool constexpr isFirstContextParallelRank() const noexcept#
+
+ +
+
+inline SizeType32 constexpr getLastRank() const noexcept#
+
+ +
+
+inline bool constexpr enableAttentionDP() const noexcept#
+
+ +
+
+std::vector<SizeType32> getPipelineParallelGroup() const#
+
+ +
+
+std::vector<SizeType32> getTensorParallelGroup() const#
+
+ +
+
+std::vector<SizeType32> getContextParallelGroup() const#
+
+ +
+
+bool validMpiConfig() const#
+
+ +
+
+

Public Static Functions

+
+
+static WorldConfig mpi( + +
+
SizeType32 gpusPerNode = kDefaultGpusPerNode,
+
std::optional<SizeType32> tensorParallelism = std::nullopt,
+
std::optional<SizeType32> pipelineParallelism = std::nullopt,
+
std::optional<SizeType32> contextParallelism = std::nullopt,
+
std::optional<std::vector<SizeType32>> const &deviceIds = std::nullopt,
+
bool enableAttentionDP = false,
+
+ +)#
+
+ +
+
+

Public Static Attributes

+
+
+static SizeType32 constexpr kDefaultGpusPerNode = 1#
+
+ +
+
+

Private Members

+
+
+SizeType32 mTensorParallelism#
+
+ +
+
+SizeType32 mPipelineParallelism#
+
+ +
+
+SizeType32 mContextParallelism#
+
+ +
+
+SizeType32 mRank#
+
+ +
+
+SizeType32 mGpusPerNode#
+
+ +
+
+bool mEnableAttentionDP#
+
+ +
+
+std::vector<SizeType32> mDeviceIds#
+
+ +
+
+ +
+ +
+ +
+
+

loraModule.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+

Functions

+
+
+inline std::ostream &operator<<( + +
+
std::ostream &output,
+
LoraModule const &module,
+
+ +)#
+
+ +
+
+
+class LoraModule#
+
+

Public Types

+
+
+enum class ModuleType : SizeType32#
+

Values:

+
+
+enumerator kINVALID#
+
+ +
+
+enumerator kATTN_QKV#
+
+ +
+
+enumerator kATTN_Q#
+
+ +
+
+enumerator kATTN_K#
+
+ +
+
+enumerator kATTN_V#
+
+ +
+
+enumerator kATTN_DENSE#
+
+ +
+
+enumerator kMLP_H_TO_4H#
+
+ +
+
+enumerator kMLP_4H_TO_H#
+
+ +
+
+enumerator kMLP_GATE#
+
+ +
+
+enumerator kCROSS_ATTN_QKV#
+
+ +
+
+enumerator kCROSS_ATTN_Q#
+
+ +
+
+enumerator kCROSS_ATTN_K#
+
+ +
+
+enumerator kCROSS_ATTN_V#
+
+ +
+
+enumerator kCROSS_ATTN_DENSE#
+
+ +
+
+enumerator kMOE_H_TO_4H#
+
+ +
+
+enumerator kMOE_4H_TO_H#
+
+ +
+
+enumerator kMOE_GATE#
+
+ +
+
+enumerator kMOE_ROUTER#
+
+ +
+
+enumerator kMLP_ROUTER#
+
+ +
+
+enumerator kMLP_GATE_UP#
+
+ +
+ +
+
+using TensorPtr = ITensor::SharedPtr#
+
+ +
+
+

Public Functions

+
+
+inline explicit constexpr LoraModule( + +
+
ModuleType const &t,
+
SizeType32 inDim,
+
SizeType32 outDim,
+
bool inDimFirst,
+
bool outDimFirst,
+
SizeType32 inTpSplitDim,
+
SizeType32 outTpSplitDim,
+
+ +) noexcept#
+
+ +
+
+inline explicit constexpr LoraModule() noexcept#
+
+ +
+
+explicit constexpr LoraModule(LoraModule const &o) = default#
+
+ +
+
+constexpr LoraModule &operator=(LoraModule const &o) = default#
+
+ +
+
+inline SizeType32 constexpr flattenedInOutSize( + +
+
SizeType32 adapterSize,
+
bool isDora,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr inSize( + +
+
SizeType32 adapterSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr outSize( + +
+
SizeType32 adapterSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localInSize( + +
+
SizeType32 adapterSize,
+
SizeType32 tpSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localOutSize( + +
+
SizeType32 adapterSize,
+
SizeType32 tpSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localScalesSize( + +
+
SizeType32 tpSize,
+
bool isDora,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localInDim( + +
+
SizeType32 tpSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localOutDim( + +
+
SizeType32 tpSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localInAdapterSize( + +
+
SizeType32 adapterSize,
+
SizeType32 tpSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localOutAdapterSize( + +
+
SizeType32 adapterSize,
+
SizeType32 tpSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localInOutSize( + +
+
SizeType32 adapterSize,
+
SizeType32 tpSize,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr localTotalSize( + +
+
SizeType32 adapterSize,
+
SizeType32 tpSize,
+
bool isDora,
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 constexpr value() const noexcept#
+
+ +
+
+inline std::string_view constexpr name() const noexcept#
+
+ +
+
+inline SizeType32 constexpr inDim() const noexcept#
+
+ +
+
+inline SizeType32 constexpr outDim() const noexcept#
+
+ +
+
+inline bool constexpr inDimFirst() const noexcept#
+
+ +
+
+inline bool constexpr outDimFirst() const noexcept#
+
+ +
+
+inline SizeType32 constexpr inTpSplitDim() const noexcept#
+
+ +
+
+inline SizeType32 constexpr outTpSplitDim() const noexcept#
+
+ +
+
+

Public Static Functions

+
+
+static std::vector<LoraModule> createLoraModules( + +
+
std::vector<std::string> const &loraModuleNames,
+
SizeType32 hiddenSize,
+
SizeType32 mlpHiddenSize,
+
SizeType32 numAttentionHeads,
+
SizeType32 numKvAttentionHeads,
+
SizeType32 attentionHeadSize,
+
SizeType32 tpSize,
+
SizeType32 numExperts,
+
+ +)#
+
+ +
+
+static inline ModuleType constexpr toModuleType( + +
+
std::string_view const &name,
+
+ +)#
+
+ +
+
+static inline std::string_view constexpr toModuleName( + +
+
ModuleType t,
+
+ +) noexcept#
+
+ +
+
+static inline std::string_view constexpr toModuleName(SizeType32 id)#
+
+ +
+
+

Private Members

+
+
+ModuleType mType#
+
+ +
+
+SizeType32 mInDim#
+
+ +
+
+SizeType32 mOutDim#
+
+ +
+
+bool mInDimFirst#
+
+ +
+
+bool mOutDimFirst#
+
+ +
+
+SizeType32 mInTpSplitDim#
+
+ +
+
+SizeType32 mOutTpSplitDim#
+
+ +
+
+ +
+ +
+ +
+
+

speculativeDecodingMode.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class SpeculativeDecodingMode#
+
+

Public Types

+
+
+using UnderlyingType = std::uint8_t#
+
+ +
+
+

Public Functions

+
+
+inline bool constexpr isNone() const#
+
+ +
+
+inline bool constexpr isDraftTokensExternal() const#
+
+ +
+
+inline bool constexpr isMedusa() const#
+
+ +
+
+inline bool constexpr isLookaheadDecoding() const#
+
+ +
+
+inline bool constexpr isExplicitDraftTokens() const#
+
+ +
+
+inline bool constexpr isEagle() const#
+
+ +
+
+inline bool constexpr updatesPositionIds() const#
+
+ +
+
+inline bool constexpr requiresAttentionMask() const#
+
+ +
+
+inline bool constexpr predictsDraftTokens() const#
+
+ +
+
+inline bool constexpr needsKVCacheRewind() const#
+
+ +
+
+inline bool constexpr variableDraftLength() const#
+
+ +
+
+inline bool constexpr hasDraftLogits() const#
+
+ +
+
+inline bool constexpr needsDecoderPrologue() const#
+
+ +
+
+inline bool operator==(SpeculativeDecodingMode const &other) const#
+
+ +
+
+inline explicit constexpr SpeculativeDecodingMode( + +
+
UnderlyingType state,
+
+ +)#
+
+ +
+
+

Public Static Functions

+
+
+static inline auto constexpr None()#
+
+ +
+
+static inline auto constexpr DraftTokensExternal()#
+
+ +
+
+static inline auto constexpr Medusa()#
+
+ +
+
+static inline auto constexpr LookaheadDecoding()#
+
+ +
+
+static inline auto constexpr ExplicitDraftTokens()#
+
+ +
+
+static inline auto constexpr Eagle()#
+
+ +
+
+

Private Functions

+
+
+inline bool constexpr anyBitSet(UnderlyingType bits) const#
+
+ +
+
+inline bool constexpr allBitSet(UnderlyingType bits) const#
+
+ +
+
+

Private Members

+
+
+UnderlyingType mState = {kNone}#
+
+ +
+
+

Private Static Attributes

+
+
+static UnderlyingType constexpr kNone = {1U << 0U}#
+
+ +
+
+static UnderlyingType constexpr kDraftTokensExternal = {1U << 1U}#
+
+ +
+
+static UnderlyingType constexpr kMedusa = {1U << 2U}#
+
+ +
+
+static UnderlyingType constexpr kLookaheadDecoding = {1U << 3U}#
+
+ +
+
+static UnderlyingType constexpr kExplicitDraftTokens = {1U << 4U}#
+
+ +
+
+static UnderlyingType constexpr kEagle = {1U << 5U}#
+
+ +
+
+ +
+ +
+ +
+
+

cudaEvent.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class CudaEvent#
+
+

Public Types

+
+
+using pointer = cudaEvent_t#
+
+ +
+
+

Public Functions

+
+
+inline explicit CudaEvent(unsigned int flags = cudaEventDisableTiming)#
+

Creates a new cuda event. The event will be destroyed in the destructor.

+
+
Parameters:
+

flags – Flags for event creation. By default, event timing is disabled.

+
+
+
+ +
+
+inline explicit CudaEvent(pointer event, bool ownsEvent = true)#
+

Pass an existing cuda event to this object.

+
+
Parameters:
+
    +

  • event – The event to pass to this object.

    • +

  • ownsEvent – Whether this object owns the event and destroys it in the destructor.

    • +
+
+
+
+ +
+
+inline pointer get() const#
+

Returns the event associated with this object.

+
+ +
+
+inline void synchronize() const#
+

Synchronizes the event.

+
+ +
+
+

Private Types

+
+
+using element_type = std::remove_pointer_t<pointer>#
+
+ +
+
+using EventPtr = std::unique_ptr<element_type, Deleter>#
+
+ +
+
+

Private Members

+
+
+EventPtr mEvent#
+
+ +
+
+
+class Deleter#
+
+

Public Functions

+
+
+inline explicit Deleter(bool ownsEvent)#
+
+ +
+
+inline explicit Deleter()#
+
+ +
+
+inline constexpr void operator()(pointer event) const#
+
+ +
+
+

Private Members

+
+
+bool mOwnsEvent#
+
+ +
+
+ +
+ +
+ +
+ +
+
+

decodingInput.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class DecodingInput#
+
+#include <decodingInput.h>
+

Represents the inputs to the decoder.

+

This input type is assumed immutable. It represents whatever the decoder received initially, and can always be referred to as such.

+
+

Public Types

+
+
+using TensorConstPtr = ITensor::SharedConstPtr#
+
+ +
+
+using TensorPtr = ITensor::SharedPtr#
+
+ +
+
+

Public Functions

+
+
+DecodingInput() = default#
+
+ +
+
+

Public Members

+
+
+SizeType32 step = {}#
+

Mandatory parameters The index of the decoding step we are on. Only used in Python runtime

+
+ +
+
+SizeType32 maxLength = {}#
+

The maximum number of tokens to decode.

+
+ +
+
+SizeType32 maxAttentionWindow = {}#
+

The maximum length of the attention window to consider while decoding.

+
+ +
+
+SizeType32 sinkTokenLength = {}#
+

The number of tokens to use as attention sinks, https://arxiv.org/html/2309.17453v3.

+
+ +
+
+SizeType32 batchSize = {}#
+

The number of samples in the batch.

+
+ +
+
+std::vector<SizeType32> beamWidths#
+

The beam widths of each request, [batchSize].

+
+ +
+
+SizeType32 maxStopWordsLen = {}#
+

The maximum value in the stopWordsLens tensor.

+
+ +
+
+SizeType32 maxBadWordsLen = {}#
+

The maximum value in the badWordsLens tensor.

+
+ +
+
+std::vector<TensorConstPtr> logitsVec#
+

The output of the model forward computation, a probability distribution over the vocabulary [batchSize][numGenTokens, beamWidth, vocabSizePadded] on gpu

+
+ +
+
+TensorConstPtr endIds#
+

The end ids, [batchSize * beamWidth] on gpu.

+
+ +
+
+TensorConstPtr batchSlots#
+

Address map of the linear batch id to to the seq slots, [batchSize] on pinned, int32_t.

+
+ +
+
+TensorConstPtr finishReasons#
+

Optional parameters Finished states at current iteration (skip decoding step of a request if true), [batchSize, beamWidth] on gpu

+
+ +
+
+TensorConstPtr sequenceLimitLength#
+

The maximum sequence length for each sequence in the batch, [batchSize] on gpu.

+
+ +
+
+TensorConstPtr embeddingBias#
+
+ +
+
+TensorConstPtr lengths#
+
+ +
+
+std::vector<TensorPtr> badWordsLists#
+
+ +
+
+TensorConstPtr badWordsPtrs#
+
+ +
+
+TensorConstPtr badWordsLens#
+
+ +
+
+std::vector<TensorPtr> stopWordsLists#
+
+ +
+
+TensorConstPtr stopWordsPtrs#
+
+ +
+
+TensorConstPtr stopWordsLens#
+
+ +
+
+TensorConstPtr noRepeatNgramSize#
+
+ +
+
+TensorPtr cacheIndirection#
+

Parameters for beam search KV cache index for beam search, [batchSize, beamWidth, maxSeqLen] on gpu

+
+ +
+
+std::optional<std::vector<SizeType32>> generationSteps#
+

Steps of each request, for Variable-Beam-Width-Search, [batchSize].

+
+ +
+
+std::optional<MedusaInputs> medusaInputs#
+
+ +
+
+std::optional<ExplicitDraftTokensInputs> explicitDraftTokensInputs#
+
+ +
+
+std::optional<LookaheadInputs> lookaheadInputs#
+
+ +
+
+std::optional<ExternalDraftTokensInputs> externalDraftTokensInputs#
+
+ +
+
+std::optional<EagleInputs> eagleInputs#
+
+ +
+
+
+struct EagleInputs#
+
+

Public Members

+
+
+TensorConstPtr nextDraftTokens#
+
+ +
+
+TensorConstPtr nextDraftLens#
+
+ +
+
+TensorConstPtr nextDraftPaths#
+
+ +
+
+TensorConstPtr lastDraftTokens#
+
+ +
+
+TensorConstPtr lastDraftLens#
+
+ +
+
+TensorConstPtr lastDraftPaths#
+
+ +
+
+TensorConstPtr acceptedTokens#
+
+ +
+
+TensorConstPtr acceptedLens#
+
+ +
+
+TensorConstPtr acceptedPathIds#
+
+ +
+
+TensorConstPtr chunkedContextNextTokens#
+
+ +
+
+TensorConstPtr seqSlots#
+
+ +
+
+ +
+
+class ExplicitDraftTokensInputs#
+
+

Public Members

+
+
+TensorConstPtr nextDraftTokens#
+
+ +
+
+TensorConstPtr nextFlatTokens#
+
+ +
+
+TensorConstPtr nextDraftIndices#
+
+ +
+
+TensorConstPtr nextDraftProbs#
+
+ +
+
+TensorConstPtr lastDraftTokens#
+
+ +
+
+TensorConstPtr lastDraftIndices#
+
+ +
+
+TensorConstPtr masks#
+
+ +
+
+TensorConstPtr packedPositionIds#
+
+ +
+
+TensorConstPtr bestPathLengths#
+
+ +
+
+TensorConstPtr bestPathIndices#
+
+ +
+
+TensorConstPtr nextGenerationLengths#
+
+ +
+
+TensorConstPtr lastPositionIdsBase#
+
+ +
+
+TensorConstPtr lastGenerationLengths#
+
+ +
+
+TensorConstPtr maxGenLengthDevice#
+
+ +
+
+TensorConstPtr seqSlots#
+
+ +
+
+ +
+
+class ExternalDraftTokensInputs#
+
+

Public Members

+
+
+TensorPtr draftLogits#
+
+ +
+
+TensorPtr draftLogitsHost#
+
+ +
+
+TensorPtr draftProbs#
+
+ +
+
+TensorPtr targetProbs#
+
+ +
+
+TensorPtr numDraftTokens#
+
+ +
+
+TensorPtr numDraftTokensHost#
+
+ +
+
+TensorPtr draftTokenIds#
+
+ +
+
+TensorPtr draftTokenIdsHost#
+
+ +
+
+TensorPtr useDraftLogits#
+
+ +
+
+TensorPtr useDraftLogitsHost#
+
+ +
+
+SizeType32 step#
+
+ +
+
+float constantThreshold#
+
+ +
+
+bool useRandomAcceptanceThreshold#
+
+ +
+
+ +
+
+struct LookaheadInputs#
+
+

Public Members

+
+
+TensorPtr tokensPerStep#
+
+ +
+
+ +
+
+class MedusaInputs#
+
+

Public Members

+
+
+TensorConstPtr medusaPaths#
+

[batchSize, maxTokensPerStep, maxMedusaHeads + 1], on gpu

+
+ +
+
+TensorConstPtr medusaTreeIds#
+

[batchSize, maxTokensPerStep], on gpu

+
+ +
+
+std::vector<std::vector<TensorPtr>> medusaLogits#
+

[batchSize][maxAcceptedDraftTokensPerStep][maxDraftTokens + 1, vocabSizePadded], on gpu

+
+ +
+
+TensorPtr medusaCurTokensPerStep#
+

[batchSize], on gpu

+
+ +
+
+TensorConstPtr medusaTargetTokensPerStep#
+

[batchSize], on gpu

+
+ +
+
+ +
+ +
+ +
+ +
+
+

speculativeDecodingModule.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class SpeculativeDecodingModule#
+

Subclassed by tensorrt_llm::runtime::EagleModule, tensorrt_llm::runtime::LookaheadModule, tensorrt_llm::runtime::MedusaModule

+
+

Public Functions

+
+
+inline explicit SpeculativeDecodingModule( + +
+
SizeType32 maxDraftPathLen,
+
SizeType32 maxDecodingDraftTokens,
+
SizeType32 maxNumPaths,
+
+ +) noexcept#
+
+ +
+
+inline explicit SpeculativeDecodingModule() noexcept#
+
+ +
+
+virtual ~SpeculativeDecodingModule() = default#
+
+ +
+
+SpeculativeDecodingModule(SpeculativeDecodingModule const &o) = default#
+
+ +
+
+SpeculativeDecodingModule &operator=( + +
+
SpeculativeDecodingModule const &o,
+
+ +) = default#
+
+ +
+
+inline SizeType32 getMaxDraftPathLen() const noexcept#
+
+
Returns:
+

max number of draft tokens that can be accepted by one step of the decoder

+
+
+
+ +
+
+inline SizeType32 getMaxPathLen() const noexcept#
+

+one more than draft path len for prediction from primary head

+
+
Returns:
+

max number of tokens that a request can grow in one step of the decoder

+
+
+
+ +
+
+inline SizeType32 getMaxDecodingDraftTokens() const noexcept#
+
+
Returns:
+

max number of draft tokens processed by one step of the decoder

+
+
+
+ +
+
+inline SizeType32 getMaxDecodingTokens() const noexcept#
+

+one more than decoding draft tokens for prediction from primary head

+
+
Returns:
+

max number of tokens processed by one step of the decoder

+
+
+
+ +
+
+inline SizeType32 getNumPackedMasks() const noexcept#
+
+ +
+
+inline SizeType32 getMaxNumPaths() const noexcept#
+
+ +
+
+inline void setMaxDraftTokens(SizeType32 maxDraftTokens) noexcept#
+
+ +
+
+inline void setMaxDraftPathLen(SizeType32 maxDraftPathLen) noexcept#
+
+ +
+
+inline void setMaxNumPaths(SizeType32 maxNumPaths) noexcept#
+
+ +
+
+

Private Functions

+
+
+inline void computeNumPackedMasks() noexcept#
+
+ +
+
+

Private Members

+
+
+SizeType32 mMaxDraftPathLen#
+
+ +
+
+SizeType32 mMaxDecodingDraftTokens#
+
+ +
+
+SizeType32 mMaxNumPaths#
+
+ +
+
+SizeType32 mMaxNumPackedMasks#
+
+ +
+
+ +
+ +
+ +
+
+

iGptDecoderBatched.h#

+
+
+namespace tensorrt_llm
+
+
+namespace batch_manager
+
+ +
+
+namespace runtime
+
+
+class IGptDecoderBatched#
+
+#include <iGptDecoderBatched.h>
+

GPT decoder class with support for in-flight batching.

+

Subclassed by tensorrt_llm::runtime::GptDecoderBatched

+
+

Public Types

+
+
+using CudaStreamPtr = std::shared_ptr<CudaStream>#
+
+ +
+
+using LlmRequestPtr = std::shared_ptr<tensorrt_llm::batch_manager::LlmRequest>#
+
+ +
+
+using RequestVector = std::vector<LlmRequestPtr>#
+
+ +
+
+using TensorPtr = std::shared_ptr<ITensor>#
+
+ +
+
+

Public Functions

+
+
+virtual void setup( + +
+
executor::DecodingMode const &mode,
+
SizeType32 maxNumSequences,
+
SizeType32 maxBeamWidth,
+
nvinfer1::DataType dtype,
+
ModelConfig const &modelConfig,
+
WorldConfig const &worldConfig,
+
+ +) = 0#
+

Setup the decoder before calling forward()

+
+ +
+
+virtual void disableLookahead( + +
+
RequestVector const &genRequests,
+
TensorPtr const &batchSlots,
+
+ +) = 0#
+

Disable Lookahead decoding.

+
+ +
+
+virtual CudaEvent forwardAsync( + +
+
decoder::DecoderState const &decoderState,
+
batch_manager::DecoderInputBuffers const &input,
+
+ +) = 0#
+

Run one step for all requests without blocking the host process and return the token for synchronization.

+
+ +
+
+virtual void forward( + +
+
decoder::DecoderState const &decoderState,
+
batch_manager::DecoderInputBuffers const &input,
+
+ +) = 0#
+

Run one step for all requests and wait for completion on the host.

+
+ +
+
+virtual CudaEvent finalize( + +
+
decoder::DecoderState const &decoderState,
+
SizeType32 batchSlot,
+
SamplingConfig const &samplingConfig,
+
bool streaming,
+
+ +) const = 0#
+

Gather final beam search results for request batchIdx. Result will only be available after event returned.

+
+ +
+
+

Protected Functions

+
+
+IGptDecoderBatched() = default#
+
+ +
+
+virtual ~IGptDecoderBatched() = default#
+
+ +
+
+ +
+
+namespace decoder#
+
+ +
+ +
+ +
+
+

eagleModule.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class EagleModule : public tensorrt_llm::runtime::SpeculativeDecodingModule#
+
+

Public Functions

+
+
+inline explicit EagleModule( + +
+
SizeType32 maxDraftPathLen,
+
SizeType32 maxDecodingDraftTokens,
+
SizeType32 numTransformersLayer,
+
SizeType32 maxNonLeafNodesPerLayer,
+
+ +) noexcept#
+
+ +
+
+inline explicit EagleModule() noexcept#
+
+ +
+
+inline executor::EagleChoices const &getDefaultEagleChoices( + +
+
+ +) const noexcept#
+
+ +
+
+inline SizeType32 getNumTransformerLayers() const noexcept#
+
+ +
+
+inline SizeType32 getMaxNonLeafNodesPerLayer() const noexcept#
+
+ +
+
+

Private Members

+
+
+SizeType32 mNumTransformersLayer#
+
+ +
+
+SizeType32 mMaxNonLeafNodesPerLayer#
+
+ +
+
+executor::EagleChoices mDefaultEagleChoices = {{0}, {0, 0}, {1}, {0, 1}, {2}, {0, 0, 0}, {1, 0}, {0, 2}, {3}, {0, 3}, {4}, {0, 4}, {2, 0}, {0, 5}, {0, 0, 1}, {5}, {0, 6}, {6}, {0, 7}, {0, 1, 0}, {1, 1}, {7}, {0, 8}, {0, 0, 2}, {3, 0}, {0, 9}, {8}, {9}, {1, 0, 0}, {0, 2, 0}, {1, 2}, {0, 0, 3}, {4, 0}, {2, 1}, {0, 0, 4}, {0, 0, 5}, {0, 0, 0, 0}, {0, 1, 1}, {0, 0, 6}, {0, 3, 0}, {5, 0}, {1, 3}, {0, 0, 7}, {0, 0, 8}, {0, 0, 9}, {6, 0}, {0, 4, 0}, {1, 4}, {7, 0}, {0, 1, 2}, {2, 0, 0}, {3, 1}, {2, 2}, {8, 0}, {0, 5, 0}, {1, 5}, {1, 0, 1}, {0, 2, 1}, {9, 0}, {0, 6, 0}, {0, 0, 0, 1}, {1, 6}, {0, 7, 0}}#
+
+ +
+
+ +
+ +
+ +
+
+

tllmLogger.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class TllmLogger : public nvinfer1::ILogger#
+
+

Public Functions

+
+
+void log( + +
+
Severity severity,
+
nvinfer1::AsciiChar const *msg,
+
+ +) noexcept override#
+
+ +
+
+Severity getLevel()#
+
+ +
+
+void setLevel(Severity level)#
+
+ +
+
+ +
+ +
+ +
+
+

gptDecoderBatched.h#

+
+
+namespace tensorrt_llm
+
+
+namespace batch_manager
+
+ +
+
+namespace runtime
+
+
+class GptDecoderBatched : public tensorrt_llm::runtime::IGptDecoderBatched#
+
+#include <gptDecoderBatched.h>
+

GPT decoder class with support for in-flight batching.

+
+

Public Types

+
+
+using CudaStreamPtr = std::shared_ptr<CudaStream>#
+
+ +
+
+using LlmRequestPtr = std::shared_ptr<tensorrt_llm::batch_manager::LlmRequest>#
+
+ +
+
+using RequestVector = std::vector<LlmRequestPtr>#
+
+ +
+
+using TensorPtr = ITensor::SharedPtr#
+
+ +
+
+

Public Functions

+
+
+explicit GptDecoderBatched(CudaStreamPtr stream)#
+
+ +
+
+virtual void setup( + +
+
executor::DecodingMode const &mode,
+
SizeType32 maxNumSequences,
+
SizeType32 maxBeamWidth,
+
nvinfer1::DataType dtype,
+
ModelConfig const &modelConfig,
+
WorldConfig const &worldConfig,
+
+ +) override#
+

Setup the decoder before calling forward()

+
+ +
+
+virtual void disableLookahead( + +
+
RequestVector const &genRequests,
+
TensorPtr const &batchSlots,
+
+ +) override#
+

Disable Lookahead decoding.

+
+ +
+
+virtual CudaEvent forwardAsync( + +
+
decoder::DecoderState const &decoderState,
+
batch_manager::DecoderInputBuffers const &input,
+
+ +) override#
+

Run one step for all requests without blocking the host process and return the token for synchronization.

+
+ +
+
+virtual void forward( + +
+
decoder::DecoderState const &decoderState,
+
batch_manager::DecoderInputBuffers const &input,
+
+ +) override#
+

Run one step for all requests and wait for completion on the host.

+
+ +
+
+virtual CudaEvent finalize( + +
+
decoder::DecoderState const &decoderState,
+
SizeType32 batchSlot,
+
SamplingConfig const &samplingConfig,
+
bool streaming,
+
+ +) const override#
+

Gather final beam search results for request batchSlot. Result will only be available after event returned.

+
+ +
+
+inline CudaStreamPtr getDecoderStream() const#
+
+ +
+
+inline IGptDecoder &getUnderlyingDecoder() const#
+
+ +
+
+inline BufferManager const &getBufferManager() const#
+
+ +
+
+

Private Types

+
+
+using GptDecoderPtr = std::unique_ptr<IGptDecoder>#
+
+ +
+
+

Private Functions

+
+
+void forwardDispatch( + +
+
decoder::DecoderState const &decoderState,
+
batch_manager::DecoderInputBuffers const &input,
+
+ +)#
+

Calls decoders for tokens per engine step.

+
+ +
+
+

Private Members

+
+
+CudaStreamPtr mRuntimeStream#
+
+ +
+
+CudaStreamPtr mDecoderStream#
+
+ +
+
+BufferManager mBufferManager#
+
+ +
+
+GptDecoderPtr mDecoder#
+
+ +
+
+ +
+ +
+ +
+
+

cudaStream.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class CudaStream#
+
+

Public Functions

+
+
+inline explicit CudaStream( + +
+
unsigned int flags = cudaStreamNonBlocking,
+
int priority = 0,
+
+ +)#
+

Creates a new cuda stream on the current device. The stream will be destroyed in the destructor.

+
+
Parameters:
+
    +

  • flags – Flags for stream creation. See cudaStreamCreateWithFlags for a list of valid flags that can be passed.

    • +

  • priority – Priority of the stream. Lower numbers represent higher priorities. See cudaDeviceGetStreamPriorityRange for more information about the meaningful stream priorities that can be passed.

    • +
+
+
+
+ +
+
+inline explicit CudaStream( + +
+
cudaStream_t stream,
+
int device,
+
bool ownsStream = true,
+
+ +)#
+

Pass an existing cuda stream to this object.

+
+
Parameters:
+
    +

  • stream – The stream to pass to this object.

    • +

  • device – The device on which the stream was created.

    • +

  • ownsStream – Whether this object owns the stream and destroys it in the destructor.

    • +
+
+
+
+ +
+
+inline explicit CudaStream(cudaStream_t stream)#
+

Construct with an existing cuda stream or the default stream by passing nullptr.

+
+ +
+
+inline int getDevice() const#
+

Returns the device on which the stream was created.

+
+ +
+
+inline cudaStream_t get() const#
+

Returns the stream associated with this object.

+
+ +
+
+inline void synchronize() const#
+

Synchronizes the stream.

+
+ +
+
+inline void record(CudaEvent::pointer event) const#
+

Record an event on the stream.

+
+ +
+
+inline void record(CudaEvent const &event) const#
+

Record an event on the stream.

+
+ +
+
+inline void wait(CudaEvent::pointer event) const#
+

Wait for an event.

+
+ +
+
+inline void wait(CudaEvent const &event) const#
+

Wait for an event.

+
+ +
+
+

Private Types

+
+
+using StreamPtr = std::unique_ptr<std::remove_pointer_t<cudaStream_t>, Deleter>#
+
+ +
+
+

Private Members

+
+
+StreamPtr mStream#
+
+ +
+
+int mDevice = {-1}#
+
+ +
+
+
+class Deleter#
+
+

Public Functions

+
+
+inline explicit Deleter(bool ownsStream)#
+
+ +
+
+inline explicit Deleter()#
+
+ +
+
+inline constexpr void operator()(cudaStream_t stream) const#
+
+ +
+
+

Private Members

+
+
+bool mOwnsStream#
+
+ +
+
+ +
+ +
+ +
+ +
+
+

ipcNvlsMemory.h#

+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+

Functions

+
+
+void MPI_group_barrier(std::set<int> ranks)#
+
+ +
+
+bool ipcNvlsSupported()#
+
+ +
+
+IpcNvlsHandle *ipcNvlsAllocate(size_t size, std::set<int> ranks)#
+
+ +
+
+void ipcNvlsFree(IpcNvlsHandle *handle)#
+
+ +
+
+
+template<typename T>
class DeviceAllocationNvls#
+
+

Public Functions

+
+
+DeviceAllocationNvls() = default#
+
+ +
+
+inline ~DeviceAllocationNvls()#
+
+ +
+
+inline void reset(size_t size, std::set<int> ranks)#
+
+ +
+
+inline T *getMulticastPointer() const#
+
+ +
+
+inline T *getUnicastPointer() const#
+
+ +
+
+inline T **getIpcUnicastPointers()#
+
+ +
+
+inline size_t getCapacity() const#
+
+ +
+
+inline void free()#
+
+ +
+
+

Private Members

+
+
+size_t _capacity = 0#
+
+ +
+
+IpcNvlsHandle *_handle#
+
+ +
+
+ +
+
+struct IpcNvlsHandle#
+
+

Public Members

+
+
+size_t size = 0#
+
+ +
+
+uintptr_t uc_ptr = 0#
+
+ +
+
+uintptr_t mc_ptr = 0#
+
+ +
+
+std::vector<uintptr_t> ipc_uc_ptrs#
+
+ +
+
+CUdeviceptr uc_va#
+
+ +
+
+CUdeviceptr mc_va#
+
+ +
+
+std::vector<CUdeviceptr> ipc_uc_vas#
+
+ +
+
+CUmemGenericAllocationHandle uc_handle#
+
+ +
+
+CUmemGenericAllocationHandle mc_handle#
+
+ +
+
+std::vector<CUmemGenericAllocationHandle> ipc_uc_handles#
+
+ +
+
+ +
+ +
+ +
+
+

samplingConfig.h#

+
+

Defines

+
+
+SET_FROM_OPTIONAL(varName, VarName, VarType)#
+
+ +
+
+
+namespace tensorrt_llm
+
+
+namespace runtime
+
+
+class SamplingConfig#
+
+

Public Functions

+
+
+inline explicit SamplingConfig(SizeType32 beamWidth = 1)#
+
+ +
+
+inline explicit SamplingConfig( + +
+
std::vector<SamplingConfig> const &configs,
+
+ +)#
+
+ +
+
+inline explicit SamplingConfig( + +
+
executor::SamplingConfig const &samplingConfig,
+
std::optional<executor::ExternalDraftTokensConfig> const &externalDraftTokensConfig = std::nullopt,
+
+ +)#
+
+ +
+
+inline bool validate()#
+
+ +
+
+template<typename T>
inline bool useDefaultValues( + +
+
OptVec<T> const &vec,
+
T defaultValue,
+
+ +)#
+
+ +
+
+inline bool operator==(SamplingConfig const &other) const#
+
+ +
+
+inline SizeType32 getNumReturnBeams() const#
+
+ +
+
+inline SizeType32 getMaxBeamWidth() const noexcept#
+
+ +
+
+

Public Members

+
+
+SizeType32 beamWidth#
+
+ +
+
+std::optional<SizeType32> numReturnSequences#
+
+ +
+
+OptVec<FloatType> temperature#
+
+ +
+
+OptVec<FloatType> originalTemperature#
+
+ +
+
+OptVec<SizeType32> minLength#
+
+ +
+
+OptVec<FloatType> repetitionPenalty#
+
+ +
+
+OptVec<FloatType> presencePenalty#
+
+ +
+
+OptVec<FloatType> frequencyPenalty#
+
+ +
+
+OptVec<SizeType32> promptIgnoreLength#
+
+ +
+
+OptVec<SizeType32> noRepeatNgramSize#
+
+ +
+
+OptVec<bool> outputLogProbs#
+
+ +
+
+OptVec<bool> cumLogProbs#
+
+ +
+
+OptVec<SizeType32> topK#
+
+ +
+
+OptVec<FloatType> topP#
+
+ +
+
+OptVec<uint64_t> randomSeed#
+
+ +
+
+OptVec<FloatType> topPDecay#
+
+ +
+
+OptVec<FloatType> topPMin#
+
+ +
+
+OptVec<TokenIdType> topPResetIds#
+
+ +
+
+OptVec<FloatType> minP#
+
+ +
+
+OptVec<FloatType> beamSearchDiversityRate#
+
+ +
+
+OptVec<FloatType> lengthPenalty#
+
+ +
+
+OptVec<SizeType32> earlyStopping#
+
+ +
+
+OptVec<std::vector<SizeType32>> beamWidthArray#
+
+ +
+
+OptVec<FloatType> draftAcceptanceThreshold#
+
+ +
+
+OptVec<std::vector<SizeType32>> topKMedusaHeads#
+
+ +
+
+std::optional<bool> normalizeLogProbs#
+
+ +
+
+

Private Types

+
+
+using FloatType = float#
+
+ +
+
+template<typename T>
using OptVec = std::optional<std::vector<T>>#
+
+ +
+
+

Private Functions

+
+
+template<typename T>
inline bool validateVec( + +
+
std::string name,
+
OptVec<T> const &vec,
+
T min,
+
std::optional<T> max = std::nullopt,
+
+ +)#
+
+ +
+
+

Private Static Functions

+
+
+template<typename T>
static inline OptVec<T> fuseValues( + +
+
std::vector<SamplingConfig> const &configs,
+
std::function<OptVec<T>(size_t ci)> accessor,
+
T defaultValue,
+
+ +)#
+
+ +
+
+ +
+ +
+

decoderState.h#

@@ -6526,4404 +12305,6 @@ one more than decoding draft tokens for prediction from primary head

-
-
-

lookaheadBuffers.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class LookaheadDecodingBuffers#
-
-

Public Types

-
-
-using TensorPtr = ITensor::SharedPtr#
-
- -
-
-

Public Functions

-
-
-LookaheadDecodingBuffers( - -
-
SizeType32 maxNumSequences,
-
SizeType32 maxTokensPerStep,
-
BufferManager const &bufferManager,
-
- -)#
-
- -
-
-

Public Members

-
-
-TensorPtr generationLengths#
-
- -
-
-TensorPtr positionOffsets#
-
- -
-
-TensorPtr packedMasks#
-
- -
-
-TensorPtr positionIds#
-
- -
-
- -
-
-class LookaheadRuntimeBuffers#
-
-

Public Types

-
-
-using TensorPtr = ITensor::SharedPtr#
-
- -
-
-using TensorMap = StringPtrMap<ITensor>#
-
- -
-
-

Public Functions

-
-
-LookaheadRuntimeBuffers( - -
-
SizeType32 maxBatchSize,
-
SizeType32 maxBeamWidth,
-
BufferManager const &manager,
-
ModelConfig const &modelConfig,
-
WorldConfig const &worldConfig,
-
executor::DecodingConfig const &decodingConfig,
-
TllmRuntime const &runtime,
-
- -)#
-
- -
-
-void setFromInputs( - -
-
SizeType32 numCtxSequences,
-
SizeType32 numGenSequences,
-
ITensor const &requestTypes,
-
ITensor const &seqSlots,
-
LookaheadDecodingBuffers const &decoderLookaheadBuffers,
-
TllmRuntime const &runtime,
-
ModelConfig const &modelConfig,
-
WorldConfig const &worldConfig,
-
- -) const#
-
- -
-
-void reshape( - -
-
SizeType32 numCtxSequences,
-
SizeType32 numGenSequences,
-
SizeType32 tokensPerStep,
-
- -)#
-
- -
-
-void insertInputTensors( - -
-
TensorMap &inputBuffers,
-
TensorMap &outputBuffers,
-
WorldConfig const &worldConfig,
-
- -) const#
-
- -
-
-void enableLookaheadDecoding( - -
-
SizeType32 maxBatchSize,
-
SizeType32 tokensPerStep,
-
- -)#
-
- -
-
-void disableLookaheadDecoding()#
-
- -
-
-

Public Members

-
-
-TensorPtr cumSumLength#
-
- -
-
-TensorPtr packedMasksDevice#
-
- -
-
-TensorPtr generationLengthsDevice#
-
- -
-
-TensorPtr positionOffsetsDevice#
-
- -
-
-TensorPtr positionIdsDevice#
-
- -
-
-TensorPtr packedMaskHost#
-
- -
-
-TensorPtr generationLengthsHost#
-
- -
-
-TensorPtr positionOffsetsHost#
-
- -
-
-TensorPtr positionIdsHost#
-
- -
-
-TensorPtr packedMaskHostCopy#
-
- -
-
-TensorPtr generationLengthsHostCopy#
-
- -
-
-TensorPtr positionOffsetsHostCopy#
-
- -
-
-TensorPtr positionIdsHostCopy#
-
- -
-
-TensorPtr useSpecDecoding#
-
- -
-
-TensorPtr batchSlotsHostCopy#
-
- -
-
- -
- -
- -
-
-

eagleModule.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class EagleModule : public tensorrt_llm::runtime::SpeculativeDecodingModule#
-
-

Public Functions

-
-
-inline explicit EagleModule( - -
-
SizeType32 maxDraftPathLen,
-
SizeType32 maxDecodingDraftTokens,
-
SizeType32 numTransformersLayer,
-
SizeType32 maxNonLeafNodesPerLayer,
-
- -) noexcept#
-
- -
-
-inline explicit EagleModule() noexcept#
-
- -
-
-inline executor::EagleChoices const &getDefaultEagleChoices( - -
-
- -) const noexcept#
-
- -
-
-inline SizeType32 getNumTransformerLayers() const noexcept#
-
- -
-
-inline SizeType32 getMaxNonLeafNodesPerLayer() const noexcept#
-
- -
-
-

Private Members

-
-
-SizeType32 mNumTransformersLayer#
-
- -
-
-SizeType32 mMaxNonLeafNodesPerLayer#
-
- -
-
-executor::EagleChoices mDefaultEagleChoices = {{0}, {0, 0}, {1}, {0, 1}, {2}, {0, 0, 0}, {1, 0}, {0, 2}, {3}, {0, 3}, {4}, {0, 4}, {2, 0}, {0, 5}, {0, 0, 1}, {5}, {0, 6}, {6}, {0, 7}, {0, 1, 0}, {1, 1}, {7}, {0, 8}, {0, 0, 2}, {3, 0}, {0, 9}, {8}, {9}, {1, 0, 0}, {0, 2, 0}, {1, 2}, {0, 0, 3}, {4, 0}, {2, 1}, {0, 0, 4}, {0, 0, 5}, {0, 0, 0, 0}, {0, 1, 1}, {0, 0, 6}, {0, 3, 0}, {5, 0}, {1, 3}, {0, 0, 7}, {0, 0, 8}, {0, 0, 9}, {6, 0}, {0, 4, 0}, {1, 4}, {7, 0}, {0, 1, 2}, {2, 0, 0}, {3, 1}, {2, 2}, {8, 0}, {0, 5, 0}, {1, 5}, {1, 0, 1}, {0, 2, 1}, {9, 0}, {0, 6, 0}, {0, 0, 0, 1}, {1, 6}, {0, 7, 0}}#
-
- -
-
- -
- -
- -
-
-

runtimeDefaults.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-struct RuntimeDefaults#
-
-

Public Functions

-
-
-inline RuntimeDefaults( - -
-
std::optional<std::vector<SizeType32>> maxAttentionWindowVec,
-
std::optional<SizeType32> sinkTokenLength,
-
- -)#
-
- -
-
-RuntimeDefaults() = default#
-
- -
-
-

Public Members

-
-
-std::optional<std::vector<SizeType32>> maxAttentionWindowVec#
-
- -
-
-std::optional<SizeType32> sinkTokenLength#
-
- -
-
- -
- -
- -
-
-

decodingOutput.h#

-
-
-namespace tensorrt_llm
-
-
-namespace batch_manager
-
- -
-
-namespace runtime
-
-
-class DecodingOutput#
-
-

Public Types

-
-
-using TensorPtr = ITensor::SharedPtr#
-
- -
-
-

Public Functions

-
-
-DecodingOutput() = default#
-
- -
-
-

Public Members

-
-
-TensorPtr ids#
-

Mandatory parameters Previously generated token ids for all steps before DecodingInput.step, [BS, BM, MSL]

-
- -
-
-TensorPtr gatheredIds#
-

The tokens computed during the gatherTree step, [BS, BM, MSL] Necessary for “Streaming + Beam Search” mode since beam search kernels store ungathered tokens in ids.

-
- -
-
-TensorPtr newTokensSteps#
-

New tokens at each generated token of maxTokensPerStep, [maxTokensPerStep, BS, BM].

-
- -
-
-TensorPtr newTokens#
-

A view of newTokensSteps for the current token, [BS, BM].

-
- -
-
-std::vector<TensorPtr> newTokensVec#
-

A Vector of views on newTokensSteps for each token [BS, BM].

-
- -
-
-TensorPtr finishReasons#
-

Optional parameters FinishedState by decoding if any of the stop conditions are met or if DecodingInput.finished is true, [BS, BM]

-
- -
-
-TensorPtr finishedSum#
-

The sum of finished sequences per request, in pinned memory, [BS].

-
- -
-
-TensorPtr logProbs#
-

Mandatory parameters for Beam Search log-probility of generated tokens, [BS, BM, MSL], float

-
- -
-
-TensorPtr cumLogProbs#
-

Sum log-probility of all generated tokens, [BS, BM].

-
- -
-
-TensorPtr parentIds#
-

Index of the beam where the previous token is, [BS, BM, MSL].

-
- -
-
-TensorPtr lengths#
-

Total sequence lengths including padding, [BS, BM].

-
- -
-
-TensorPtr cacheIndirection#
-

K/V indirection for next generation step, [BS, BM, MSL].

-
- -
-
-TensorPtr logProbsTiled#
-

Buffer used to store the transpose of the logProbs, [MSL, BS, BM].

-
- -
-
-BeamHypotheses beamHypotheses#
-
- -
-
-std::optional<SpeculativeDecodingOutputs> speculativeDecodingOutputs#
-
- -
-
-std::optional<ExplicitDraftTokensBuffers::Inputs> explicitDraftTokensBuffers#
-
- -
-
-std::optional<LookaheadDecodingBuffers> lookaheadOutputs#
-
- -
-
-std::optional<EagleBuffers::Inputs> eagleBuffers#
-
- -
-
-

Public Static Attributes

-
-
-static float constexpr kNegativeInfinity = -1e20f#
-
- -
-
-
-class BeamHypotheses#
-
-

Public Functions

-
-
-void empty(BufferManager const &manager)#
-
- -
-
-void reshape( - -
-
SizeType32 batchSize,
-
SizeType32 beamWidth,
-
SizeType32 maxSequenceLength,
-
- -)#
-
- -
-
-void release()#
-
- -
-
-void init(BufferManager const &manager, TokenIdType endId)#
-
- -
-
-BeamHypotheses slice(SizeType32 batchIndex, SizeType32 size) const#
-
- -
-
-

Public Members

-
-
-TensorPtr outputIdsCBA#
-
- -
-
-TensorPtr logProbsCBA#
-
- -
-
-TensorPtr sequenceLengthsCBA#
-
- -
-
-TensorPtr cumLogProbsCBA#
-
- -
-
-TensorPtr normedScoresCBA#
-
- -
-
-TensorPtr numBeamsCBA#
-
- -
-
-TensorPtr minNormedScoresCBA#
-
- -
-
-TensorPtr batchDones#
-
- -
-
- -
-
-class SpeculativeDecodingOutputs#
-
-

Public Members

-
-
-TensorPtr nextDraftTokens#
-
- -
-
-TensorPtr nextDraftTokensLen#
-
- -
-
-TensorPtr prevDraftTokensLen#
-
- -
-
-TensorPtr acceptedTokensLen#
-
- -
-
-TensorPtr acceptedLengthsCumSum#
-
- -
-
-TensorPtr pathsOffsets#
-
- -
-
- -
- -
- -
- -
-
-

decodingInput.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class DecodingInput#
-
-#include <decodingInput.h>
-

Represents the inputs to the decoder.

-

This input type is assumed immutable. It represents whatever the decoder received initially, and can always be referred to as such.

-
-

Public Types

-
-
-using TensorConstPtr = ITensor::SharedConstPtr#
-
- -
-
-using TensorPtr = ITensor::SharedPtr#
-
- -
-
-

Public Functions

-
-
-DecodingInput() = default#
-
- -
-
-

Public Members

-
-
-SizeType32 step = {}#
-

Mandatory parameters The index of the decoding step we are on. Only used in Python runtime

-
- -
-
-SizeType32 maxLength = {}#
-

The maximum number of tokens to decode.

-
- -
-
-SizeType32 maxAttentionWindow = {}#
-

The maximum length of the attention window to consider while decoding.

-
- -
-
-SizeType32 sinkTokenLength = {}#
-

The number of tokens to use as attention sinks, https://arxiv.org/html/2309.17453v3.

-
- -
-
-SizeType32 batchSize = {}#
-

The number of samples in the batch.

-
- -
-
-std::vector<SizeType32> beamWidths#
-

The beam widths of each request, [batchSize].

-
- -
-
-SizeType32 maxStopWordsLen = {}#
-

The maximum value in the stopWordsLens tensor.

-
- -
-
-SizeType32 maxBadWordsLen = {}#
-

The maximum value in the badWordsLens tensor.

-
- -
-
-std::vector<TensorConstPtr> logitsVec#
-

The output of the model forward computation, a probability distribution over the vocabulary [batchSize][numGenTokens, beamWidth, vocabSizePadded] on gpu

-
- -
-
-TensorConstPtr endIds#
-

The end ids, [batchSize * beamWidth] on gpu.

-
- -
-
-TensorConstPtr batchSlots#
-

Address map of the linear batch id to to the seq slots, [batchSize] on pinned, int32_t.

-
- -
-
-TensorConstPtr finishReasons#
-

Optional parameters Finished states at current iteration (skip decoding step of a request if true), [batchSize, beamWidth] on gpu

-
- -
-
-TensorConstPtr sequenceLimitLength#
-

The maximum sequence length for each sequence in the batch, [batchSize] on gpu.

-
- -
-
-TensorConstPtr embeddingBias#
-
- -
-
-TensorConstPtr lengths#
-
- -
-
-std::vector<TensorPtr> badWordsLists#
-
- -
-
-TensorConstPtr badWordsPtrs#
-
- -
-
-TensorConstPtr badWordsLens#
-
- -
-
-std::vector<TensorPtr> stopWordsLists#
-
- -
-
-TensorConstPtr stopWordsPtrs#
-
- -
-
-TensorConstPtr stopWordsLens#
-
- -
-
-TensorConstPtr noRepeatNgramSize#
-
- -
-
-TensorPtr cacheIndirection#
-

Parameters for beam search KV cache index for beam search, [batchSize, beamWidth, maxSeqLen] on gpu

-
- -
-
-std::optional<std::vector<SizeType32>> generationSteps#
-

Steps of each request, for Variable-Beam-Width-Search, [batchSize].

-
- -
-
-std::optional<MedusaInputs> medusaInputs#
-
- -
-
-std::optional<ExplicitDraftTokensInputs> explicitDraftTokensInputs#
-
- -
-
-std::optional<LookaheadInputs> lookaheadInputs#
-
- -
-
-std::optional<ExternalDraftTokensInputs> externalDraftTokensInputs#
-
- -
-
-std::optional<EagleInputs> eagleInputs#
-
- -
-
-
-struct EagleInputs#
-
-

Public Members

-
-
-TensorConstPtr nextDraftTokens#
-
- -
-
-TensorConstPtr nextDraftLens#
-
- -
-
-TensorConstPtr nextDraftPaths#
-
- -
-
-TensorConstPtr lastDraftTokens#
-
- -
-
-TensorConstPtr lastDraftLens#
-
- -
-
-TensorConstPtr lastDraftPaths#
-
- -
-
-TensorConstPtr acceptedTokens#
-
- -
-
-TensorConstPtr acceptedLens#
-
- -
-
-TensorConstPtr acceptedPathIds#
-
- -
-
-TensorConstPtr chunkedContextNextTokens#
-
- -
-
-TensorConstPtr seqSlots#
-
- -
-
- -
-
-class ExplicitDraftTokensInputs#
-
-

Public Members

-
-
-TensorConstPtr nextDraftTokens#
-
- -
-
-TensorConstPtr nextFlatTokens#
-
- -
-
-TensorConstPtr nextDraftIndices#
-
- -
-
-TensorConstPtr nextDraftProbs#
-
- -
-
-TensorConstPtr lastDraftTokens#
-
- -
-
-TensorConstPtr lastDraftIndices#
-
- -
-
-TensorConstPtr masks#
-
- -
-
-TensorConstPtr packedPositionIds#
-
- -
-
-TensorConstPtr bestPathLengths#
-
- -
-
-TensorConstPtr bestPathIndices#
-
- -
-
-TensorConstPtr nextGenerationLengths#
-
- -
-
-TensorConstPtr lastPositionIdsBase#
-
- -
-
-TensorConstPtr lastGenerationLengths#
-
- -
-
-TensorConstPtr maxGenLengthDevice#
-
- -
-
-TensorConstPtr seqSlots#
-
- -
-
- -
-
-class ExternalDraftTokensInputs#
-
-

Public Members

-
-
-TensorPtr draftLogits#
-
- -
-
-TensorPtr draftLogitsHost#
-
- -
-
-TensorPtr draftProbs#
-
- -
-
-TensorPtr targetProbs#
-
- -
-
-TensorPtr numDraftTokens#
-
- -
-
-TensorPtr numDraftTokensHost#
-
- -
-
-TensorPtr draftTokenIds#
-
- -
-
-TensorPtr draftTokenIdsHost#
-
- -
-
-TensorPtr useDraftLogits#
-
- -
-
-TensorPtr useDraftLogitsHost#
-
- -
-
-SizeType32 step#
-
- -
-
-float constantThreshold#
-
- -
-
-bool useRandomAcceptanceThreshold#
-
- -
-
- -
-
-struct LookaheadInputs#
-
-

Public Members

-
-
-TensorPtr tokensPerStep#
-
- -
-
- -
-
-class MedusaInputs#
-
-

Public Members

-
-
-TensorConstPtr medusaPaths#
-

[batchSize, maxTokensPerStep, maxMedusaHeads + 1], on gpu

-
- -
-
-TensorConstPtr medusaTreeIds#
-

[batchSize, maxTokensPerStep], on gpu

-
- -
-
-std::vector<std::vector<TensorPtr>> medusaLogits#
-

[batchSize][maxAcceptedDraftTokensPerStep][maxDraftTokens + 1, vocabSizePadded], on gpu

-
- -
-
-TensorPtr medusaCurTokensPerStep#
-

[batchSize], on gpu

-
- -
-
-TensorConstPtr medusaTargetTokensPerStep#
-

[batchSize], on gpu

-
- -
-
- -
- -
- -
- -
-
-

worldConfig.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class WorldConfig#
-
-

Public Functions

-
-
-explicit WorldConfig( - -
-
SizeType32 tensorParallelism = 1,
-
SizeType32 pipelineParallelism = 1,
-
SizeType32 contextParallelism = 1,
-
SizeType32 rank = 0,
-
SizeType32 gpusPerNode = kDefaultGpusPerNode,
-
std::optional<std::vector<SizeType32>> const &deviceIds = std::nullopt,
-
bool enableAttentionDP = false,
-
- -)#
-
- -
-
-inline SizeType32 constexpr getSize() const noexcept#
-
- -
-
-inline SizeType32 constexpr getTensorParallelism() const noexcept#
-
- -
-
-inline bool constexpr isTensorParallel() const noexcept#
-
- -
-
-inline SizeType32 constexpr getPipelineParallelism() const noexcept#
-
- -
-
-inline bool constexpr isPipelineParallel() const noexcept#
-
- -
-
-inline SizeType32 constexpr getContextParallelism() const noexcept#
-
- -
-
-inline bool constexpr isContextParallel() const noexcept#
-
- -
-
-inline SizeType32 constexpr getRank() const noexcept#
-
- -
-
-inline SizeType32 constexpr getGpusPerNode() const noexcept#
-
- -
-
-inline SizeType32 getGpusPerGroup() const noexcept#
-
- -
-
-inline SizeType32 getDevice() const noexcept#
-
- -
-
-inline SizeType32 getDeviceOf(SizeType32 rank) const noexcept#
-
- -
-
-inline SizeType32 constexpr getPipelineParallelRank() const noexcept#
-
- -
-
-inline SizeType32 constexpr getTensorParallelRank() const noexcept#
-
- -
-
-inline SizeType32 constexpr getContextParallelRank() const noexcept#
-
- -
-
-inline SizeType32 constexpr getLocalRank() const noexcept#
-
- -
-
-inline SizeType32 constexpr getNodeRank() const noexcept#
-
- -
-
-inline SizeType32 constexpr getNodeRankOf( - -
-
SizeType32 rank,
-
- -) const noexcept#
-
- -
-
-inline bool constexpr isFirstPipelineParallelRank() const noexcept#
-
- -
-
-inline bool constexpr isLastPipelineParallelRank() const noexcept#
-

Is my rank the last rank in its pipeline?

-
- -
-
-inline bool constexpr isFirstTensorParallelRank() const noexcept#
-
- -
-
-inline bool constexpr isFirstContextParallelRank() const noexcept#
-
- -
-
-inline SizeType32 constexpr getLastRank() const noexcept#
-
- -
-
-inline bool constexpr enableAttentionDP() const noexcept#
-
- -
-
-std::vector<SizeType32> getPipelineParallelGroup() const#
-
- -
-
-std::vector<SizeType32> getTensorParallelGroup() const#
-
- -
-
-std::vector<SizeType32> getContextParallelGroup() const#
-
- -
-
-bool validMpiConfig() const#
-
- -
-
-

Public Static Functions

-
-
-static WorldConfig mpi( - -
-
SizeType32 gpusPerNode = kDefaultGpusPerNode,
-
std::optional<SizeType32> tensorParallelism = std::nullopt,
-
std::optional<SizeType32> pipelineParallelism = std::nullopt,
-
std::optional<SizeType32> contextParallelism = std::nullopt,
-
std::optional<std::vector<SizeType32>> const &deviceIds = std::nullopt,
-
bool enableAttentionDP = false,
-
- -)#
-
- -
-
-

Public Static Attributes

-
-
-static SizeType32 constexpr kDefaultGpusPerNode = 1#
-
- -
-
-

Private Members

-
-
-SizeType32 mTensorParallelism#
-
- -
-
-SizeType32 mPipelineParallelism#
-
- -
-
-SizeType32 mContextParallelism#
-
- -
-
-SizeType32 mRank#
-
- -
-
-SizeType32 mGpusPerNode#
-
- -
-
-bool mEnableAttentionDP#
-
- -
-
-std::vector<SizeType32> mDeviceIds#
-
- -
-
- -
- -
- -
-
-

gptDecoderBatched.h#

-
-
-namespace tensorrt_llm
-
-
-namespace batch_manager
-
- -
-
-namespace runtime
-
-
-class GptDecoderBatched : public tensorrt_llm::runtime::IGptDecoderBatched#
-
-#include <gptDecoderBatched.h>
-

GPT decoder class with support for in-flight batching.

-
-

Public Types

-
-
-using CudaStreamPtr = std::shared_ptr<CudaStream>#
-
- -
-
-using LlmRequestPtr = std::shared_ptr<tensorrt_llm::batch_manager::LlmRequest>#
-
- -
-
-using RequestVector = std::vector<LlmRequestPtr>#
-
- -
-
-using TensorPtr = ITensor::SharedPtr#
-
- -
-
-

Public Functions

-
-
-explicit GptDecoderBatched(CudaStreamPtr stream)#
-
- -
-
-virtual void setup( - -
-
executor::DecodingMode const &mode,
-
SizeType32 maxNumSequences,
-
SizeType32 maxBeamWidth,
-
nvinfer1::DataType dtype,
-
ModelConfig const &modelConfig,
-
WorldConfig const &worldConfig,
-
- -) override#
-

Setup the decoder before calling forward()

-
- -
-
-virtual void disableLookahead( - -
-
RequestVector const &genRequests,
-
TensorPtr const &batchSlots,
-
- -) override#
-

Disable Lookahead decoding.

-
- -
-
-virtual CudaEvent forwardAsync( - -
-
decoder::DecoderState const &decoderState,
-
decoder_batch::Input const &input,
-
- -) override#
-

Run one step for all requests without blocking the host process and return the token for synchronization.

-
- -
-
-virtual void forward( - -
-
decoder::DecoderState const &decoderState,
-
decoder_batch::Input const &input,
-
- -) override#
-

Run one step for all requests and wait for completion on the host.

-
- -
-
-virtual CudaEvent finalize( - -
-
decoder::DecoderState const &decoderState,
-
SizeType32 batchSlot,
-
SamplingConfig const &samplingConfig,
-
bool streaming,
-
- -) const override#
-

Gather final beam search results for request batchSlot. Result will only be available after event returned.

-
- -
-
-inline CudaStreamPtr getDecoderStream() const#
-
- -
-
-inline IGptDecoder &getUnderlyingDecoder() const#
-
- -
-
-inline BufferManager const &getBufferManager() const#
-
- -
-
-

Private Types

-
-
-using GptDecoderPtr = std::unique_ptr<IGptDecoder>#
-
- -
-
-

Private Functions

-
-
-void forwardDispatch( - -
-
decoder::DecoderState const &decoderState,
-
decoder_batch::Input const &input,
-
- -)#
-

Calls decoders for tokens per engine step.

-
- -
-
-

Private Members

-
-
-CudaStreamPtr mRuntimeStream#
-
- -
-
-CudaStreamPtr mDecoderStream#
-
- -
-
-BufferManager mBufferManager#
-
- -
-
-GptDecoderPtr mDecoder#
-
- -
-
- -
- -
- -
-
-

explicitDraftTokensBuffers.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class ExplicitDraftTokensBuffers#
-
-

Public Types

-
-
-using SizeType32 = runtime::SizeType32#
-
- -
-
-using ITensor = runtime::ITensor#
-
- -
-
-using BufferPtr = runtime::IBuffer::SharedPtr#
-
- -
-
-using TensorPtr = runtime::ITensor::SharedPtr#
-
- -
-
-using TensorMap = runtime::StringPtrMap<runtime::ITensor>#
-
- -
-
-

Public Functions

-
-
-ExplicitDraftTokensBuffers( - -
-
SizeType32 maxBatchSize,
-
SizeType32 maxBeamWidth,
-
runtime::BufferManager const &manager,
-
runtime::ModelConfig const &modelConfig,
-
runtime::WorldConfig const &worldConfig,
-
- -)#
-
- -
-
-void reshape( - -
-
SizeType32 numCtxSequences,
-
SizeType32 numGenSequences,
-
runtime::ModelConfig const &modelConfig,
-
- -)#
-
- -
-
-void setFromInputs( - -
-
SizeType32 numCtxSequences,
-
SizeType32 numGenSequences,
-
runtime::ITensor const &requestTypes,
-
ITensor const &seqSlots,
-
ExplicitDraftTokensBuffers::Inputs const &decoderBuffers,
-
ITensor const &contextPositionIds,
-
runtime::ModelConfig const &modelConfig,
-
runtime::WorldConfig const &worldConfig,
-
runtime::BufferManager const &manager,
-
runtime::CudaStream const &stream,
-
- -) const#
-
- -
-
-void insertInputTensors( - -
-
TensorMap &inputBuffers,
-
TensorMap &outputBuffers,
-
runtime::WorldConfig const &worldConfig,
-
- -) const#
-
- -
-
-

Public Members

-
-
-tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs engineInputs#
-
- -
-
-class tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs engineOutputs#
-
- -
-
-std::size_t scanTempStorageBytes = {0}#
-
- -
-
-BufferPtr scanTempStorage#
-
- -
-
-TensorPtr cumSumGenerationLengths#
-
- -
-
-

Private Functions

-
-
-template<typename T>
void setFromInputs( - -
-
SizeType32 numCtxSequences,
-
SizeType32 numGenSequences,
-
SizeType32 vocabSizePadded,
-
ITensor const &seqSlots,
-
ExplicitDraftTokensBuffers::Inputs const &draftBuffers,
-
ITensor const &contextPositionIds,
-
runtime::ExplicitDraftTokensModule const &explicitDraftTokensModule,
-
runtime::CudaStream const &stream,
-
- -) const#
-
- -
-
-
-class EngineInputs : public tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs#
-
-

Public Members

-
-
-TensorPtr requestTypesDevice#
-

[numSequences], on gpu

-
- -
-
-TensorPtr positionOffsets#
-

[numGenSequences]

-
- -
-
- -
-
-class EngineOutputs#
-
-

Public Members

-
-
-TensorPtr nextGenerationLengths#
-

[batchSize]

-
- -
-
-TensorPtr nextPositionOffsets#
-

[batchSize]

-
- -
-
-TensorPtr masks#
-

[batchSize, maxDecodingTokens, maxDecodingTokens], bool

-
- -
-
-TensorPtr nextDraftTokens#
-

[batchSize, maxNumPaths, maxPathLen]

-
- -
-
-TensorPtr nextDraftIndices#
-

[batchSize, maxNumPaths, maxPathLen]

-
- -
-
-TensorPtr nextDraftProbs#
-

[batchSize, maxNumPaths, maxDraftPathLen, vocabSize]

-
- -
-
-TensorPtr nextFlatTokens#
-

[batchSize * maxDecodingTokens]

-
- -
-
-TensorPtr bestPathLengths#
-

[batchSize]

-
- -
-
-TensorPtr bestPathIndices#
-

[batchSize]

-
- -
-
-TensorPtr maxGenToken#
-

[1]

-
- -
-
-TensorPtr totalGenToken#
-

[1]

-
- -
-
-TensorPtr packedPositionIds#
-

[batchSize * maxDecodingTokens]

-
- -
-
- -
-
-class Inputs#
-

Subclassed by tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs

-
-

Public Functions

-
-
-void create( - -
-
SizeType32 maxNumSequences,
-
runtime::BufferManager const &manager,
-
runtime::ModelConfig const &modelConfig,
-
runtime::WorldConfig const &worldConfig,
-
- -)#
-
- -
-
-

Public Members

-
-
-TensorPtr temperatures#
-

[maxBatchSize]

-
- -
-
-TensorPtr positionIdsBase#
-

[maxBatchSize]

-
- -
-
-TensorPtr generationLengths#
-

[maxBatchSize] or [numGenSequences]

-
- -
-
-TensorPtr randomDataSample#
-

[maxBatchSize]

-
- -
-
-TensorPtr randomDataValidation#
-

[maxBatchSize, maxNumPaths, maxPathDraftLen] or [numGenSequences, maxNumPaths, maxPathDraftLen]

-
- -
-
-TensorPtr draftTokens#
-

[maxBatchSize, maxNumPaths, maxPathLen] or [numGenSequences, maxNumPaths, maxPathLen]

-
- -
-
-TensorPtr draftIndices#
-

[maxBatchSize, maxNumPaths, maxPathLen] or [numGenSequences, maxNumPaths, maxPathLen]

-
- -
-
-TensorPtr draftProbs#
-

[maxBatchSize, maxNumPaths, maxPathDraftLen, vocabSize] or [numGenSequences, maxNumPaths, maxPathDraftLen, vocabSize]

-
- -
-
-TensorPtr packedMasks#
-

[maxBatchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] or [numGenSequences, maxDecodingTokens, ceil(maxDecodingTokens / 32)]

-
- -
-
-TensorPtr positionIds#
-

[maxBatchSize] or [numGenSequences]

-
- -
-
-TensorPtr maxGenLengthHost#
-
- -
-
-TensorPtr generationLengthsHost#
-
- -
-
-TensorPtr useSpecDecoding#
-
- -
-
- -
- -
- -
- -
-
-

bufferManager.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class BufferManager#
-
-#include <bufferManager.h>
-

A helper class for managing memory on host and device.

-
-

Public Types

-
-
-using IBufferPtr = IBuffer::UniquePtr#
-
- -
-
-using ITensorPtr = ITensor::UniquePtr#
-
- -
-
-using CudaStreamPtr = std::shared_ptr<CudaStream>#
-
- -
-
-using CudaMemPoolPtr = std::shared_ptr<CudaMemPool>#
-
- -
-
-

Public Functions

-
-
-explicit BufferManager(CudaStreamPtr stream, bool trimPool = false)#
-

Construct a BufferManager.

-
-
Parameters:
-

cudaStream – [in] The cuda stream to use for all operations on GPU (allocation, de-allocation, copying, etc.).

-
-
-
- -
-
-inline ~BufferManager()#
-

Destructor.

-
- -
-
-IBufferPtr gpu( - -
-
std::size_t size,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -) const#
-

Allocates an IBuffer of the given size on the GPU, using cudaMallocAsync.

-
- -
-
-ITensorPtr gpu( - -
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -) const#
-

Allocates an ITensor of the given dimensions on the GPU, using cudaMallocAsync.

-
- -
-
-IBufferPtr allocate( - -
-
MemoryType memoryType,
-
std::size_t size,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -) const#
-

Allocates an IBuffer of the given size and memory type.

-
- -
-
-ITensorPtr allocate( - -
-
MemoryType memoryType,
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -) const#
-

Allocates an ITensor of the given dimensions and memory type.

-
- -
-
-inline IBufferPtr emptyBuffer( - -
-
MemoryType memoryType,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -) const#
-

Create an empty IBuffer of the given memory type. It may be resized later.

-
- -
-
-inline ITensorPtr emptyTensor( - -
-
MemoryType memoryType,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -) const#
-

Create an empty ITensor of the given memory type. It may be reshaped later.

-
- -
-
-void setMem(IBuffer &buffer, int32_t value) const#
-

Set the contents of the given buffer to value.

-
- -
-
-void setZero(IBuffer &buffer) const#
-

Set the contents of the given buffer to zero.

-
- -
-
-void copy(void const *src, IBuffer &dst, MemoryType srcType) const#
-

Copy src to dst.

-
- -
-
-void copy(IBuffer const &src, void *dst, MemoryType dstType) const#
-

Copy src to dst.

-
- -
-
-inline void copy(void const *src, IBuffer &dst) const#
-

Copy src to dst.

-
- -
-
-inline void copy(IBuffer const &src, void *dst) const#
-

Copy src to dst.

-
- -
-
-void copy(IBuffer const &src, IBuffer &dst) const#
-

Copy src to dst.

-
- -
-
-IBufferPtr copyFrom(IBuffer const &src, MemoryType memoryType) const#
-

Copy src into a new IBuffer with a potentially different memory type.

-
- -
-
-ITensorPtr copyFrom(ITensor const &src, MemoryType memoryType) const#
-

Copy src into a new ITensor with a potentially different memory type.

-
- -
-
-template<typename T>
inline IBufferPtr copyFrom( - -
-
std::vector<T> const &src,
-
MemoryType memoryType,
-
- -) const#
-

Copy src into a new IBuffer with a potentially different memory type.

-
- -
-
-template<typename T>
inline ITensorPtr copyFrom( - -
-
T *src,
-
nvinfer1::Dims dims,
-
MemoryType memoryType,
-
- -) const#
-

Copy src into a new ITensor with a potentially different memory type.

-
- -
-
-template<typename T>
inline ITensorPtr copyFrom( - -
-
std::vector<T> const &src,
-
nvinfer1::Dims dims,
-
MemoryType memoryType,
-
- -) const#
-

Copy src into a new ITensor with a potentially different memory type.

-
- -
-
-CudaStream const &getStream() const#
-

Get the underlying cuda stream.

-
- -
-
-std::size_t memoryPoolReserved() const#
-

The current size of the memory reserved by the memory pool.

-
- -
-
-std::size_t memoryPoolUsed() const#
-

The current size of the memory used by the memory pool.

-
- -
-
-std::size_t memoryPoolFree() const#
-

The current size of the memory free in the memory pool.

-
- -
-
-void memoryPoolTrimTo(std::size_t size)#
-

Try to trim the memory reserved by the pool to size bytes. This synchronizes implicitly with the stream.

-
- -
-
-

Public Static Functions

-
-
-static IBufferPtr gpuSync( - -
-
std::size_t size,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates an IBuffer of the given size on the GPU, using cudaMalloc.

-
- -
-
-static ITensorPtr gpuSync( - -
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates an ITensor of the given dimensions on the GPU, using cudaMalloc.

-
- -
-
-static IBufferPtr cpu( - -
-
std::size_t size,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates an IBuffer of the given size on the CPU.

-
- -
-
-static ITensorPtr cpu( - -
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates an ITensor of the given dimensions on the CPU.

-
- -
-
-static IBufferPtr pinned( - -
-
std::size_t size,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates a pinned IBuffer of the given size on the CPU.

-
- -
-
-static ITensorPtr pinned( - -
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates a pinned ITensor of the given dimensions on the CPU.

-
- -
-
-static IBufferPtr pinnedPool( - -
-
std::size_t size,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates a pinned IBuffer of the given size on the CPU in the default memory pool.

-
- -
-
-static ITensorPtr pinnedPool( - -
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates a pinned ITensor of the given dimensions on the CPU in the default memory pool.

-
- -
-
-static IBufferPtr managed( - -
-
std::size_t size,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates an IBuffer of the given size in UVM.

-
- -
-
-static ITensorPtr managed( - -
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type = kBYTE_TYPE,
-
- -)#
-

Allocates an ITensor of the given dimensions in UVM.

-
- -
-
-static ITensorPtr ipcNvls( - -
-
std::set<int> ranks,
-
nvinfer1::Dims dims,
-
nvinfer1::DataType type,
-
- -)#
-

Allocates an ITensor of the given dimensions for NVLS.

-
- -
-
-

Public Static Attributes

-
-
-static auto constexpr kBYTE_TYPE = nvinfer1::DataType::kUINT8#
-
- -
-
-

Private Members

-
-
-CudaStreamPtr mStream#
-
- -
-
-CudaMemPoolPtr mPool#
-
- -
-
-bool const mTrimPool#
-
- -
-
-

Friends

-
-
-friend class ::BufferManagerTest
-
- -
-
- -
- -
- -
-
-

loraModule.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-

Functions

-
-
-inline std::ostream &operator<<( - -
-
std::ostream &output,
-
LoraModule const &module,
-
- -)#
-
- -
-
-
-class LoraModule#
-
-

Public Types

-
-
-enum class ModuleType : SizeType32#
-

Values:

-
-
-enumerator kINVALID#
-
- -
-
-enumerator kATTN_QKV#
-
- -
-
-enumerator kATTN_Q#
-
- -
-
-enumerator kATTN_K#
-
- -
-
-enumerator kATTN_V#
-
- -
-
-enumerator kATTN_DENSE#
-
- -
-
-enumerator kMLP_H_TO_4H#
-
- -
-
-enumerator kMLP_4H_TO_H#
-
- -
-
-enumerator kMLP_GATE#
-
- -
-
-enumerator kCROSS_ATTN_QKV#
-
- -
-
-enumerator kCROSS_ATTN_Q#
-
- -
-
-enumerator kCROSS_ATTN_K#
-
- -
-
-enumerator kCROSS_ATTN_V#
-
- -
-
-enumerator kCROSS_ATTN_DENSE#
-
- -
-
-enumerator kMOE_H_TO_4H#
-
- -
-
-enumerator kMOE_4H_TO_H#
-
- -
-
-enumerator kMOE_GATE#
-
- -
-
-enumerator kMOE_ROUTER#
-
- -
-
-enumerator kMLP_ROUTER#
-
- -
-
-enumerator kMLP_GATE_UP#
-
- -
- -
-
-using TensorPtr = ITensor::SharedPtr#
-
- -
-
-

Public Functions

-
-
-inline explicit constexpr LoraModule( - -
-
ModuleType const &t,
-
SizeType32 inDim,
-
SizeType32 outDim,
-
bool inDimFirst,
-
bool outDimFirst,
-
SizeType32 inTpSplitDim,
-
SizeType32 outTpSplitDim,
-
- -) noexcept#
-
- -
-
-inline explicit constexpr LoraModule() noexcept#
-
- -
-
-explicit constexpr LoraModule(LoraModule const &o) = default#
-
- -
-
-constexpr LoraModule &operator=(LoraModule const &o) = default#
-
- -
-
-inline SizeType32 constexpr flattenedInOutSize( - -
-
SizeType32 adapterSize,
-
bool isDora,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr inSize( - -
-
SizeType32 adapterSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr outSize( - -
-
SizeType32 adapterSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localInSize( - -
-
SizeType32 adapterSize,
-
SizeType32 tpSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localOutSize( - -
-
SizeType32 adapterSize,
-
SizeType32 tpSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localScalesSize( - -
-
SizeType32 tpSize,
-
bool isDora,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localInDim( - -
-
SizeType32 tpSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localOutDim( - -
-
SizeType32 tpSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localInAdapterSize( - -
-
SizeType32 adapterSize,
-
SizeType32 tpSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localOutAdapterSize( - -
-
SizeType32 adapterSize,
-
SizeType32 tpSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localInOutSize( - -
-
SizeType32 adapterSize,
-
SizeType32 tpSize,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr localTotalSize( - -
-
SizeType32 adapterSize,
-
SizeType32 tpSize,
-
bool isDora,
-
- -) const noexcept#
-
- -
-
-inline SizeType32 constexpr value() const noexcept#
-
- -
-
-inline std::string_view constexpr name() const noexcept#
-
- -
-
-inline SizeType32 constexpr inDim() const noexcept#
-
- -
-
-inline SizeType32 constexpr outDim() const noexcept#
-
- -
-
-inline bool constexpr inDimFirst() const noexcept#
-
- -
-
-inline bool constexpr outDimFirst() const noexcept#
-
- -
-
-inline SizeType32 constexpr inTpSplitDim() const noexcept#
-
- -
-
-inline SizeType32 constexpr outTpSplitDim() const noexcept#
-
- -
-
-

Public Static Functions

-
-
-static std::vector<LoraModule> createLoraModules( - -
-
std::vector<std::string> const &loraModuleNames,
-
SizeType32 hiddenSize,
-
SizeType32 mlpHiddenSize,
-
SizeType32 numAttentionHeads,
-
SizeType32 numKvAttentionHeads,
-
SizeType32 attentionHeadSize,
-
SizeType32 tpSize,
-
SizeType32 numExperts,
-
- -)#
-
- -
-
-static inline ModuleType constexpr toModuleType( - -
-
std::string_view const &name,
-
- -)#
-
- -
-
-static inline std::string_view constexpr toModuleName( - -
-
ModuleType t,
-
- -) noexcept#
-
- -
-
-static inline std::string_view constexpr toModuleName(SizeType32 id)#
-
- -
-
-

Private Members

-
-
-ModuleType mType#
-
- -
-
-SizeType32 mInDim#
-
- -
-
-SizeType32 mOutDim#
-
- -
-
-bool mInDimFirst#
-
- -
-
-bool mOutDimFirst#
-
- -
-
-SizeType32 mInTpSplitDim#
-
- -
-
-SizeType32 mOutTpSplitDim#
-
- -
-
- -
- -
- -
-
-

eagleBuffers.h#

-
-
-namespace tensorrt_llm
-
-
-namespace batch_manager
-
- -
-
-namespace runtime
-
-
-class EagleBuffers#
-
-

Public Types

-
-
-using LlmRequestPtr = std::shared_ptr<tensorrt_llm::batch_manager::LlmRequest>#
-
- -
-
-using RequestVector = std::vector<LlmRequestPtr>#
-
- -
-
-using SizeType32 = runtime::SizeType32#
-
- -
-
-using ITensor = runtime::ITensor#
-
- -
-
-using BufferPtr = runtime::IBuffer::SharedPtr#
-
- -
-
-using TensorPtr = runtime::ITensor::SharedPtr#
-
- -
-
-using TensorMap = runtime::StringPtrMap<runtime::ITensor>#
-
- -
-
-

Public Functions

-
-
-EagleBuffers( - -
-
SizeType32 maxBatchSize,
-
SizeType32 maxBeamWidth,
-
runtime::BufferManager const &manager,
-
runtime::ModelConfig const &modelConfig,
-
runtime::WorldConfig const &worldConfig,
-
executor::DecodingConfig const &decodingConfig,
-
- -)#
-
- -
-
-void reshape( - -
-
SizeType32 numCtxSequences,
-
SizeType32 numGenSequences,
-
runtime::ModelConfig const &modelConfig,
-
- -)#
-
- -
-
-void setFromInputs( - -
-
RequestVector const &contextRequests,
-
RequestVector const &genRequests,
-
runtime::ITensor const &requestTypes,
-
ITensor const &seqSlots,
-
EagleBuffers::Inputs const &decoderBuffers,
-
runtime::BufferManager const &manager,
-
runtime::ModelConfig const &modelConfig,
-
runtime::WorldConfig const &worldConfig,
-
- -) const#
-
- -
-
-void insertInputTensors( - -
-
TensorMap &inputBuffers,
-
TensorMap &outputBuffers,
-
runtime::WorldConfig const &worldConfig,
-
- -) const#
-
- -
-
-

Public Members

-
-
-Inputs engineInputs#
-
- -
-
-class tensorrt_llm::runtime::EagleBuffers::EngineOutputs engineOutputs#
-
- -
-
-

Private Functions

-
-
-template<typename T>
void setFromInputs( - -
-
RequestVector const &contextRequests,
-
RequestVector const &genRequests,
-
SizeType32 vocabSizePadded,
-
ITensor const &seqSlots,
-
EagleBuffers::Inputs const &draftBuffers,
-
runtime::EagleModule const &eagleModule,
-
runtime::BufferManager const &manager,
-
- -) const#
-
- -
-
-

Private Members

-
-
-std::size_t scanReduceTempStorageBytes = {0}#
-
- -
-
-float mDefaultPosteriorThreshold = {0.09f}#
-
- -
-
-bool mDoGreedySampling = {true}#
-
- -
-
-BufferPtr scanReduceTempStorage#
-
- -
-
-TensorPtr cumSumGenerationLengths#
-
- -
-
-TensorPtr maxGenerationLength#
-
- -
-
-TensorPtr chunkedContextNextTokensHost#
-
- -
-
-TensorPtr greedySamplingHost#
-
- -
-
-TensorPtr posteriorAlphaHost#
-
- -
-
-TensorPtr posteriorThresholdHost#
-
- -
-
-
-class EngineOutputs#
-
-

Public Members

-
-
-TensorPtr nextDraftTokens#
-

[batchSize, maxDecodingDraftTokens]

-
- -
-
-TensorPtr nextDraftLens#
-

[batchSize]

-
- -
-
-TensorPtr nextDraftPaths#
-

[batchSize, maxNumPaths, maxPathLen]

-
- -
-
-TensorPtr acceptedTokens#
-

[batchSize, maxPathLen]

-
- -
-
-TensorPtr acceptedLens#
-

[batchSize]

-
- -
-
-TensorPtr acceptedPaths#
-

[batchSize]

-
- -
-
-TensorPtr chunkedContextNextTokens#
-

[batchSize]

-
- -
-
- -
-
-class Inputs#
-
-

Public Functions

-
-
-void create( - -
-
SizeType32 maxNumSequences,
-
BufferManager const &manager,
-
ModelConfig const &modelConfig,
-
WorldConfig const &worldConfig,
-
- -)#
-
- -
-
-

Public Members

-
-
-TensorPtr temperatures#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr posteriorAlpha#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr posteriorThreshold#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr randomDataSample#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr randomDataValidation#
-

[maxBatchSize, maxDecodingTokens] or [numSequences, maxDecodingTokens]

-
- -
-
-TensorPtr draftTokens#
-

[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens]

-
- -
-
-TensorPtr draftLens#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr draftPaths#
-

[maxBatchSize, maxNumPaths, maxPathLen] or [numSequences, maxNumPaths, maxPathLen]

-
- -
-
-TensorPtr draftPathsHost#
-

[maxBatchSize, maxNumPaths, maxPathLen] or [numSequences, maxNumPaths, maxPathLen]

-
- -
-
-TensorPtr specDecodingGenerationLengths#
-

[maxBatchSize] or [numGenSequences]

-
- -
-
-TensorPtr specDecodingGenerationLengthsHost#
-

[maxBatchSize] or [numGenSequences]

-
- -
-
-TensorPtr specDecodingPackedMasks#
-

[maxBatchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] or [numGenSequences, maxDecodingTokens, ceil(maxDecodingTokens / 32)]

-
- -
-
-TensorPtr specDecodingPositionOffsets#
-

[maxBatchSize] or [numGenSequences]

-
- -
-
-TensorPtr eagleNetCtxRequestTypesHost#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr eagleNetCtxContextLengthsHost#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr eagleNetCtxPastKeyValueLengthsHost#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr eagleNetGenRequestTypesHost#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr eagleNetGenContextLengthsHost#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr eagleNetGenPastKeyValueLengthsHost#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr inputGenTokensHost#
-

[maxBatchSize * maxDecodingTokens] or [numSequences * maxDecodingTokens]

-
- -
-
-TensorPtr chunkedContextNextTokens#
-

[maxBatchSize] or [numSequences]

-
- -
-
-TensorPtr useSpecDecoding#
-

[1]

-
- -
-
-TensorPtr useDynamicTreeHost#
-

[1]

-
- -
-
-TensorPtr dynamicTreeMaxTopKHost#
-

[1]

-
- -
-
-TensorPtr prevScores#
-

[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens]

-
- -
-
-TensorPtr currentExpandIndices#
-

[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens]

-
- -
-
-TensorPtr allLayersScores#
-

[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens]

-
- -
-
-TensorPtr allLayersDraftTokenIds#
-

[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens]

-
- -
-
-TensorPtr allLayersDraftTokenIdsPredecessor#
-

[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens]

-
- -
-
- -
- -
- -
- -
-
-

speculativeDecodingMode.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class SpeculativeDecodingMode#
-
-

Public Types

-
-
-using UnderlyingType = std::uint8_t#
-
- -
-
-

Public Functions

-
-
-inline bool constexpr isNone() const#
-
- -
-
-inline bool constexpr isDraftTokensExternal() const#
-
- -
-
-inline bool constexpr isMedusa() const#
-
- -
-
-inline bool constexpr isLookaheadDecoding() const#
-
- -
-
-inline bool constexpr isExplicitDraftTokens() const#
-
- -
-
-inline bool constexpr isEagle() const#
-
- -
-
-inline bool constexpr updatesPositionIds() const#
-
- -
-
-inline bool constexpr requiresAttentionMask() const#
-
- -
-
-inline bool constexpr predictsDraftTokens() const#
-
- -
-
-inline bool constexpr needsKVCacheRewind() const#
-
- -
-
-inline bool constexpr variableDraftLength() const#
-
- -
-
-inline bool constexpr hasDraftLogits() const#
-
- -
-
-inline bool constexpr needsDecoderPrologue() const#
-
- -
-
-inline bool operator==(SpeculativeDecodingMode const &other) const#
-
- -
-
-inline explicit constexpr SpeculativeDecodingMode( - -
-
UnderlyingType state,
-
- -)#
-
- -
-
-

Public Static Functions

-
-
-static inline auto constexpr None()#
-
- -
-
-static inline auto constexpr DraftTokensExternal()#
-
- -
-
-static inline auto constexpr Medusa()#
-
- -
-
-static inline auto constexpr LookaheadDecoding()#
-
- -
-
-static inline auto constexpr ExplicitDraftTokens()#
-
- -
-
-static inline auto constexpr Eagle()#
-
- -
-
-

Private Functions

-
-
-inline bool constexpr anyBitSet(UnderlyingType bits) const#
-
- -
-
-inline bool constexpr allBitSet(UnderlyingType bits) const#
-
- -
-
-

Private Members

-
-
-UnderlyingType mState = {kNone}#
-
- -
-
-

Private Static Attributes

-
-
-static UnderlyingType constexpr kNone = {1U << 0U}#
-
- -
-
-static UnderlyingType constexpr kDraftTokensExternal = {1U << 1U}#
-
- -
-
-static UnderlyingType constexpr kMedusa = {1U << 2U}#
-
- -
-
-static UnderlyingType constexpr kLookaheadDecoding = {1U << 3U}#
-
- -
-
-static UnderlyingType constexpr kExplicitDraftTokens = {1U << 4U}#
-
- -
-
-static UnderlyingType constexpr kEagle = {1U << 5U}#
-
- -
-
- -
- -
- -
-
-

promptTuningParams.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-template<typename TTensor>
class GenericPromptTuningParams#
-
-

Public Types

-
-
-using TensorPtr = TTensor#
-
- -
-
-using SizeType32 = tensorrt_llm::runtime::SizeType32#
-
- -
-
-

Public Functions

-
-
-inline explicit GenericPromptTuningParams( - -
-
TensorPtr embeddingTable = TensorPtr(),
-
TensorPtr tasks = TensorPtr(),
-
TensorPtr vocabSize = TensorPtr(),
-
- -)#
-
- -
-
-

Public Members

-
-
-TensorPtr embeddingTable#
-
- -
-
-TensorPtr tasks#
-
- -
-
-TensorPtr vocabSize#
-
- -
-
-std::vector<bool> promptTuningEnabled#
-
- -
-
- -
-
-class PromptTuningParams : public tensorrt_llm::runtime::GenericPromptTuningParams<ITensor::SharedPtr>#
-
-

Public Types

-
-
-using TensorPtr = ITensor::SharedPtr#
-
- -
-
-using SizeType32 = GenericPromptTuningParams::SizeType32#
-
- -
-
-

Public Functions

-
-
-inline explicit PromptTuningParams( - -
-
TensorPtr embeddingTable = nullptr,
-
TensorPtr tasks = nullptr,
-
TensorPtr vocabSize = nullptr,
-
- -)#
-
- -
-
-void fillTasksTensor( - -
-
TensorPtr tasksHost,
-
SizeType32 batchSize,
-
SizeType32 numContextRequests,
-
std::vector<SizeType32> const &reqBeamWidths,
-
std::vector<SizeType32> const &reqPromptLengths,
-
BufferManager const &manager,
-
bool packedInput,
-
- -)#
-
- -
-
- -
- -
- -
-
-

gptDecoder.h#

-
-
-namespace tensorrt_llm
-
-
-namespace layers#
-
- -
-
-namespace runtime
-
-

Functions

-
-
-inline runtime::ITensor::SharedConstPtr getDefaultBatchSlots( - -
-
runtime::SizeType32 batchSize,
-
- -)#
-

Helper function to produce batch slots [0, 1, …, batchSize - 1] for paths that do not explicitly provide batch slots to the decoder.

-
- -
-
-
-template<typename T>
class GptDecoder : public virtual tensorrt_llm::runtime::IGptDecoder#
-
-

Public Types

-
-
-using CudaStreamPtr = BufferManager::CudaStreamPtr#
-
- -
-
-using TensorPtr = std::shared_ptr<ITensor>#
-
- -
-
-

Public Functions

-
-
-GptDecoder( - -
-
executor::DecodingMode const &mode,
-
size_t maxNumSequences,
-
size_t maxBeamWidth,
-
size_t vocabSize,
-
size_t vocabSizePadded,
-
CudaStreamPtr const &stream,
-
std::shared_ptr<SpeculativeDecodingModule const> speculativeDecodingModule = nullptr,
-
- -)#
-
- -
-
-virtual void setup( - -
-
SamplingConfig const &samplingConfig,
-
size_t batchSize,
-
TensorConstPtr const &batchSlots,
-
std::optional<DecodingOutput> const &output = std::nullopt,
-
std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt,
-
std::optional<std::vector<TensorConstPtr>> const &lookaheadPrompt = std::nullopt,
-
std::optional<std::vector<executor::LookaheadDecodingConfig>> const &lookaheadAlgoConfigs = std::nullopt,
-
- -) override#
-
-
Parameters:
-

explicitDraftTokensDType – is only used by ExplicitDraftTokens model to WAR the lack of bf16 decoder.

-
-
-
- -
-
-virtual void forwardAsync( - -
-
DecodingOutput &output,
-
DecodingInput const &input,
-
- -) override#
-
- -
-
-virtual void forwardSync( - -
-
DecodingOutput &output,
-
DecodingInput const &input,
-
- -) override#
-
- -
-
-inline virtual SamplingConfig const &getSamplingConfig() override#
-
- -
-
-virtual void disableLookahead( - -
-
std::optional<SamplingConfig> const &samplingConfig,
-
SizeType32 batchSize,
-
TensorConstPtr batchSlots,
-
- -) override#
-
- -
-
-

Private Members

-
-
-std::shared_ptr<BufferManager> mManager#
-
- -
-
-std::shared_ptr<tensorrt_llm::layers::DynamicDecodeLayer<T>> mDynamicDecodeLayer#
-
- -
-
-std::shared_ptr<tensorrt_llm::runtime::DecodingLayerWorkspace> mDecodingLayerWorkspace#
-
- -
-
-SamplingConfig mSamplingConfig#
-
- -
-
-size_t mMaxNumSequences#
-
- -
-
-size_t mVocabSize#
-
- -
-
-size_t mVocabSizePadded#
-
- -
-
-executor::DecodingMode mDecodingMode#
-
- -
-
- -
-
-class IGptDecoder#
-

Subclassed by tensorrt_llm::runtime::GptDecoder< T >

-
-

Public Types

-
-
-using TensorPtr = runtime::ITensor::SharedPtr#
-
- -
-
-using TensorConstPtr = runtime::ITensor::SharedConstPtr#
-
- -
-
-

Public Functions

-
-
-virtual ~IGptDecoder() = default#
-
- -
-
-virtual void setup( - -
-
SamplingConfig const &samplingConfig,
-
size_t batchSize,
-
TensorConstPtr const &batchSlots,
-
std::optional<DecodingOutput> const &output = std::nullopt,
-
std::optional<nvinfer1::DataType> explicitDraftTokensDType = std::nullopt,
-
std::optional<std::vector<TensorConstPtr>> const &lookaheadPrompt = std::nullopt,
-
std::optional<std::vector<executor::LookaheadDecodingConfig>> const &lookaheadAlgoConfigs = std::nullopt,
-
- -) = 0#
-
-
Parameters:
-

explicitDraftTokensDType – is only used by ExplicitDraftTokens model to WAR the lack of bf16 decoder.

-
-
-
- -
-
-virtual void forwardAsync( - -
-
DecodingOutput &output,
-
DecodingInput const &input,
-
- -) = 0#
-
- -
-
-virtual void forwardSync( - -
-
DecodingOutput &output,
-
DecodingInput const &input,
-
- -) = 0#
-
- -
-
-virtual SamplingConfig const &getSamplingConfig() = 0#
-
- -
-
-virtual void disableLookahead( - -
-
std::optional<SamplingConfig> const &samplingConfig,
-
SizeType32 batchSize,
-
TensorConstPtr batchSlots,
-
- -) = 0#
-
- -
-
-

Public Static Functions

-
-
-static inline std::unique_ptr<IGptDecoder> create( - -
-
executor::DecodingMode const &mode,
-
nvinfer1::DataType dtype,
-
size_t maxNumSequences,
-
size_t maxBeamWidth,
-
size_t vocabSize,
-
size_t vocabSizePadded,
-
BufferManager::CudaStreamPtr const &stream,
-
std::shared_ptr<SpeculativeDecodingModule const> const &speculativeDecodingModule = nullptr,
-
- -)#
-
- -
-
- -
- -
- -
-
-

memoryCounters.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class MemoryCounters#
-
-

Public Types

-
-
-using SizeType32 = std::size_t#
-
- -
-
-using DiffType = std::ptrdiff_t#
-
- -
-
-

Public Functions

-
-
-MemoryCounters() = default#
-
- -
-
-inline SizeType32 getGpu() const#
-
- -
-
-inline SizeType32 getCpu() const#
-
- -
-
-inline SizeType32 getPinned() const#
-
- -
-
-inline SizeType32 getUVM() const#
-
- -
-
-inline SizeType32 getPinnedPool() const#
-
- -
-
-inline DiffType getGpuDiff() const#
-
- -
-
-inline DiffType getCpuDiff() const#
-
- -
-
-inline DiffType getPinnedDiff() const#
-
- -
-
-inline DiffType getUVMDiff() const#
-
- -
-
-inline DiffType getPinnedPoolDiff() const#
-
- -
-
-template<MemoryType T>
inline void allocate(SizeType32 size)#
-
- -
-
-void allocate(MemoryType memoryType, SizeType32 size)#
-
- -
-
-template<MemoryType T>
inline void deallocate(SizeType32 size)#
-
- -
-
-void deallocate(MemoryType memoryType, SizeType32 size)#
-
- -
-
-std::string toString() const#
-
- -
-
-

Public Static Functions

-
-
-static MemoryCounters &getInstance()#
-
- -
-
-static std::string bytesToString(SizeType32 bytes, int precision = 2)#
-
- -
-
-static std::string bytesToString(DiffType bytes, int precision = 2)#
-
- -
-
-

Private Members

-
-
-std::atomic<SizeType32> mGpu = {}#
-
- -
-
-std::atomic<SizeType32> mCpu = {}#
-
- -
-
-std::atomic<SizeType32> mPinned = {}#
-
- -
-
-std::atomic<SizeType32> mUVM = {}#
-
- -
-
-std::atomic<SizeType32> mPinnedPool = {}#
-
- -
-
-std::atomic<DiffType> mGpuDiff = {}#
-
- -
-
-std::atomic<DiffType> mCpuDiff = {}#
-
- -
-
-std::atomic<DiffType> mPinnedDiff = {}#
-
- -
-
-std::atomic<DiffType> mUVMDiff = {}#
-
- -
-
-std::atomic<DiffType> mPinnedPoolDiff = {}#
-
- -
-
- -
- -
- -
-
-

ipcNvlsMemory.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-

Functions

-
-
-void MPI_group_barrier(std::set<int> ranks)#
-
- -
-
-bool ipcNvlsSupported()#
-
- -
-
-IpcNvlsHandle *ipcNvlsAllocate(size_t size, std::set<int> ranks)#
-
- -
-
-void ipcNvlsFree(IpcNvlsHandle *handle)#
-
- -
-
-
-template<typename T>
class DeviceAllocationNvls#
-
-

Public Functions

-
-
-DeviceAllocationNvls() = default#
-
- -
-
-inline ~DeviceAllocationNvls()#
-
- -
-
-inline void reset(size_t size, std::set<int> ranks)#
-
- -
-
-inline T *getMulticastPointer() const#
-
- -
-
-inline T *getUnicastPointer() const#
-
- -
-
-inline T **getIpcUnicastPointers()#
-
- -
-
-inline size_t getCapacity() const#
-
- -
-
-inline void free()#
-
- -
-
-

Private Members

-
-
-size_t _capacity = 0#
-
- -
-
-IpcNvlsHandle *_handle#
-
- -
-
- -
-
-struct IpcNvlsHandle#
-
-

Public Members

-
-
-size_t size = 0#
-
- -
-
-uintptr_t uc_ptr = 0#
-
- -
-
-uintptr_t mc_ptr = 0#
-
- -
-
-std::vector<uintptr_t> ipc_uc_ptrs#
-
- -
-
-CUdeviceptr uc_va#
-
- -
-
-CUdeviceptr mc_va#
-
- -
-
-std::vector<CUdeviceptr> ipc_uc_vas#
-
- -
-
-CUmemGenericAllocationHandle uc_handle#
-
- -
-
-CUmemGenericAllocationHandle mc_handle#
-
- -
-
-std::vector<CUmemGenericAllocationHandle> ipc_uc_handles#
-
- -
-
- -
- -
- -
-
-

rawEngine.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-
-class RawEngine#
-
-

Public Types

-
-
-enum Type#
-

Values:

-
-
-enumerator FilePath#
-
- -
-
-enumerator AddressWithSize#
-
- -
-
-enumerator HostMemory#
-
- -
- -
-
-

Public Functions

-
-
-inline explicit RawEngine(std::filesystem::path enginePath) noexcept#
-
- -
-
-inline explicit RawEngine( - -
-
void const *engineAddr,
-
std::size_t engineSize,
-
- -) noexcept#
-
- -
-
-inline explicit RawEngine( - -
-
nvinfer1::IHostMemory const *engineBuffer,
-
- -) noexcept#
-
- -
-
-inline Type getType() const#
-
- -
-
-inline std::filesystem::path getPath() const#
-
- -
-
-inline std::optional<std::filesystem::path> getPathOpt() const#
-
- -
-
-inline void setPath(std::filesystem::path enginePath)#
-
- -
-
-inline std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> const &getManagedWeightsMapOpt( - -
-
- -) const#
-
- -
-
-inline void setManagedWeightsMap( - -
-
std::map<std::string, tensorrt_llm::executor::Tensor> managedWeightsMap,
-
- -)#
-
- -
-
-inline void const *getAddress() const#
-
- -
-
-inline std::size_t getSize() const#
-
- -
-
-inline nvinfer1::IHostMemory const *getHostMemory() const#
-
- -
-
-

Public Members

-
-
-void const *mEngineAddr = {}#
-
- -
-
-std::size_t mEngineSize = {}#
-
- -
-
-

Private Members

-
-
-Type mType#
-
- -
-
-std::optional<std::filesystem::path> mEnginePath#
-
- -
-
-struct tensorrt_llm::runtime::RawEngine
-
- -
-
-nvinfer1::IHostMemory const *mEngineBuffer = {}#
-
- -
-
-std::optional<std::map<std::string, tensorrt_llm::executor::Tensor>> mManagedWeightsMap#
-
- -
-
- -
- -
-

ipcUtils.h#

@@ -11123,1220 +12504,8 @@ one more than decoding draft tokens for prediction from primary head

-
-

iBuffer.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-

Typedefs

-
-
-template<typename T>
using PointerElementType = typename std::remove_reference_t<T>::element_type#
-
- -
-
-

Enums

-
-
-enum class MemoryType : std::int32_t#
-

Values:

-
-
-enumerator kGPU#
-
- -
-
-enumerator kCPU#
-
- -
-
-enumerator kPINNED#
-
- -
-
-enumerator kUVM#
-
- -
-
-enumerator kPINNEDPOOL#
-
- -
- -
-
-

Functions

-
-
-template<typename T>
std::shared_ptr<std::remove_const_t<T>> constPointerCast( - -
-
std::shared_ptr<T> const &ptr,
-
- -) noexcept#
-
- -
-
-template<typename T, typename D>
std::shared_ptr<std::remove_const_t<T>> constPointerCast( - -
-
std::unique_ptr<T, D> &&ptr,
-
- -) noexcept#
-
- -
-
-template<typename T>
T const *bufferCast(IBuffer const &buffer)#
-

Gets a typed pointer to the constant underlying data of the buffer.

-
-
Template Parameters:
-

T – The type of the underlying data.

-
-
Parameters:
-

buffer – The buffer to get a pointer to.

-
-
Returns:
-

A pointer to constant T.

-
-
-
- -
-
-template<typename T>
T *bufferCast(IBuffer &buffer)#
-

Gets a typed pointer to the underlying data of the buffer.

-
-
Template Parameters:
-

T – The type of the underlying data.

-
-
Parameters:
-

buffer – The buffer to get a pointer to.

-
-
Returns:
-

A pointer to T.

-
-
-
- -
-
-template<typename T>
T *bufferCastOrNull( - -
-
IBuffer::SharedPtr const &bufferPtr,
-
- -)#
-

Retrieves a T typed pointer to the underlying data of the buffer pointed to by the bufferPtr, or nullptr if the bufferPtr is null.

-
-
Template Parameters:
-

T – The type of the underlying data.

-
-
Parameters:
-

bufferPtr – A possibly null shared ptr.

-
-
Returns:
-

A pointer to T, possibly nullptr.

-
-
-
- -
-
-template<typename T>
T const *bufferCastOrNull( - -
-
IBuffer::SharedConstPtr const &bufferPtr,
-
- -)#
-

Retrieves a T const typed pointer to the underlying data of the buffer pointed to by the bufferPtr, or nullptr if the bufferPtr is null.

-
-
Template Parameters:
-

T – The type of the underlying data.

-
-
Parameters:
-

bufferPtr – A possibly null shared ptr.

-
-
Returns:
-

A pointer to const T, possibly nullptr.

-
-
-
- -
-
-template<typename T>
T *bufferCastOrNull( - -
-
std::optional<IBuffer::SharedPtr> const &optionalBufferPtr,
-
- -)#
-

Retrieves a T typed pointer to the underlying data of the buffer pointed to by the buffer pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value.

-
-
Template Parameters:
-

T – The type of the underlying data.

-
-
Parameters:
-

optionalBufferPtr – A possibly empty optional.

-
-
Returns:
-

A pointer to T, possibly nullptr.

-
-
-
- -
-
-template<typename T>
T const *bufferCastOrNull( - -
-
std::optional<IBuffer::SharedConstPtr> const &optionalBufferPtr,
-
- -)#
-

Retrieves a T const typed pointer to the underlying data of the buffer pointed to by the buffer pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value.

-
-
Template Parameters:
-

T – The type of the underlying data.

-
-
Parameters:
-

optionalBufferPtr – A possibly empty optional.

-
-
Returns:
-

A pointer to const T, possibly nullptr.

-
-
-
- -
-
-std::ostream &operator<<( - -
-
std::ostream &output,
-
IBuffer const &buffer,
-
- -)#
-

Utility function to print a buffer.

-
- -
-
-
-class BufferDataType#
-
-#include <iBuffer.h>
-

A wrapper around nvinfer1::DataType that provides a support for pointer types.

-
-

Public Functions

-
-
-inline constexpr BufferDataType( - -
-
nvinfer1::DataType dataType,
-
bool _unsigned = false,
-
bool pointer = false,
-
- -)#
-
- -
-
-inline constexpr operator nvinfer1::DataType() const noexcept#
-
- -
-
-inline constexpr nvinfer1::DataType getDataType() const noexcept#
-
- -
-
-inline constexpr bool isPointer() const noexcept#
-
- -
-
-inline constexpr bool isUnsigned() const#
-
- -
-
-inline constexpr std::size_t getSize() const noexcept#
-
- -
-
-inline constexpr std::size_t getSizeInBits() const noexcept#
-
- -
-
-

Public Static Attributes

-
-
-static auto constexpr kTrtPointerType = nvinfer1::DataType::kINT64#
-
- -
-
-

Private Members

-
-
-nvinfer1::DataType mDataType#
-
- -
-
-bool mUnsigned#
-
- -
-
-bool mPointer#
-
- -
-
- -
-
-template<typename T>
class BufferRange : public tensorrt_llm::common::ArrayView<T>#
-
-

Public Types

-
-
-using Base = tensorrt_llm::common::ArrayView<T>#
-
- -
-
-

Public Functions

-
-
-inline BufferRange(T *data, size_type size)#
-
- -
-
-template<typename U = T, std::enable_if_t<!std::is_const_v<U>, bool> = true>
inline explicit BufferRange( - -
-
IBuffer &buffer,
-
- -)#
-
- -
-
-template<typename U = T, std::enable_if_t<std::is_const_v<U>, bool> = true>
inline explicit BufferRange( - -
-
IBuffer const &buffer,
-
- -)#
-
- -
-
- -
-
-template<nvinfer1::DataType kDataType, bool kIsUnsigned = false, bool kIsPointer = false>
struct DataTypeTraits#
-
-#include <iBuffer.h>
-

For converting a TensorRT data type to a C++ data type.

-
- -
-
-template<nvinfer1::DataType kDataType, bool kUnsigned>
struct DataTypeTraits<kDataType, kUnsigned, true>#
-
-

Public Types

-
-
-using type = typename DataTypeTraits<kDataType, kUnsigned, false>::type*#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "*"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<bool kUnsigned>
struct DataTypeTraits<nvinfer1::DataType::kBOOL, kUnsigned>#
-
-

Public Types

-
-
-using type = bool#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "bool"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<>
struct DataTypeTraits<nvinfer1::DataType::kFLOAT>#
-
-

Public Types

-
-
-using type = float#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "float"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<>
struct DataTypeTraits<nvinfer1::DataType::kHALF>#
-
-

Public Types

-
-
-using type = half#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "half"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<>
struct DataTypeTraits<nvinfer1::DataType::kINT32>#
-
-

Public Types

-
-
-using type = std::int32_t#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "int32"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<>
struct DataTypeTraits<nvinfer1::DataType::kINT32, true>#
-
-

Public Types

-
-
-using type = std::uint32_t#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "uint32"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<>
struct DataTypeTraits<nvinfer1::DataType::kINT64>#
-
-

Public Types

-
-
-using type = std::int64_t#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "int64"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<>
struct DataTypeTraits<nvinfer1::DataType::kINT64, true>#
-
-

Public Types

-
-
-using type = std::uint64_t#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "uint64"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<>
struct DataTypeTraits<nvinfer1::DataType::kINT8>#
-
-

Public Types

-
-
-using type = std::int8_t#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "int8"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-template<bool kUnsigned>
struct DataTypeTraits<nvinfer1::DataType::kUINT8, kUnsigned>#
-
-

Public Types

-
-
-using type = std::uint8_t#
-
- -
-
-

Public Static Attributes

-
-
-static char constexpr name[] = "uint8"#
-
- -
-
-static auto constexpr size = sizeof(type)#
-
- -
-
- -
-
-class IBuffer#
-

Subclassed by tensorrt_llm::runtime::ITensor

-
-

Public Types

-
-
-using UniquePtr = std::unique_ptr<IBuffer>#
-
- -
-
-using SharedPtr = std::shared_ptr<IBuffer>#
-
- -
-
-using UniqueConstPtr = std::unique_ptr<IBuffer const>#
-
- -
-
-using SharedConstPtr = std::shared_ptr<IBuffer const>#
-
- -
-
-using DataType = nvinfer1::DataType#
-
- -
-
-

Public Functions

-
-
-virtual void *data() = 0#
-

Returns a pointer to underlying array.

-
- -
-
-virtual void const *data() const = 0#
-

Returns a pointer to underlying array.

-
- -
-
-inline virtual void *data(std::size_t index)#
-

Returns a pointer to the underlying array at a given element index.

-
- -
-
-inline virtual void const *data(std::size_t index) const#
-

Returns a pointer to the underlying array at a given element index.

-
- -
-
-virtual std::size_t getSize() const = 0#
-

Returns the size (in number of elements) of the buffer.

-
- -
-
-inline virtual std::size_t getSizeInBytes() const#
-

Returns the size (in bytes) of the buffer.

-
- -
-
-virtual std::size_t getCapacity() const = 0#
-

Returns the capacity of the buffer.

-
- -
-
-virtual DataType getDataType() const = 0#
-

Returns the data type of the buffer.

-
- -
-
-virtual char const *getDataTypeName() const#
-
- -
-
-virtual MemoryType getMemoryType() const = 0#
-

Returns the memory type of the buffer.

-
- -
-
-virtual char const *getMemoryTypeName() const#
-
- -
-
-virtual void resize(std::size_t newSize) = 0#
-

Resizes the buffer. This is a no-op if the new size is smaller than or equal to the current capacity.

-
- -
-
-virtual void release() = 0#
-

Releases the buffer. It will be reset to nullptr.

-
- -
-
-virtual ~IBuffer() = default#
-
- -
-
-IBuffer(IBuffer const&) = delete#
-

Not allowed to copy.

-
- -
-
-IBuffer &operator=(IBuffer const&) = delete#
-

Not allowed to copy.

-
- -
-
-

Public Static Functions

-
-
-static char const *getDataTypeName(DataType dataType)#
-
- -
-
-static UniquePtr slice( - -
-
SharedPtr buffer,
-
std::size_t offset,
-
std::size_t size,
-
- -)#
-

Creates a sliced view on the underlying buffer. The view will have the same data type as buffer.

-
-
Parameters:
-
    -

  • buffer – The buffer to view.

    • -

  • offset – The offset of the view.

    • -

  • size – The size of the view.

    • -
-
-
Returns:
-

A view on the buffer.

-
-
-
- -
-
-template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( - -
-
TConstPtr &&tensor,
-
std::size_t offset,
-
std::size_t size,
-
- -)#
-
- -
-
-static inline UniquePtr slice(SharedPtr buffer, std::size_t offset)#
-
- -
-
-template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr slice( - -
-
TConstPtr &&tensor,
-
std::size_t offset,
-
- -)#
-
- -
-
-static inline UniquePtr view(SharedPtr tensor)#
-

Returns a view on the underlying tensor which can be independently resized.

-
-
Parameters:
-

tensor – The tensor to view.

-
-
Returns:
-

A view on the tensor.

-
-
-
- -
-
-static inline UniquePtr view(SharedPtr tensor, std::size_t size)#
-

Returns a view on the underlying tensor with a different size.

-
-
Parameters:
-
    -

  • tensor – The tensor to view.

    • -

  • size – The size of the view.

    • -
-
-
Returns:
-

A view on the tensor.

-
-
-
- -
-
-template<typename TConstPtr, std::enable_if_t<std::is_const_v<PointerElementType<TConstPtr>>, int> = 0>
static inline UniqueConstPtr view( - -
-
TConstPtr &&tensor,
-
std::size_t size,
-
- -)#
-
- -
-
-static UniquePtr wrap( - -
-
void *data,
-
DataType type,
-
std::size_t size,
-
std::size_t capacity,
-
- -)#
-

Wraps the given data in an IBuffer. The IBuffer will not own the underlying data and cannot be resized beyond capacity.

-
-
Parameters:
-
    -

  • data – The data to wrap.

    • -

  • type – The data type of the data.

    • -

  • size – The size of the buffer.

    • -

  • capacity – The capacity of the buffer.

    • -
-
-
Returns:
-

An IBuffer.

-
-
-
- -
-
-static inline UniquePtr wrap( - -
-
void *data,
-
DataType type,
-
std::size_t size,
-
- -)#
-
- -
-
-template<typename T>
static inline UniquePtr wrap( - -
-
T *data,
-
std::size_t size,
-
std::size_t capacity,
-
- -)#
-
- -
-
-template<typename T>
static inline UniquePtr wrap( - -
-
T *data,
-
std::size_t size,
-
- -)#
-
- -
-
-template<typename T>
static inline UniquePtr wrap( - -
-
std::vector<T> &v,
-
- -)#
-
- -
-
-static MemoryType memoryType(void const *data)#
-

Determine the memory type of a pointer.

-
- -
-
-

Protected Functions

-
-
-IBuffer() = default#
-
- -
-
-inline std::size_t toBytes(std::size_t size) const#
-

Returns an array index or size in bytes.

-
- -
-
- -
-
-template<MemoryType T>
struct MemoryTypeString#
-
- -
-
-template<>
struct MemoryTypeString<MemoryType::kCPU>#
-
-

Public Static Attributes

-
-
-static auto constexpr value = "CPU"#
-
- -
-
- -
-
-template<>
struct MemoryTypeString<MemoryType::kGPU>#
-
-

Public Static Attributes

-
-
-static auto constexpr value = "GPU"#
-
- -
-
- -
-
-template<>
struct MemoryTypeString<MemoryType::kPINNED>#
-
-

Public Static Attributes

-
-
-static auto constexpr value = "PINNED"#
-
- -
-
- -
-
-template<>
struct MemoryTypeString<MemoryType::kPINNEDPOOL>#
-
-

Public Static Attributes

-
-
-static auto constexpr value = "PINNEDPOOL"#
-
- -
-
- -
-
-template<>
struct MemoryTypeString<MemoryType::kUVM>#
-
-

Public Static Attributes

-
-
-static auto constexpr value = "UVM"#
-
- -
-
- -
-
-template<typename T, bool = false>
struct TRTDataType#
-
-#include <iBuffer.h>
-

For converting a C++ data type to a TensorRT data type.

-
- -
-
-template<>
struct TRTDataType<bool>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = nvinfer1::DataType::kBOOL#
-
- -
-
- -
-
-template<>
struct TRTDataType<float>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = nvinfer1::DataType::kFLOAT#
-
- -
-
- -
-
-template<>
struct TRTDataType<half>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = nvinfer1::DataType::kHALF#
-
- -
-
- -
-
-template<>
struct TRTDataType<kernels::FinishedState>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = TRTDataType<kernels::FinishedState::UnderlyingType>::value#
-
- -
-
- -
-
-template<>
struct TRTDataType<kernels::KVCacheIndex>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = TRTDataType<kernels::KVCacheIndex::UnderlyingType>::value#
-
- -
-
- -
-
-template<>
struct TRTDataType<runtime::RequestType>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = TRTDataType<std::underlying_type_t<runtime::RequestType>>::value#
-
- -
-
- -
-
-template<>
struct TRTDataType<std::int32_t>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = nvinfer1::DataType::kINT32#
-
- -
-
- -
-
-template<>
struct TRTDataType<std::int64_t>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = nvinfer1::DataType::kINT64#
-
- -
-
- -
-
-template<>
struct TRTDataType<std::int8_t>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = nvinfer1::DataType::kINT8#
-
- -
-
- -
-
-template<>
struct TRTDataType<std::uint32_t>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = BufferDataType{nvinfer1::DataType::kINT32, true}#
-
- -
-
- -
-
-template<>
struct TRTDataType<std::uint64_t>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = BufferDataType{nvinfer1::DataType::kINT64, true}#
-
- -
-
- -
-
-template<>
struct TRTDataType<std::uint8_t>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = nvinfer1::DataType::kUINT8#
-
- -
-
- -
-
-template<typename T>
struct TRTDataType<T*>#
-
-

Public Static Attributes

-
-
-static auto constexpr value = BufferDataType{kUnderlyingType.getDataType(), kUnderlyingType.isUnsigned(), true}#
-
- -
-
-

Private Static Attributes

-
-
-static auto constexpr kUnderlyingType = BufferDataType{TRTDataType<std::remove_const_t<T>, false>::value}#
-
- -
-
- -
-
-template<>
struct TRTDataType<void*>#
-
-

Public Static Attributes

-
-
-static constexpr auto value = BufferDataType::kTrtPointerType#
-
- -
-
- -
- -
- -
-
-

gptJsonConfig.h#

+
+

memoryCounters.h#

namespace tensorrt_llm
@@ -12344,398 +12513,172 @@ one more than decoding draft tokens for prediction from primary head

namespace runtime
-
-class GptJsonConfig#
+
+class MemoryCounters#
+

Public Types

+
+
+using SizeType32 = std::size_t#
+
+ +
+
+using DiffType = std::ptrdiff_t#
+
+ +
+

Public Functions

-
-inline GptJsonConfig( - -
-
std::string name,
-
std::string version,
-
std::string precision,
-
SizeType32 tensorParallelism,
-
SizeType32 pipelineParallelism,
-
SizeType32 contextParallelism,
-
SizeType32 gpusPerNode,
-
ModelConfig modelConfig,
-
std::optional<RuntimeDefaults> runtimeDefaults = std::nullopt,
-
- -)#
+
+MemoryCounters() = default#
-
-inline ModelConfig const &getModelConfig() const#
+
+inline SizeType32 getGpu() const#
-
-inline ModelConfig &getModelConfigMutable()#
+
+inline SizeType32 getCpu() const#
-
-inline std::string const &getName() const#
+
+inline SizeType32 getPinned() const#
-
-inline std::string const &getVersion() const#
+
+inline SizeType32 getUVM() const#
-
-inline std::string const &getPrecision() const#
+
+inline SizeType32 getPinnedPool() const#
-
-inline SizeType32 constexpr getTensorParallelism() const#
+
+inline DiffType getGpuDiff() const#
-
-inline SizeType32 constexpr getPipelineParallelism() const#
+
+inline DiffType getCpuDiff() const#
-
-inline SizeType32 constexpr getContextParallelism() const#
+
+inline DiffType getPinnedDiff() const#
-
-inline SizeType32 constexpr getGpusPerNode() const#
+
+inline DiffType getUVMDiff() const#
-
-inline SizeType32 constexpr getWorldSize() const#
+
+inline DiffType getPinnedPoolDiff() const#
-
-inline std::optional<RuntimeDefaults> getRuntimeDefaults() const#
+
+template<MemoryType T>
inline void allocate(SizeType32 size)#
-
-std::string engineFilename( - -
-
WorldConfig const &worldConfig,
-
std::string const &model,
-
- -) const#
+
+void allocate(MemoryType memoryType, SizeType32 size)#
-
-inline std::string engineFilename( +
+template<MemoryType T>
inline void deallocate(SizeType32 size)#
+
-
-
WorldConfig const &worldConfig,
-
+
+
+void deallocate(MemoryType memoryType, SizeType32 size)#
+
-) const#
+
+
+std::string toString() const#

Public Static Functions

-
-static GptJsonConfig parse(std::string const &json)#
+
+static MemoryCounters &getInstance()#
-
-static GptJsonConfig parse(std::istream &json)#
+
+static std::string bytesToString(SizeType32 bytes, int precision = 2)#
-
-static GptJsonConfig parse(std::filesystem::path const &path)#
+
+static std::string bytesToString(DiffType bytes, int precision = 2)#

Private Members

-
-std::string const mName#
+
+std::atomic<SizeType32> mGpu = {}#
-
-std::string const mVersion#
+
+std::atomic<SizeType32> mCpu = {}#
-
-std::string const mPrecision#
+
+std::atomic<SizeType32> mPinned = {}#
-
-SizeType32 const mTensorParallelism#
+
+std::atomic<SizeType32> mUVM = {}#
-
-SizeType32 const mPipelineParallelism#
+
+std::atomic<SizeType32> mPinnedPool = {}#
-
-SizeType32 const mContextParallelism#
+
+std::atomic<DiffType> mGpuDiff = {}#
-
-SizeType32 const mGpusPerNode#
+
+std::atomic<DiffType> mCpuDiff = {}#
-
-ModelConfig mModelConfig#
+
+std::atomic<DiffType> mPinnedDiff = {}#
-
-std::optional<RuntimeDefaults> mRuntimeDefaults#
-
- -
-
- -
- - - -
-
-

loraCachePageManagerConfig.h#

-
-
-namespace tensorrt_llm
-
-
-namespace runtime
-
-

Functions

-
-
-inline std::ostream &operator<<( - -
-
std::ostream &os,
-
LoraCachePageManagerConfig const &c,
-
- -)#
-
- -
-
-inline std::string to_string(LoraCachePageManagerConfig const &c)#
-
- -
-
-
-class LoraCachePageManagerConfig#
-
-#include <loraCachePageManagerConfig.h>
-

Configuration for LoraCachePageManager

-

See LoraCache docs for description of pages, slots, and page blocks.

-
-

Public Functions

-
-
-inline explicit constexpr LoraCachePageManagerConfig( - -
-
runtime::MemoryType memType,
-
nvinfer1::DataType dType,
-
SizeType32 totalNumPages,
-
SizeType32 maxPagesPerBlock,
-
SizeType32 slotsPerPage,
-
SizeType32 pageWidth,
-
SizeType32 numCopyStreams,
-
- -)#
-
- -
-
-inline runtime::MemoryType constexpr getMemoryType() const noexcept#
-
- -
-
-inline void constexpr setMemoryType( - -
-
runtime::MemoryType const &memoryType,
-
- -) noexcept#
-
- -
-
-inline nvinfer1::DataType constexpr getDataType() const noexcept#
-
- -
-
-inline void constexpr setDataType( - -
-
nvinfer1::DataType const &dtype,
-
- -) noexcept#
-
- -
-
-inline SizeType32 constexpr getTotalNumPages() const noexcept#
-
- -
-
-inline void constexpr setTotalNumPage( - -
-
SizeType32 const &totalNumPages,
-
- -) noexcept#
-
- -
-
-inline SizeType32 constexpr getMaxPagesPerBlock() const noexcept#
-
- -
-
-inline void constexpr setMaxPagesPerBlock( - -
-
SizeType32 const &maxPagesPerBlock,
-
- -) noexcept#
-
- -
-
-inline SizeType32 constexpr getSlotsPerPage() const noexcept#
-
- -
-
-inline void constexpr setSlotsPerPage( - -
-
SizeType32 const &slotsPerPage,
-
- -) noexcept#
-
- -
-
-inline SizeType32 constexpr getPageWidth() const noexcept#
-
- -
-
-inline void constexpr setPageWidth( - -
-
SizeType32 const &pageWidth,
-
- -) noexcept#
-
- -
-
-inline bool constexpr getInitToZero() const noexcept#
-
- -
-
-inline void constexpr setInitToZero(bool initToZero) noexcept#
-
- -
-
-inline SizeType32 constexpr getNumCopyStreams() const noexcept#
-
- -
-
-inline void constexpr setNumCopyStreams( - -
-
SizeType32 numCopyStreams,
-
- -) noexcept#
-
- -
-
-

Private Members

-
-
-runtime::MemoryType mMemoryType#
+
+std::atomic<DiffType> mUVMDiff = {}#
-
-nvinfer1::DataType mDataType#
-
- -
-
-SizeType32 mTotalNumPages#
-
- -
-
-SizeType32 mMaxPagesPerBlock#
-
- -
-
-SizeType32 mSlotsPerPage#
-
- -
-
-SizeType32 mPageWidth#
-
- -
-
-SizeType32 mNumCopyStreams = 1#
-
- -
-
-bool mInitToZero#
+
+std::atomic<DiffType> mPinnedPoolDiff = {}#
@@ -12779,72 +12722,42 @@ one more than decoding draft tokens for prediction from primary head

@@ -14691,10 +14621,12 @@ one more than decoding draft tokens for prediction from primary head

+ +
@@ -14716,12 +14648,7 @@ one more than decoding draft tokens for prediction from primary head

- Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -14769,9 +14696,9 @@ one more than decoding draft tokens for prediction from primary head

-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_downloads/b509390ba70e52fabb10dbd9d15d5118/attention.py b/latest/_downloads/b509390ba70e52fabb10dbd9d15d5118/attention.py index c8686a55d6..7b5c5e429c 100644 --- a/latest/_downloads/b509390ba70e52fabb10dbd9d15d5118/attention.py +++ b/latest/_downloads/b509390ba70e52fabb10dbd9d15d5118/attention.py @@ -23,7 +23,7 @@ from ..distributed import AllReduceParams, alltoall_helix from ..model_config import ModelConfig from ..peft.lora.layer import LoraLayer, LoraModuleType from ..utils import (Fp4QuantizedTensor, get_model_extra_attrs, - is_piecewise_running, is_torch_compiling) + is_torch_compiling, maybe_compile) from .linear import Linear, TensorParallelMode, WeightMode, WeightsLoadingConfig from .multi_stream_utils import maybe_execute_in_parallel from .rms_norm import RMSNorm @@ -76,17 +76,6 @@ def extract_extra_attrs(layer_idx: str, attn_type: str): return metadata, attn_layer -def maybe_compile(func): - - def wrapper(*args, **kwargs): - if is_piecewise_running(): - # When piecewise running, we don't need to compile the function to avoid host overhead in attention op. - return func(*args, **kwargs) - return torch.compile(func)(*args, **kwargs) - - return wrapper - - @maybe_compile def maybe_compiled_copy_(dst, src): dst.copy_(src) @@ -413,9 +402,9 @@ class Attention(nn.Module): out_scale = None out_scale_sf = None - if self.has_quant_scale: + if self.has_quant_scale and not self.attn_output_gate: out_scale = self.o_proj.inv_input_scale - if self.o_proj.has_nvfp4 and self.support_nvfp4_output and enable_attn_nvfp4_output: + if self.o_proj.has_nvfp4 and self.support_nvfp4_output and enable_attn_nvfp4_output and not self.attn_output_gate: out_scale_sf = self.o_proj.input_scale kv_scales_sf = None @@ -751,7 +740,7 @@ class MLA(nn.Module): self.register_to_config = True # only support one kind of sparse attention, dsa now. - if config.sparse_attention_config is not None: + if config is not None and config.sparse_attention_config is not None: self.is_dsa = True else: self.is_dsa = False @@ -963,8 +952,6 @@ class MLA(nn.Module): if not config.skip_create_weights_in_init: self.create_weights() - self.indexer = self.mqa.indexer if self.is_dsa else None - def create_weights(self): # self.mha/mqa has no weights but has states that are related to quant_config, # which could be modified after __init__ @@ -1198,7 +1185,7 @@ class MLA(nn.Module): assert position_ids is not None k_pe_gen = self.apply_rope(q_gen, k_pe_gen, position_ids) - self.forward_absorption( + self.forward_absorption_generation( q_gen, compressed_kv_gen, k_pe_gen, @@ -1234,9 +1221,21 @@ class MLA(nn.Module): if position_ids is not None: position_ids = position_ids[..., :num_tokens] - q, compressed_kv, k_pe = self.kv_a_proj_with_mqa(hidden_states).split( - [self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], -1) + if self.fuse_a_indexer_k_weight: + q, compressed_kv, k_pe, indexer_k, indexer_weights = self.kv_a_proj_with_mqa( + hidden_states).split([ + self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim, + self.indexer.head_dim, self.indexer.n_heads + ], -1) + else: + q, compressed_kv, k_pe = self.kv_a_proj_with_mqa( + hidden_states).split([ + self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim + ], -1) + indexer_k = None + indexer_weights = None + # TODO: possibly overlap/fuse q_a_rmsnorm + kv_a_rmsnorm + indexer.k_layernorm? q, compressed_kv = maybe_execute_in_parallel( lambda: self.q_a_layernorm(q), lambda: self.kv_a_layernorm(compressed_kv), @@ -1245,12 +1244,18 @@ class MLA(nn.Module): self.aux_stream, ) qr = q - q, latent_cache = maybe_execute_in_parallel( - lambda: self.q_b_proj(q), - lambda: torch.concat([compressed_kv, k_pe], dim=-1), - self.ln_events[0], - self.ln_events[1], - self.aux_stream, + latent_cache = torch.concat([compressed_kv, k_pe], dim=-1) + + # TODO: fuse wq_b + (indexer) wlq here + q = self.q_b_proj(q) + # Indexer + topk_indices = self.indexer( + qr, + hidden_states, + attn_metadata, + position_ids, + indexer_k=indexer_k, # indexer K proj + indexer_weights=indexer_weights, # indexer weights proj ) assert q.shape[ @@ -1258,10 +1263,6 @@ class MLA(nn.Module): assert output is not None, "output must be provided" - # Indexer - topk_indices = self.indexer(qr, hidden_states, attn_metadata, - position_ids) - if num_contexts > 0: q_ctx = q[:num_ctx_tokens, ...] compressed_kv_ctx = compressed_kv[:num_ctx_tokens, ...] @@ -1355,14 +1356,13 @@ class MLA(nn.Module): topk_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: if get_sm_version() >= 100: - return self.forward_absorption(q, - compressed_kv, - k_pe, - attn_metadata, - output, - latent_cache=latent_cache, - topk_indices=topk_indices, - is_generation=False) + return self.forward_absorption_context(q, + compressed_kv, + k_pe, + attn_metadata, + output, + latent_cache=latent_cache, + topk_indices=topk_indices) else: return self.forward_sparse_mla_kvcache_bf16(q, latent_cache, @@ -1382,13 +1382,13 @@ class MLA(nn.Module): topk_indices: Optional[torch.Tensor] = None, ) -> torch.Tensor: if get_sm_version() >= 100: - return self.forward_absorption(q, - compressed_kv, - k_pe, - attn_metadata, - output, - latent_cache=latent_cache, - topk_indices=topk_indices) + return self.forward_absorption_generation(q, + compressed_kv, + k_pe, + attn_metadata, + output, + latent_cache=latent_cache, + topk_indices=topk_indices) else: return self.forward_sparse_mla_kvcache_bf16(q, latent_cache, @@ -1653,7 +1653,172 @@ class MLA(nn.Module): position_ids, attn_metadata, output, latent_cache) - def forward_absorption( + def forward_absorption_generation( + self, + q: torch.Tensor, + compressed_kv: torch.Tensor, + k_pe: torch.Tensor, + attn_metadata: AttentionMetadata, + output: torch.Tensor, + position_ids: Optional[torch.Tensor] = None, + latent_cache: Optional[torch.Tensor] = None, + topk_indices: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + num_tokens = q.shape[0] + q_nope, q_pe = q.view([-1, self.num_heads_tp, self.qk_head_dim]).split( + [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + + # fused_q contains 1) the result of the following bmm with shape [num_tokens, num_heads, kv_lora_rank] + # 2) rope(q_pe) with shape [num_tokens, num_heads, qk_rope_head_dim]. rope is applied inside AttentionOp + num_seqs = attn_metadata.kv_lens_cuda_runtime.size(0) + + cu_q_seqlens = torch.empty(num_seqs + 1, + dtype=torch.int32, + device=q.device) + cu_kv_seqlens = torch.empty(num_seqs + 1, + dtype=torch.int32, + device=q.device) + fmha_scheduler_counter = torch.empty(1, + dtype=torch.uint32, + device=q.device) + has_fp8_kv_cache = self.mqa.has_fp8_kv_cache if hasattr( + self.mqa, 'has_fp8_kv_cache') else False + + mla_bmm1_scale = None + mla_bmm2_scale = None + quant_q_buffer = None + if has_fp8_kv_cache: + mla_bmm1_scale = torch.empty(2, + dtype=torch.float32, + device=q.device) + mla_bmm2_scale = torch.empty(1, + dtype=torch.float32, + device=q.device) + quant_q_buffer = torch.empty( + num_tokens, + self.num_heads_tp, (self.kv_lora_rank + self.qk_rope_head_dim), + dtype=torch.uint8, + device=q.device) + + fused_q = torch.empty( + [ + num_tokens, self.num_heads_tp, + (self.kv_lora_rank + self.qk_rope_head_dim) + ], + dtype=q.dtype, + device=q.device, + ) + + rope_stream = self.aux_stream if not has_fp8_kv_cache else None + if self.k_b_proj_trans.dtype == torch.bfloat16: + # [num_heads, num_tokens, self.qk_nope_head_dim] + q_nope_t = q_nope.transpose(0, 1) + # [num_heads, num_tokens, self.kv_lora_rank] + q_nope_out = fused_q[..., :self.kv_lora_rank].transpose(0, 1) + + # [num_heads, num_tokens, self.qk_nope_head_dim] x [num_heads, kv_lora_rank, qk_nope_head_dim] + # -> [num_heads, num_tokens, kv_lora_rank] -> [num_tokens, num_heads, kv_lora_rank] + # The output of bmm is written directly into fused_q + maybe_execute_in_parallel( + lambda: torch.ops.trtllm.bmm_out( + q_nope_t, self.k_b_proj_trans.transpose(1, 2), q_nope_out), + lambda: self.mqa.mla_rope_generation( + fused_q, q_pe, latent_cache, attn_metadata, cu_q_seqlens, + cu_kv_seqlens, fmha_scheduler_counter, mla_bmm1_scale, + mla_bmm2_scale, quant_q_buffer), + self.ln_events[0], + self.ln_events[1], + rope_stream, + ) + + elif self.k_b_proj_trans.dtype == torch.float8_e4m3fn: + # [num_heads, num_tokens, self.kv_lora_rank] + q_nope_out = fused_q[..., :self.kv_lora_rank].transpose(0, 1) + + maybe_execute_in_parallel( + lambda: fp8_block_scaling_bmm_out( + q_nope, + self.k_b_proj_trans, + self.k_b_proj_trans_scale, + q_nope_out, + self.k_b_proj_trans_dequant, + ), + lambda: self.mqa.mla_rope_generation( + fused_q, q_pe, latent_cache, attn_metadata, cu_q_seqlens, + cu_kv_seqlens, fmha_scheduler_counter, mla_bmm1_scale, + mla_bmm2_scale, quant_q_buffer), + self.ln_events[0], + self.ln_events[1], + rope_stream, + ) + else: + raise NotImplementedError( + f"Missing bmm impl for dtype: {self.k_b_proj_trans.dtype}.") + + fused_q = fused_q.view([ + num_tokens, + self.num_heads_tp * (self.kv_lora_rank + self.qk_rope_head_dim) + ]) + + # Use generation_only for generation phase and context_only for context phase in DSA attention + attention_input_type = AttentionInputType.generation_only + + attn_out_latent = self._attn_forward_gen( + self.mqa, + fused_q, + None, + None, + position_ids, + attn_metadata, + attention_input_type=attention_input_type, + out_scale=self.out_scale, + latent_cache=latent_cache, # kvcache and k_pe + q_pe=q_pe, # used by `invokeMLARopeGeneration` + topk_indices=topk_indices, # used by DSA attention + is_generation=True, # used by DSA attention + cu_q_seqlens=cu_q_seqlens, # used by `mlaGeneration` + cu_kv_seqlens=cu_kv_seqlens, # used by `mlaGeneration` + fmha_scheduler_counter= + fmha_scheduler_counter, # used by `mlaGeneration` + mla_bmm1_scale=mla_bmm1_scale, # used by `mlaGeneration` + mla_bmm2_scale=mla_bmm2_scale, # used by `mlaGeneration` + quant_q_buffer=quant_q_buffer, # used by `mlaGeneration` + ) + fused_q = None + + # note: if we do not have CP, then num_heads_tp_cp == num_heads_tp + assert (attn_out_latent.shape[0] == q.shape[0] + and attn_out_latent.shape[1] + == self.num_heads_tp_cp * self.kv_lora_rank) + + # [seq, num_heads, kv_lora_rank] + attn_out_latent = attn_out_latent.view( + [-1, self.num_heads_tp_cp, self.kv_lora_rank]) + + attn_output = output.view( + [num_tokens, self.num_heads_tp_cp, self.v_head_dim]) + + if self.v_b_proj.dtype == torch.bfloat16: + # [num_heads, seq, kv_lora_rank] x [num_heads, kv_lora_rank, v_head_dim] + # -> [num_heads, seq, v_head_dim] + torch.ops.trtllm.bmm_out(attn_out_latent.transpose(0, 1), + self.v_b_proj.transpose(1, 2), + attn_output.transpose(0, 1)) + elif self.v_b_proj.dtype == torch.float8_e4m3fn: + fp8_block_scaling_bmm_out( + attn_out_latent, + self.v_b_proj, + self.v_b_proj_scale, + attn_output.transpose(0, 1), + self.v_b_proj_dequant, + ) + else: + raise NotImplementedError( + f"Missing bmm impl for dtype: {self.v_b_proj.dtype}.") + + return output + + def forward_absorption_context( self, q: torch.Tensor, compressed_kv: torch.Tensor, @@ -1663,7 +1828,6 @@ class MLA(nn.Module): position_ids: Optional[torch.Tensor] = None, latent_cache: Optional[torch.Tensor] = None, topk_indices: Optional[torch.Tensor] = None, - is_generation: bool = True, ) -> torch.Tensor: num_tokens = q.shape[0] q_nope, q_pe = q.view([-1, self.num_heads_tp, self.qk_head_dim]).split( @@ -1715,7 +1879,7 @@ class MLA(nn.Module): ]) # Use generation_only for generation phase and context_only for context phase in DSA attention - attention_input_type = AttentionInputType.generation_only if is_generation else AttentionInputType.context_only + attention_input_type = AttentionInputType.context_only attn_out_latent = self._attn_forward_gen( self.mqa, fused_q, @@ -1728,7 +1892,7 @@ class MLA(nn.Module): latent_cache=latent_cache, # kvcache and k_pe q_pe=q_pe, # used by `invokeMLARopeGeneration` topk_indices=topk_indices, # used by DSA attention - is_generation=is_generation, # used by DSA attention + is_generation=False, # used by DSA attention ) fused_q = None @@ -1794,10 +1958,10 @@ class MLA(nn.Module): q, latent_cache, attn_metadata, is_generation=is_generation) num_tokens = q.shape[0] - q_nope, q_rope = q.view(-1, self.num_heads, self.qk_head_dim).split( + q_nope, q_rope = q.view(-1, self.num_heads_tp, self.qk_head_dim).split( [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) q_nope_out = torch.empty( - [num_tokens, self.num_heads, (self.kv_lora_rank)], + [num_tokens, self.num_heads_tp, (self.kv_lora_rank)], dtype=q.dtype, device=q.device, ) @@ -1836,23 +2000,23 @@ class MLA(nn.Module): # FlashMLA sparse kernel (bf16) requires num_heads=128 on sm100 or multiple of 64 on sm90 if sm_version >= 100: padding = 128 - assert self.num_heads <= padding, ( + assert self.num_heads_tp <= padding, ( f"SM100 FlashMLA sparse kernel requires exactly {padding} heads, " - f"got {self.num_heads}. Padding from values > {padding} is not supported." + f"got {self.num_heads_tp}. Padding from values > {padding} is not supported." ) else: # SM90 - padding = ((self.num_heads + 63) // 64) * 64 # multiple of 64 + padding = ((self.num_heads_tp + 63) // 64) * 64 # multiple of 64 - if self.num_heads != padding: + if self.num_heads_tp != padding: logger.warning_once( - f"Padding num_heads from {self.num_heads} to {padding} " + f"Padding num_heads from {self.num_heads_tp} to {padding} " f"due to FlashMLA sparse attention kernel requirement", key="sparse_mla_padding_warning") # Create padded tensor with zeros for extra heads q_padded = q_concat.new_empty( (num_tokens, padding, q_concat.shape[2])) - q_padded[:, :self.num_heads, :] = q_concat + q_padded[:, :self.num_heads_tp, :] = q_concat q_concat = q_padded # Convert indices and return all-layer KV pool @@ -1874,17 +2038,17 @@ class MLA(nn.Module): "flash_mla_sparse_fwd not available. Please ensure FlashMLA module is built." ) - # [seq, num_heads, kv_lora_rank] - attn_out_latent = attn_out_latent[:, :self. - num_heads, :] # account for padding + # [seq, num_heads, kv_lora_rank], account for padding + attn_out_latent = attn_out_latent[:, :self.num_heads_tp, :] # TODO: seems we need .contiguous() here when padding enabled before pass to bmm? attn_out_latent = attn_out_latent.view( - [-1, self.num_heads, self.kv_lora_rank]) + [-1, self.num_heads_tp, self.kv_lora_rank]) assert (attn_out_latent.shape[0] == q.shape[0] - and attn_out_latent.shape[1] == self.num_heads) + and attn_out_latent.shape[1] == self.num_heads_tp) - attn_output = output.view([num_tokens, self.num_heads, self.v_head_dim]) + attn_output = output.view( + [num_tokens, self.num_heads_tp, self.v_head_dim]) if self.v_b_proj.dtype == torch.bfloat16: # [num_heads, seq, kv_lora_rank] x [num_heads, kv_lora_rank, v_head_dim] diff --git a/latest/_downloads/c68095123d889975e6e5e839a4241d22/model_engine.py b/latest/_downloads/c68095123d889975e6e5e839a4241d22/model_engine.py index e6da9fc216..8fb26a54f8 100644 --- a/latest/_downloads/c68095123d889975e6e5e839a4241d22/model_engine.py +++ b/latest/_downloads/c68095123d889975e6e5e839a4241d22/model_engine.py @@ -54,13 +54,12 @@ from ..speculative.utils import SpecDecodingTensor from ..utils import (get_model_extra_attrs, set_per_request_piecewise_cuda_graph_flag, set_torch_compiling, with_model_extra_attrs) -from .config import _construct_checkpoint_loader from .config_utils import is_mla -from .cuda_graph_runner import CUDAGraphRunner +from .cuda_graph_runner import CUDAGraphRunner, CUDAGraphRunnerConfig from .guided_decoder import CapturableGuidedDecoder from .layerwise_nvtx_marker import LayerwiseNvtxMarker -from .llm_request import get_draft_token_length -from .model_loader import ModelLoader +from .llm_request import LlmRequest, get_draft_token_length +from .model_loader import ModelLoader, _construct_checkpoint_loader from .resource_manager import (BaseResourceManager, KVCacheManager, ResourceManager, ResourceManagerType) from .sampler import SampleStateTensors @@ -74,14 +73,13 @@ class ModelEngine(ABC): raise NotImplementedError @abstractmethod - def forward( - self, - scheduled_requests: ScheduledRequests, - resource_manager: ResourceManager, - new_tensors_device: Optional[SampleStateTensors], - gather_context_logits: bool = False, - cache_indirection_buffer: Optional[torch.Tensor] = None, - ): + def forward(self, + scheduled_requests: ScheduledRequests, + resource_manager: ResourceManager, + new_tensors_device: Optional[SampleStateTensors], + gather_context_logits: bool = False, + cache_indirection_buffer: Optional[torch.Tensor] = None, + num_accepted_tokens_device: Optional[torch.Tensor] = None): raise NotImplementedError def warmup(self, resource_manager: ResourceManager) -> None: @@ -366,14 +364,60 @@ class PyTorchModelEngine(ModelEngine): if self.use_mrope: self.mrope_position_ids_cuda = torch.empty( (3, 1, self.max_num_tokens), dtype=torch.int, device='cuda') - self.iter_counter = 0 + + # Pre-allocated buffers for draft model to avoid implicit synchronization + # These are used to build index tensors without creating tensors from Python lists + if is_draft_model: + # Buffers for context and first_draft input_ids updates + self.draft_ctx_token_indices_cuda = torch.empty((self.batch_size, ), + dtype=torch.long, + device='cuda') + self.draft_ctx_seq_slots_cuda = torch.empty((self.batch_size, ), + dtype=torch.long, + device='cuda') + # Buffers for first_draft requests (max_draft_len+1 tokens per request) + max_first_draft_tokens = self.batch_size * ( + self.original_max_draft_len + + 1) if spec_config else self.batch_size + self.draft_first_draft_indices_cuda = torch.empty( + (max_first_draft_tokens, ), dtype=torch.long, device='cuda') + self.draft_first_draft_seq_slots_cuda = torch.empty( + (max_first_draft_tokens, ), dtype=torch.long, device='cuda') + # Buffers for seq_slots and request indices + self.draft_seq_slots_buffer_cuda = torch.empty((self.batch_size, ), + dtype=torch.int, + device='cuda') + self.draft_request_indices_buffer_cuda = torch.empty( + (self.batch_size, ), dtype=torch.int, device='cuda') # We look up this key in resource_manager during forward to find the # kv cache manager. Can be changed to support multiple model engines # with different KV cache managers. - self.kv_cache_manager_key = ResourceManagerType.KV_CACHE_MANAGER + self.kv_cache_manager_key = ResourceManagerType.DRAFT_KV_CACHE_MANAGER if is_draft_model else ResourceManagerType.KV_CACHE_MANAGER self.lora_model_config: Optional[LoraModelConfig] = None - self.cuda_graph_runner = CUDAGraphRunner(self) + + # Create config and runner + cuda_graph_runner_config = CUDAGraphRunnerConfig( + use_cuda_graph=self.cuda_graph_config is not None, + cuda_graph_padding_enabled=self._cuda_graph_padding_enabled, + cuda_graph_batch_sizes=self._cuda_graph_batch_sizes, + max_cuda_graph_batch_size=self._max_cuda_graph_batch_size, + max_beam_width=self.max_beam_width, + spec_config=self.spec_config, + cuda_graph_mem_pool=self._cuda_graph_mem_pool, + max_num_tokens=self.max_num_tokens, + use_mrope=self.use_mrope, + original_max_draft_len=self.original_max_draft_len, + original_max_total_draft_tokens=self. + original_max_total_draft_tokens, + is_draft_model=self.is_draft_model, + enable_attention_dp=self.enable_attention_dp, + batch_size=self.batch_size, + mapping=self.mapping, + dist=self.dist, + kv_cache_manager_key=self.kv_cache_manager_key, + ) + self.cuda_graph_runner = CUDAGraphRunner(cuda_graph_runner_config) # Setup the local cache indirection buffer only once and reuse it. # This way it can also be used for CUDA graphs. @@ -587,6 +631,13 @@ class PyTorchModelEngine(ModelEngine): with self._release_batch_context(warmup_request, resource_manager) as batch: if batch is not None: + # Reset the flag is_first_draft for the draft model. + # This is necessary for overlap scheduler. + spec_resource_manager = resource_manager.get_resource_manager( + ResourceManagerType.SPEC_RESOURCE_MANAGER) + if self.is_draft_model and isinstance( + spec_resource_manager, Eagle3ResourceManager): + spec_resource_manager.is_first_draft = True self.forward(batch, new_tensors_device=None, resource_manager=resource_manager) @@ -909,6 +960,18 @@ class PyTorchModelEngine(ModelEngine): self.attn_runtime_features.cache_reuse or self.attn_runtime_features.chunked_prefill) cache_indirection = self.cache_indirection_attention if self.attn_backend.Metadata is TrtllmAttentionMetadata else None + num_attention_heads = getattr(self.model.model_config.pretrained_config, + 'num_attention_heads', None) + if num_attention_heads is not None: + num_key_value_heads = getattr( + self.model.model_config.pretrained_config, + 'num_key_value_heads', None) + if num_key_value_heads is not None: + num_heads_per_kv = num_attention_heads // num_key_value_heads + else: + num_heads_per_kv = 1 + else: + num_heads_per_kv = 1 if kv_cache_manager is None: return self.attn_backend.Metadata( max_num_requests=self.batch_size, @@ -921,7 +984,8 @@ class PyTorchModelEngine(ModelEngine): enable_context_mla_with_cached_kv= enable_context_mla_with_cached_kv, cache_indirection=cache_indirection, - sparse_attention_config=self.sparse_attention_config) + sparse_attention_config=self.sparse_attention_config, + num_heads_per_kv=num_heads_per_kv) if self.attn_metadata is not None: # This assertion can be relaxed if needed: just create a new metadata @@ -939,7 +1003,9 @@ class PyTorchModelEngine(ModelEngine): enable_flash_mla=self.model.model_config.enable_flash_mla, enable_context_mla_with_cached_kv=enable_context_mla_with_cached_kv, cache_indirection=cache_indirection, - sparse_attention_config=self.sparse_attention_config) + sparse_attention_config=self.sparse_attention_config, + num_heads_per_kv=num_heads_per_kv, + ) return self.attn_metadata @@ -1224,10 +1290,13 @@ class PyTorchModelEngine(ModelEngine): attn_metadata: AttentionMetadata, spec_metadata: Optional[SpecMetadata] = None, new_tensors_device: Optional[SampleStateTensors] = None, - cache_indirection_buffer: Optional[torch.Tensor] = None): + cache_indirection_buffer: Optional[torch.Tensor] = None, + num_accepted_tokens_device: Optional[torch.Tensor] = None, + req_id_to_old_request: Optional[Dict[int, LlmRequest]] = None): """ Prepare inputs for Pytorch Model. """ + new_tokens_device, new_tokens_lens_device, next_draft_tokens_device = None, None, None if new_tensors_device is not None: # speculative decoding cases: [batch, 1 + draft_len], others: [batch] @@ -1258,6 +1327,19 @@ class PyTorchModelEngine(ModelEngine): mrope_position_ids = [] num_accepted_draft_tokens = [] # per request + # Variables for updating the inputs of draft model + # Base values for gather_ids computation + first_draft_base_gather_ids = [] + # seq_slots to index into num_accepted_tokens_device + first_draft_seq_slots = [] + # Indices in the num_accepted_draft_tokens list + first_draft_request_indices = [] + + # (start_idx, end_idx, seq_slot) for context requests + context_input_ids_positions = [] + # (start_idx, end_idx, seq_slot) for first_draft requests + first_draft_input_ids_positions = [] + for request in scheduled_requests.context_requests: request_ids.append(request.py_request_id) all_prompt_tokens = request.get_tokens(0) @@ -1267,7 +1349,20 @@ class PyTorchModelEngine(ModelEngine): prompt_tokens = all_prompt_tokens[begin_compute:end_compute] position_ids.extend( range(begin_compute, begin_compute + len(prompt_tokens))) - input_ids.extend(prompt_tokens) + + # Track position for updating the inputs of draft model + if self.is_draft_model and num_accepted_tokens_device is not None: + start_idx = len(input_ids) + input_ids.extend(prompt_tokens) + end_idx = len(input_ids) + slot_idx = req_id_to_old_request[ + request.py_request_id].py_seq_slot + context_input_ids_positions.append( + (start_idx, end_idx - 1, + slot_idx)) # end_idx-1 is the last token position + else: + input_ids.extend(prompt_tokens) + gather_ids.append(len(input_ids) - 1) sequence_lengths.append(len(prompt_tokens)) num_accepted_draft_tokens.append(len(prompt_tokens) - 1) @@ -1411,14 +1506,9 @@ class PyTorchModelEngine(ModelEngine): previous_batch_indices.append(previous_batch_idx) previous_pos_indices.extend([previous_batch_idx] * (1 + self.runtime_draft_len)) - if self.spec_config.spec_dec_mode.has_draft_model(): - # In the overlap scheduler workflow, if having draft model, we already updated the previous batch before launching the target model, - # so we only need to add the runtime_draft_len to the past_seen_token_num. - num_cached_tokens_per_seq.append(past_seen_token_num + - self.runtime_draft_len) - else: - num_cached_tokens_per_seq.append(past_seen_token_num + - self.runtime_draft_len + 1) + + num_cached_tokens_per_seq.append(past_seen_token_num + + self.runtime_draft_len + 1) request.cached_tokens = num_cached_tokens_per_seq[-1] if self.enable_spec_decode and spec_config.spec_dec_mode.extend_ctx( self.attn_backend): @@ -1436,13 +1526,39 @@ class PyTorchModelEngine(ModelEngine): prompt_tokens = all_prompt_tokens[begin_compute:end_compute] position_ids.extend( range(begin_compute, begin_compute + len(prompt_tokens))) - input_ids.extend(prompt_tokens) - gather_ids.append( - len(input_ids) - 1 - (self.original_max_draft_len - - request.py_num_accepted_draft_tokens)) + + # Track position for updating the inputs of draft model + if self.is_draft_model and num_accepted_tokens_device is not None: + start_idx = len(input_ids) + input_ids.extend(prompt_tokens) + end_idx = len(input_ids) + # For first_draft, we need to replace the last original_max_draft_len+1 tokens + slot_idx = req_id_to_old_request[ + request.py_request_id].py_seq_slot + first_draft_input_ids_positions.append( + (start_idx, end_idx, slot_idx)) + + # Store info for GPU computation of gather_ids and num_accepted_draft_tokens + base_gather_id = len( + input_ids) - 1 - self.original_max_draft_len + # Placeholder, will be corrected on GPU + gather_ids.append(base_gather_id) + first_draft_base_gather_ids.append(base_gather_id) + first_draft_seq_slots.append(slot_idx) + first_draft_request_indices.append( + len(num_accepted_draft_tokens)) + + # Placeholder, will be corrected on GPU + num_accepted_draft_tokens.append(0) + else: + input_ids.extend(prompt_tokens) + gather_ids.append( + len(input_ids) - 1 - (self.original_max_draft_len - + request.py_num_accepted_draft_tokens)) + num_accepted_draft_tokens.append( + request.py_num_accepted_draft_tokens) + sequence_lengths.append(1 + self.original_max_draft_len) - num_accepted_draft_tokens.append( - request.py_num_accepted_draft_tokens) prompt_lengths.append(request.py_prompt_len) past_seen_token_num = begin_compute num_cached_tokens_per_seq.append(past_seen_token_num) @@ -1462,7 +1578,17 @@ class PyTorchModelEngine(ModelEngine): # skip adding input_ids of CUDA graph dummy requests so that new_tokens_device # can be aligned to the correct positions. if not request.is_cuda_graph_dummy: - input_ids.append(request.get_last_tokens(beam)) + # Track position for GPU update (draft model only) + if self.is_draft_model and num_accepted_tokens_device is not None: + start_idx = len(input_ids) + input_ids.append(request.get_last_tokens(beam)) + end_idx = len(input_ids) + slot_idx = req_id_to_old_request[ + request.py_request_id].py_seq_slot + first_draft_input_ids_positions.append( + (start_idx, end_idx, slot_idx)) + else: + input_ids.append(request.get_last_tokens(beam)) past_seen_token_num = request.max_beam_num_tokens - 1 else: # the request has previous tensor @@ -1537,6 +1663,79 @@ class PyTorchModelEngine(ModelEngine): dtype=torch.int, pin_memory=True) self.input_ids_cuda[:num_tokens].copy_(input_ids, non_blocking=True) + + # Update input_ids_cuda with new tokens from new_tensors_device (draft model only) + if self.is_draft_model and num_accepted_tokens_device is not None: + # For context requests: replace the last token with new_tensors_device[0, seq_slot, 0] + if len(context_input_ids_positions) > 0: + # Build tensors on CPU first, then copy to GPU to avoid implicit sync + num_ctx_positions = len(context_input_ids_positions) + ctx_token_indices_cpu = torch.tensor([ + last_token_idx + for _, last_token_idx, _ in context_input_ids_positions + ], + dtype=torch.long, + pin_memory=True) + ctx_seq_slots_cpu = torch.tensor([ + seq_slot + for _, _, seq_slot in context_input_ids_positions + ], + dtype=torch.long, + pin_memory=True) + # Copy to pre-allocated GPU buffers + self.draft_ctx_token_indices_cuda[:num_ctx_positions].copy_( + ctx_token_indices_cpu, non_blocking=True) + self.draft_ctx_seq_slots_cuda[:num_ctx_positions].copy_( + ctx_seq_slots_cpu, non_blocking=True) + self.input_ids_cuda[ + self. + draft_ctx_token_indices_cuda[:num_ctx_positions]] = new_tensors_device.new_tokens[ + 0, + self.draft_ctx_seq_slots_cuda[:num_ctx_positions], + 0] + + # For first_draft requests: replace the last (original_max_draft_len+1) tokens + # with new_tensors_device[:, seq_slot, 0] + if len(first_draft_input_ids_positions) > 0: + # All first_draft requests have same token length (original_max_draft_len + 1) + # Build index tensors on CPU first, then copy to GPU to avoid implicit sync + num_requests = len(first_draft_input_ids_positions) + tokens_per_request = first_draft_input_ids_positions[0][ + 1] - first_draft_input_ids_positions[0][0] + + # Create flat index array for all tokens to update on CPU + all_indices = [] + all_seq_slots = [] + for start_idx, end_idx, seq_slot in first_draft_input_ids_positions: + all_indices.extend(range(start_idx, end_idx)) + all_seq_slots.extend([seq_slot] * (end_idx - start_idx)) + + # Create CPU tensors with pinned memory + total_tokens = len(all_indices) + idx_tensor_cpu = torch.tensor(all_indices, + dtype=torch.long, + pin_memory=True) + seq_slots_tensor_cpu = torch.tensor(all_seq_slots, + dtype=torch.long, + pin_memory=True) + + # Copy to pre-allocated GPU buffers + self.draft_first_draft_indices_cuda[:total_tokens].copy_( + idx_tensor_cpu, non_blocking=True) + self.draft_first_draft_seq_slots_cuda[:total_tokens].copy_( + seq_slots_tensor_cpu, non_blocking=True) + + # Create token position indices (repeating 0..tokens_per_request for each request) + token_positions = torch.arange( + tokens_per_request, dtype=torch.long, + device='cuda').repeat(num_requests) + + self.input_ids_cuda[ + self. + draft_first_draft_indices_cuda[:total_tokens]] = new_tensors_device.new_tokens[ + token_positions, self. + draft_first_draft_seq_slots_cuda[:total_tokens], 0] + if num_draft_tokens > 0: draft_tokens = torch.tensor(draft_tokens, dtype=torch.int, @@ -1550,6 +1749,33 @@ class PyTorchModelEngine(ModelEngine): self.num_accepted_draft_tokens_cuda[:len( num_accepted_draft_tokens)].copy_(num_accepted_draft_tokens, non_blocking=True) + + # Update num_accepted_draft_tokens_cuda for first_draft_requests directly from num_accepted_tokens_device (draft model only) + if self.is_draft_model and len(first_draft_seq_slots) > 0: + # Build tensors on CPU first, then copy to GPU to avoid implicit sync + num_first_draft = len(first_draft_seq_slots) + first_draft_seq_slots_cpu = torch.tensor(first_draft_seq_slots, + dtype=torch.int, + pin_memory=True) + first_draft_indices_cpu = torch.tensor( + first_draft_request_indices, + dtype=torch.int, + pin_memory=True) + + # Copy to pre-allocated GPU buffers + self.draft_seq_slots_buffer_cuda[:num_first_draft].copy_( + first_draft_seq_slots_cpu, non_blocking=True) + self.draft_request_indices_buffer_cuda[:num_first_draft].copy_( + first_draft_indices_cpu, non_blocking=True) + + # Extract accepted tokens for first_draft requests from device tensor + accepted_tokens = num_accepted_tokens_device[ + self.draft_seq_slots_buffer_cuda[:num_first_draft]] + # Update the correct positions in num_accepted_draft_tokens_cuda + self.num_accepted_draft_tokens_cuda[ + self. + draft_request_indices_buffer_cuda[: + num_first_draft]] = accepted_tokens if next_draft_tokens_device is not None: # Initialize these two values to zeros self.previous_pos_id_offsets_cuda *= 0 @@ -1655,6 +1881,34 @@ class PyTorchModelEngine(ModelEngine): gather_ids, dtype=torch.int, pin_memory=True), non_blocking=True) + # Update gather_ids for first_draft_requests on GPU (draft model only) + if self.is_draft_model and len(first_draft_seq_slots) > 0: + # Build tensors on CPU first, then copy to GPU to avoid implicit sync + num_first_draft = len(first_draft_seq_slots) + first_draft_seq_slots_cpu = torch.tensor(first_draft_seq_slots, + dtype=torch.int, + pin_memory=True) + first_draft_indices_cpu = torch.tensor( + first_draft_request_indices, + dtype=torch.int, + pin_memory=True) + + # Copy to pre-allocated GPU buffers + self.draft_seq_slots_buffer_cuda[:num_first_draft].copy_( + first_draft_seq_slots_cpu, non_blocking=True) + self.draft_request_indices_buffer_cuda[:num_first_draft].copy_( + first_draft_indices_cpu, non_blocking=True) + + # Extract accepted tokens for first_draft requests from device tensor + accepted_tokens = num_accepted_tokens_device[ + self.draft_seq_slots_buffer_cuda[:num_first_draft]] + # Update gather_ids: gather_id = base_gather_id + num_accepted_tokens + # (since gather_id = len(input_ids) - 1 - (max_draft_len - num_accepted)) + self.gather_ids_cuda[ + self. + draft_request_indices_buffer_cuda[: + num_first_draft]] += accepted_tokens + if not attn_metadata.is_cuda_graph: # Assumes seq lens do not change between CUDA graph invocations. This applies # to draft sequences too. This means that all draft sequences must be padded. @@ -1671,8 +1925,11 @@ class PyTorchModelEngine(ModelEngine): is_cuda_graph_during_warmup = self.is_warmup and attn_metadata.is_cuda_graph if cache_indirection_buffer is not None: #Copy cache indirection to local buffer with offsets changing: seq_slots[i] -> i + # Convert to GPU tensor to avoid implicit sync + gen_request_seq_slots_tensor = torch.tensor( + gen_request_seq_slots, dtype=torch.long, device='cuda') self.cache_indirection_attention[:num_generation_requests].copy_( - cache_indirection_buffer[gen_request_seq_slots]) + cache_indirection_buffer[gen_request_seq_slots_tensor]) if cache_indirection_buffer is not None or is_cuda_graph_during_warmup: attn_metadata.beam_width = self.max_beam_width else: @@ -2253,7 +2510,9 @@ class PyTorchModelEngine(ModelEngine): attn_metadata: AttentionMetadata, spec_metadata: Optional[SpecMetadata] = None, new_tensors_device: Optional[SampleStateTensors] = None, - cache_indirection_buffer: Optional[torch.Tensor] = None): + cache_indirection_buffer: Optional[torch.Tensor] = None, + num_accepted_tokens_device: Optional[torch.Tensor] = None, + req_id_to_old_request: Optional[Dict[int, LlmRequest]] = None): if self.mapping is not None and 'cp_type' in self.mapping.cp_config: cp_type = self.mapping.cp_config['cp_type'] if CpType.STAR == cp_type: @@ -2269,19 +2528,21 @@ class PyTorchModelEngine(ModelEngine): return self._prepare_tp_inputs(scheduled_requests, kv_cache_manager, attn_metadata, spec_metadata, new_tensors_device, - cache_indirection_buffer) + cache_indirection_buffer, + num_accepted_tokens_device, + req_id_to_old_request) @torch.inference_mode() @with_model_extra_attrs(lambda self: self.model.extra_attrs) - def forward( - self, - scheduled_requests: ScheduledRequests, - resource_manager: ResourceManager, - new_tensors_device: Optional[SampleStateTensors] = None, - gather_context_logits: bool = False, - cache_indirection_buffer: Optional[torch.Tensor] = None, - spec_decoding_tensor: Optional[SpecDecodingTensor] = None, - ): + def forward(self, + scheduled_requests: ScheduledRequests, + resource_manager: ResourceManager, + new_tensors_device: Optional[SampleStateTensors] = None, + gather_context_logits: bool = False, + cache_indirection_buffer: Optional[torch.Tensor] = None, + spec_decoding_tensor: Optional[SpecDecodingTensor] = None, + num_accepted_tokens_device: Optional[torch.Tensor] = None, + req_id_to_old_request: Optional[Dict[int, LlmRequest]] = None): kv_cache_manager = resource_manager.get_resource_manager( self.kv_cache_manager_key) @@ -2320,11 +2581,20 @@ class PyTorchModelEngine(ModelEngine): return self._forward_step(inputs, gather_ids, gather_context_logits) with self.cuda_graph_runner.pad_batch( - scheduled_requests, resource_manager) as padded_requests: + scheduled_requests, resource_manager, + self.runtime_draft_len) as padded_requests: - maybe_graph, maybe_attn_metadata, maybe_spec_metadata, key = self.cuda_graph_runner.maybe_get_cuda_graph( - padded_requests, spec_resource_manager) - if maybe_graph: + maybe_attn_metadata, maybe_spec_metadata, key = self.cuda_graph_runner.maybe_get_cuda_graph( + padded_requests, + enable_spec_decode=self.enable_spec_decode, + attn_metadata=attn_metadata, + spec_metadata=spec_metadata, + draft_tokens_cuda=self.draft_tokens_cuda + if self.is_spec_decode else None, + spec_resource_manager=spec_resource_manager, + ) + can_run_graph = key is not None + if can_run_graph: attn_metadata = maybe_attn_metadata spec_metadata = maybe_spec_metadata else: @@ -2336,11 +2606,11 @@ class PyTorchModelEngine(ModelEngine): inputs, gather_ids = self._prepare_inputs( padded_requests, kv_cache_manager, attn_metadata, spec_metadata, - new_tensors_device, cache_indirection_buffer) + new_tensors_device, cache_indirection_buffer, + num_accepted_tokens_device, req_id_to_old_request) - self.iter_counter += 1 with with_shared_pool(self.cuda_graph_runner.get_graph_pool()): - if not maybe_graph: + if not can_run_graph: # Fallback to eager execution if graph was not used with MoeLoadBalancerIterContext(moe_load_balancer): outputs = self._forward_step(inputs, gather_ids, @@ -2358,9 +2628,12 @@ class PyTorchModelEngine(ModelEngine): def capture_postprocess_fn(inputs: Dict[str, Any]): self._postprocess_inputs(inputs) - self.cuda_graph_runner.capture(key, capture_forward_fn, - inputs, - capture_postprocess_fn) + self.cuda_graph_runner.capture( + key, + capture_forward_fn, + inputs, + enable_spec_decode=self.enable_spec_decode, + postprocess_fn=capture_postprocess_fn) # here we don't need to use context since cuda graph capture didn't run kernel. # maybe we need a cleaner way to do this. diff --git a/latest/_modules/index.html b/latest/_modules/index.html index ef83e9ecea..e5d9ede0b8 100644 --- a/latest/_modules/index.html +++ b/latest/_modules/index.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -606,10 +614,12 @@
+ +
@@ -631,12 +641,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -684,9 +689,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/builder.html b/latest/_modules/tensorrt_llm/builder.html index 0e3fb6f06e..3dab9a8b7e 100644 --- a/latest/_modules/tensorrt_llm/builder.html +++ b/latest/_modules/tensorrt_llm/builder.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1834,10 +1842,12 @@
+ +
@@ -1859,12 +1869,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1912,9 +1917,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/disaggregated_params.html b/latest/_modules/tensorrt_llm/disaggregated_params.html index ba18fb71a1..3791254c0b 100644 --- a/latest/_modules/tensorrt_llm/disaggregated_params.html +++ b/latest/_modules/tensorrt_llm/disaggregated_params.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -637,10 +645,12 @@
+ +
@@ -662,12 +672,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -715,9 +720,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/executor/request.html b/latest/_modules/tensorrt_llm/executor/request.html index 5afab550dd..b7287c6feb 100644 --- a/latest/_modules/tensorrt_llm/executor/request.html +++ b/latest/_modules/tensorrt_llm/executor/request.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -682,10 +690,12 @@
+ +
@@ -707,12 +717,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -760,9 +765,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/executor/result.html b/latest/_modules/tensorrt_llm/executor/result.html index 99dd6c4d20..30c151b0c2 100644 --- a/latest/_modules/tensorrt_llm/executor/result.html +++ b/latest/_modules/tensorrt_llm/executor/result.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -525,7 +533,7 @@ from tensorrt_llm import ray_stub as ray from .._ray_utils import unwrap_ray_errors -from .._utils import mpi_disabled, nvtx_range_debug +from .._utils import mpi_disabled, nvtx_range_debug, ray_use_rpc from ..bindings import executor as tllm from ..disaggregated_params import DisaggregatedParams from ..llmapi.tracer import global_tracer @@ -780,8 +788,10 @@ self._done = False self.metrics_dict = {} self.trace_headers: Optional[dict[str, str]] = None + # torch backend will use trtllm sampler in beam search mode, but it does not support return logprobs incrementally + self.use_trtllm_sampler = sampling_params.use_beam_search and sampling_params.best_of > 1 - if ray_queue is not None: + if ray_queue is not None and not ray_use_rpc(): if has_event_loop(): self.aqueue = ray_queue self.queue = self.aqueue @@ -886,20 +896,27 @@ # each streamed response_tensors.log_probs[src_idx] # contains a streamwise monotonically growing list of logprobs. # so we need to accumulate only the new ones unique to that particular streamed response - assert output._last_logprobs_len <= len( - response_tensors.log_probs[src_idx] - ), (f"_last_logprobs_len ({output._last_logprobs_len}) > log_probs length (" - f"{len(response_tensors.log_probs[src_idx])})") - output.logprobs += response_tensors.log_probs[src_idx][ - output._last_logprobs_len:] + if self.use_trtllm_sampler: + assert output._last_logprobs_len <= len( + response_tensors.log_probs[src_idx] + ), (f"_last_logprobs_len ({output._last_logprobs_len}) > log_probs length (" + f"{len(response_tensors.log_probs[src_idx])})") + output.logprobs += response_tensors.log_probs[src_idx][ + output._last_logprobs_len:] + else: + output.logprobs += response_tensors.log_probs[src_idx] + # overcome some WAR in the cpp executor - if finish_reasons[src_idx] != tllm.FinishReason.CANCELLED: + if finish_reasons[ + src_idx] != tllm.FinishReason.CANCELLED and self.use_trtllm_sampler: # Check if logprobs is a list (not a dict or other structure) if len(output.logprobs) > output.length: # LlmResult holds a reference to LogProbStorage, which may be updated by the worker before the result is serialized. # Therefore, we treat extra logprobs/logits as expected and only consume what's needed. output.logprobs = output.logprobs[:output.length] - assert len(output.logprobs) == output.length + assert len( + output.logprobs + ) == output.length, f"logprobs length: {len(output.logprobs)} != output.length: {output.length}" if response_tensors.generation_logits is not None: output.generation_logits = response_tensors.generation_logits[ @@ -1056,7 +1073,7 @@ else: raise ValueError(f"Unknown response type: {response}") - if self._done and mpi_disabled(): + if self._done and mpi_disabled() and not ray_use_rpc(): assert hasattr( self.queue, "unregister" ), "Ray path should be activated for unregistering the Ray queue." @@ -1289,7 +1306,7 @@ ) -> None: use_async_queue = has_event_loop() shared_queue = None - if executor and executor.use_ray_queue(): + if executor and executor.use_ray_queue() and not ray_use_rpc(): shared_queue = executor.async_response_queue_weakref if use_async_queue else executor.sync_response_queue_weakref super().__init__( @@ -1354,7 +1371,7 @@ return response def _result_step(self, timeout: Optional[float] = None): - if mpi_disabled(): + if mpi_disabled() and not ray_use_rpc(): with unwrap_ray_errors(): response = ray.get(self.queue.get.remote(self.request_id)) response = self._handle_ray_response(response) @@ -1365,7 +1382,7 @@ async def _aresult_step(self): assert self.aqueue is not None, "The asyncio event loop was not present during initialization, so async operations are not available." - if mpi_disabled(): + if mpi_disabled() and not ray_use_rpc(): response = await self.aqueue.get_async.remote(self.request_id) response = self._handle_ray_response(response) else: @@ -1625,10 +1642,12 @@
+ +
@@ -1650,12 +1669,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1703,9 +1717,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/executor/utils.html b/latest/_modules/tensorrt_llm/executor/utils.html index 35ebf5c3c8..40e57b01ee 100644 --- a/latest/_modules/tensorrt_llm/executor/utils.html +++ b/latest/_modules/tensorrt_llm/executor/utils.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -713,10 +721,12 @@
+ +
@@ -738,12 +748,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -791,9 +796,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/functional.html b/latest/_modules/tensorrt_llm/functional.html index e0fd8e31cd..fad45ea00c 100644 --- a/latest/_modules/tensorrt_llm/functional.html +++ b/latest/_modules/tensorrt_llm/functional.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -8692,10 +8700,12 @@
+ +
@@ -8717,12 +8727,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -8770,9 +8775,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/activation.html b/latest/_modules/tensorrt_llm/layers/activation.html index 5f646200ec..0a4e861a18 100644 --- a/latest/_modules/tensorrt_llm/layers/activation.html +++ b/latest/_modules/tensorrt_llm/layers/activation.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -564,10 +572,12 @@
+ +
@@ -589,12 +599,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -642,9 +647,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/attention.html b/latest/_modules/tensorrt_llm/layers/attention.html index 589d55d12e..020671783a 100644 --- a/latest/_modules/tensorrt_llm/layers/attention.html +++ b/latest/_modules/tensorrt_llm/layers/attention.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -3427,10 +3435,12 @@
+ +
@@ -3452,12 +3462,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -3505,9 +3510,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/cast.html b/latest/_modules/tensorrt_llm/layers/cast.html index 877c71f0db..198b24d8c4 100644 --- a/latest/_modules/tensorrt_llm/layers/cast.html +++ b/latest/_modules/tensorrt_llm/layers/cast.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -571,10 +579,12 @@
+ +
@@ -596,12 +606,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -649,9 +654,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/conv.html b/latest/_modules/tensorrt_llm/layers/conv.html index 48b2ee48f2..7772b6e206 100644 --- a/latest/_modules/tensorrt_llm/layers/conv.html +++ b/latest/_modules/tensorrt_llm/layers/conv.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -820,10 +828,12 @@
+ +
@@ -845,12 +855,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -898,9 +903,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/embedding.html b/latest/_modules/tensorrt_llm/layers/embedding.html index 10a52e8853..fa95861cd6 100644 --- a/latest/_modules/tensorrt_llm/layers/embedding.html +++ b/latest/_modules/tensorrt_llm/layers/embedding.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1287,10 +1295,12 @@
+ +
@@ -1312,12 +1322,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1365,9 +1370,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/linear.html b/latest/_modules/tensorrt_llm/layers/linear.html index df885000cf..0f78566587 100644 --- a/latest/_modules/tensorrt_llm/layers/linear.html +++ b/latest/_modules/tensorrt_llm/layers/linear.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1135,10 +1143,12 @@
+ +
@@ -1160,12 +1170,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1213,9 +1218,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/mlp.html b/latest/_modules/tensorrt_llm/layers/mlp.html index 62b926efe1..507364fc2e 100644 --- a/latest/_modules/tensorrt_llm/layers/mlp.html +++ b/latest/_modules/tensorrt_llm/layers/mlp.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1161,10 +1169,12 @@
+ +
@@ -1186,12 +1196,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1239,9 +1244,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/normalization.html b/latest/_modules/tensorrt_llm/layers/normalization.html index b0ddc32761..29a59e2669 100644 --- a/latest/_modules/tensorrt_llm/layers/normalization.html +++ b/latest/_modules/tensorrt_llm/layers/normalization.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -925,10 +933,12 @@
+ +
@@ -950,12 +960,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1003,9 +1008,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/layers/pooling.html b/latest/_modules/tensorrt_llm/layers/pooling.html index ae9be7ced1..226c180fa8 100644 --- a/latest/_modules/tensorrt_llm/layers/pooling.html +++ b/latest/_modules/tensorrt_llm/layers/pooling.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -580,10 +588,12 @@
+ +
@@ -605,12 +615,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -658,9 +663,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/llmapi/build_cache.html b/latest/_modules/tensorrt_llm/llmapi/build_cache.html index 1bd66051a2..c0702a90a3 100644 --- a/latest/_modules/tensorrt_llm/llmapi/build_cache.html +++ b/latest/_modules/tensorrt_llm/llmapi/build_cache.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -863,10 +871,12 @@
+ +
@@ -888,12 +898,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -941,9 +946,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/llmapi/llm.html b/latest/_modules/tensorrt_llm/llmapi/llm.html index df5cb76730..3f3a492c03 100644 --- a/latest/_modules/tensorrt_llm/llmapi/llm.html +++ b/latest/_modules/tensorrt_llm/llmapi/llm.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1675,10 +1683,12 @@
+ +
@@ -1700,12 +1710,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1753,9 +1758,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/llmapi/llm_args.html b/latest/_modules/tensorrt_llm/llmapi/llm_args.html index 485f81c9a4..4449c2cc8d 100644 --- a/latest/_modules/tensorrt_llm/llmapi/llm_args.html +++ b/latest/_modules/tensorrt_llm/llmapi/llm_args.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -513,9 +521,8 @@ from dataclasses import dataclass from enum import Enum, EnumMeta from pathlib import Path -from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Literal, Optional, - Set, Tuple, Type, TypeAlias, TypeVar, Union, get_args, - get_origin) +from typing import (Any, ClassVar, Dict, List, Literal, Optional, Set, Tuple, + Type, TypeAlias, TypeVar, Union, get_args, get_origin) import torch import yaml @@ -530,9 +537,6 @@ from .._utils import mpi_rank -if TYPE_CHECKING: - from tensorrt_llm._torch.pyexecutor.config import PyTorchConfig - # yapf: disable # isort: off from ..bindings.executor import (BatchingType as _BatchingType, @@ -730,6 +734,9 @@ """ return True + def get_indices_block_size(self) -> int: + return 1 +
[docs] @@ -759,6 +766,12 @@ [docs] def supports_backend(self, backend: str) -> bool: return backend == "pytorch"
+ + +
+[docs] + def get_indices_block_size(self) -> int: + return self.page_size
@@ -794,6 +807,109 @@ +class MoeLoadBalancerConfig(StrictBaseModel): + """ + Pydantic configuration model for the Mixture of Experts (MoE) load balancer. + + This model holds configuration data (`num_slots`, etc.) as well as + runtime state (`_ep_rank`, `_ep_size`) which must be set via the + `setup()` method before use. + """ + + num_slots: Optional[int] = None + initial_global_assignments: Optional[Dict[int, List[int]]] = Field( + default=None, + repr=False # Exclude this large dict from model representation + ) + layer_updates_per_iter: int = 0 + _ep_rank: Optional[int] = PrivateAttr(default=None) + _ep_size: Optional[int] = PrivateAttr(default=None) + + # --- Methods --- + + def setup(self, ep_rank: int, ep_size: int) -> None: + """ + Initializes the runtime state of the configuration. + This must be called before accessing properties like `num_local_slots`. + """ + self._ep_rank = ep_rank + self._ep_size = ep_size + + # This assertion was in the original and is critical. + if self.num_slots is None: + raise ValueError("`num_slots` cannot be None when calling setup().") + + if self.num_slots % ep_size != 0: + raise ValueError( + f"`num_slots` ({self.num_slots}) must be divisible by `ep_size` ({ep_size})." + ) + + # --- Computed Properties --- + # These properties depend on the runtime state set by setup() + + @property + def ep_rank(self) -> int: + """Public accessor for the private expert parallel rank.""" + if self._ep_rank is None: + raise AttributeError("ep_rank is not set. Call setup() first.") + return self._ep_rank + + @property + def ep_size(self) -> int: + """Public accessor for the private expert parallel size.""" + if self._ep_size is None: + raise AttributeError("ep_size is not set. Call setup() first.") + return self._ep_size + + @property + def num_local_slots(self) -> int: + """Calculates the number of slots local to this rank.""" + if self.num_slots is None or self._ep_size is None: + raise ValueError( + "Cannot calculate `num_local_slots`. " + "`num_slots` must be set and setup() must be called.") + return self.num_slots // self._ep_size + + @property + def slot_start(self) -> int: + """Calculates the starting global slot index for this rank.""" + if self._ep_rank is None: + raise ValueError( + "Cannot calculate `slot_start`. Call setup() first.") + return self._ep_rank * self.num_local_slots + + @property + def slot_end(self) -> int: + """Calculates the ending global slot index (exclusive) for this rank.""" + return self.slot_start + self.num_local_slots + + def get_layer_initial_global_assignments( + self, layer_idx: int) -> Optional[List[int]]: + """ + Retrieves the initial global assignments for a specific layer. + """ + if self.initial_global_assignments is None: + return None + + if layer_idx not in self.initial_global_assignments: + raise KeyError( + f"layer_idx {layer_idx} not found in `initial_global_assignments`." + ) + + assignments = self.initial_global_assignments[layer_idx] + + if self.num_slots is None: + raise ValueError( + "`num_slots` is not set, cannot verify assignment length.") + + if len(assignments) != self.num_slots: + raise ValueError( + f"Assignment length ({len(assignments)}) for layer {layer_idx} " + f"does not match `num_slots` ({self.num_slots}).") + + return assignments + +
[docs] class MoeConfig(StrictBaseModel): @@ -871,6 +987,7 @@ moe_tp_size: int = -1 moe_ep_size: int = -1 cp_config: dict = Field(default_factory=dict) + pp_partition: Optional[List[int]] = Field(default=None) enable_attention_dp: bool = False enable_lm_head_tp_in_adp: bool = False @@ -917,6 +1034,7 @@ gpus_per_node=self.gpus_per_node, tp_size=self.tp_size, pp_size=self.pp_size, + pp_partition=self.pp_partition, cp_size=self.cp_size, cp_config=self.cp_config, enable_attention_dp=self.enable_attention_dp, @@ -1003,6 +1121,16 @@ # this value. Otherwise, speculation will always be on. max_concurrency: Optional[int] = None + # Developer interface: dynamically adjust draft length based on active batch size in runtime. + # Maps batch size to draft lengths. For example: + # {1: 4, 4: 2, 8: 0} means: + # - batch_size >= 1: use draft_len=4 + # - batch_size >= 4: use draft_len=2 + # - batch_size >= 8: use draft_len=0 (disable speculation) + # draft_len_schedule is enforced to contain batch_size=1 and its according draft_len equals max_draft_len for consistency + # for example, if max_draft_len=4, the schedule must contain {1: 4} + draft_len_schedule: Optional[dict[int, int]] = None + load_format: Optional[str] = None # PyTorch only. # Rolling average window size (N) for acceptance length across completed requests. @@ -1040,6 +1168,51 @@ # If set, drafting uses greedy sampling, irrespective of sampling parameters. _allow_greedy_draft_tokens: bool = PrivateAttr(True) + @field_validator('draft_len_schedule') + @classmethod + def validate_draft_len_schedule_and_sort(cls, v, info): + """Validate and sort draft_len_schedule by batch size thresholds.""" + if v is not None: + # Validate values + for batch_size, draft_len in v.items(): + if batch_size < 1: + raise ValueError( + f"draft_len_schedule: batch size threshold must be >= 1, got {batch_size}" + ) + if draft_len < 0: + raise ValueError( + f"draft_len_schedule: draft length must be >= 0, got {draft_len}" + ) + + # Require batch_size=1 in schedule + if 1 not in v: + raise ValueError( + "draft_len_schedule must include batch_size=1. " + "All systems can have batch_size=1. Add {1: <max_draft_len>} to your schedule." + ) + + # Enforce schedule[1] == max_draft_len for consistency + max_draft_len = info.data.get('max_draft_len') + if max_draft_len is not None and v[1] != max_draft_len: + raise ValueError( + f"draft_len_schedule[1] must equal max_draft_len for consistency. " + f"Got schedule[1]={v[1]}, but max_draft_len={max_draft_len}. " + f"batch_size=1 should use maximum draft length.") + + # Enforce all draft lengths <= max_draft_len + if max_draft_len is not None: + for batch_size, draft_len in v.items(): + if draft_len > max_draft_len: + raise ValueError( + f"draft_len_schedule: all draft lengths must be <= max_draft_len. " + f"Got draft_len={draft_len} for batch_size={batch_size}, " + f"but max_draft_len={max_draft_len}.") + + # Return sorted dict (by batch size thresholds) + # This ensures efficient lookup + return dict(sorted(v.items(), key=lambda x: x[0])) + return v + @classmethod def from_dict(cls, data: dict): # dispatch to the correct decoding config @@ -1445,12 +1618,11 @@ # Now we need a flag when MTPDecodingConfig is updated by PyTorchModelEngine. num_nextn_predict_layers_from_model_config: int = 1 - # TODO: Hard code for DeepSeek R1 # When encounter <think>, start thinking phase. # When encounter </think>, end thinking phase. # <think> [thinking phase] </think> [real output] - BEGIN_THINKING_PHASE_TOKEN: int = 128798 - END_THINKING_PHASE_TOKEN: int = 128799 + begin_thinking_phase_token: int = 128798 + end_thinking_phase_token: int = 128799
[docs] @@ -2158,11 +2330,20 @@ "Timeout in milliseconds for KV cache transfer. Requests exceeding this timeout will be cancelled." ) + kv_transfer_sender_future_timeout_ms: Optional[int] = Field( + default=1000, + gt=0, + description= + "Timeout in milliseconds to wait for the sender future to be ready when scheduled batch size is 0. This allows the request to be eventually cancelled by the user or because of kv_transfer_timeout_ms" + ) + def _to_pybind(self): return _CacheTransceiverConfig( backend=_CacheTransceiverBackendType.from_string(self.backend), max_tokens_in_buffer=self.max_tokens_in_buffer, - kv_transfer_timeout_ms=self.kv_transfer_timeout_ms)
+ kv_transfer_timeout_ms=self.kv_transfer_timeout_ms, + kv_transfer_sender_future_timeout_ms=self. + kv_transfer_sender_future_timeout_ms)
@@ -2288,6 +2469,12 @@ description="Enable LM head TP in attention dp.", status="prototype") + pp_partition: Optional[List[int]] = Field( + default=None, + description= + "Pipeline parallel partition, a list of each rank's layer number.", + status="prototype") + cp_config: Optional[dict] = Field(default_factory=dict, description="Context parallel config.", status="prototype") @@ -2544,6 +2731,7 @@ moe_ep_size=self.moe_expert_parallel_size, enable_attention_dp=self.enable_attention_dp, enable_lm_head_tp_in_adp=self.enable_lm_head_tp_in_adp, + pp_partition=self.pp_partition, cp_config=self.cp_config) return self @@ -3323,6 +3511,9 @@ # PrivateVars _quant_config: Optional[QuantConfig] = PrivateAttr(default=None) + _disable_flash_infer_sampling: bool = PrivateAttr(default=True) + """Unless this is set to False, FlashInfer.sampling is not used, even if available.""" + @property def quant_config(self) -> QuantConfig: if self._quant_config is None: @@ -3405,7 +3596,6 @@ [docs] @model_validator(mode="after") def validate_load_balancer(self) -> 'TorchLlmArgs': - from .._torch import MoeLoadBalancerConfig if isinstance(self.moe_config.load_balancer, str): if not os.path.exists(self.moe_config.load_balancer): raise FileNotFoundError( @@ -3607,82 +3797,6 @@ executor_config = super().get_executor_config(_hf_model_dir, tokenizer) executor_config.mm_encoder_only = self.mm_encoder_only return executor_config - - - # TODO: Remove this after the PyTorch backend is fully migrated to TorchLlmArgs from ExecutorConfig -
-[docs] - def get_pytorch_backend_config(self) -> "PyTorchConfig": - from tensorrt_llm._torch.pyexecutor.config import PyTorchConfig - - return PyTorchConfig( - extra_resource_managers=self.extra_resource_managers, - use_cuda_graph=bool(self.cuda_graph_config is not None), - cuda_graph_batch_sizes=self.cuda_graph_config.batch_sizes - if self.cuda_graph_config else - CudaGraphConfig.model_fields['batch_sizes'].default, - cuda_graph_max_batch_size=self.cuda_graph_config.max_batch_size - if self.cuda_graph_config else - CudaGraphConfig.model_fields['max_batch_size'].default, - cuda_graph_padding_enabled=self.cuda_graph_config.enable_padding - if self.cuda_graph_config else - CudaGraphConfig.model_fields['enable_padding'].default, - disable_overlap_scheduler=self.disable_overlap_scheduler, - moe_max_num_tokens=self.moe_config.max_num_tokens, - moe_load_balancer=self.moe_config.load_balancer, - attn_backend=self.attn_backend, - moe_backend=self.moe_config.backend, - use_low_precision_moe_combine=self.moe_config. - use_low_precision_moe_combine, - sampler_type=self.sampler_type, - kv_cache_dtype=self.kv_cache_config.dtype, - mamba_ssm_cache_dtype=self.kv_cache_config.mamba_ssm_cache_dtype, - enable_iter_perf_stats=self.enable_iter_perf_stats, - enable_iter_req_stats=self.enable_iter_req_stats, - print_iter_log=self.print_iter_log, - torch_compile_enabled=bool(self.torch_compile_config is not None), - torch_compile_fullgraph=self.torch_compile_config.enable_fullgraph - if self.torch_compile_config is not None else - TorchCompileConfig.model_fields['enable_fullgraph'].default, - torch_compile_inductor_enabled=self.torch_compile_config. - enable_inductor if self.torch_compile_config is not None else - TorchCompileConfig.model_fields['enable_inductor'].default, - torch_compile_piecewise_cuda_graph=self.torch_compile_config. - enable_piecewise_cuda_graph - if self.torch_compile_config is not None else TorchCompileConfig. - model_fields['enable_piecewise_cuda_graph'].default, - torch_compile_piecewise_cuda_graph_num_tokens=self. - torch_compile_config.capture_num_tokens - if self.torch_compile_config is not None else - TorchCompileConfig.model_fields['capture_num_tokens'].default, - torch_compile_enable_userbuffers=self.torch_compile_config. - enable_userbuffers if self.torch_compile_config is not None else - TorchCompileConfig.model_fields['enable_userbuffers'].default, - torch_compile_max_num_streams=self.torch_compile_config. - max_num_streams if self.torch_compile_config is not None else - TorchCompileConfig.model_fields['max_num_streams'].default, - enable_autotuner=self.enable_autotuner, - enable_layerwise_nvtx_marker=self.enable_layerwise_nvtx_marker, - load_format=self.load_format, - enable_min_latency=self.enable_min_latency, - moe_disable_finalize_fusion=self.moe_config.disable_finalize_fusion, - stream_interval=self.stream_interval, - force_dynamic_quantization=self.force_dynamic_quantization, - allreduce_strategy=self.allreduce_strategy, - attention_dp_enable_balance=bool( - self.attention_dp_config is not None - and self.attention_dp_config.enable_balance), - attention_dp_time_out_iters=self.attention_dp_config.timeout_iters - if self.attention_dp_config is not None else - AttentionDpConfig.model_fields['timeout_iters'].default, - attention_dp_batching_wait_iters=self.attention_dp_config. - batching_wait_iters if self.attention_dp_config is not None else - AttentionDpConfig.model_fields['batching_wait_iters'].default, - batch_wait_timeout_ms=self.batch_wait_timeout_ms, - batch_wait_timeout_iters=self.batch_wait_timeout_iters, - batch_wait_max_tokens_ratio=self.batch_wait_max_tokens_ratio, - enable_sleep=self.enable_sleep, - )
@@ -3820,10 +3934,12 @@ + +
@@ -3845,12 +3961,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -3898,9 +4009,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/llmapi/mm_encoder.html b/latest/_modules/tensorrt_llm/llmapi/mm_encoder.html index 294626ff1d..65c742a5bc 100644 --- a/latest/_modules/tensorrt_llm/llmapi/mm_encoder.html +++ b/latest/_modules/tensorrt_llm/llmapi/mm_encoder.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -690,10 +698,12 @@
+ +
@@ -715,12 +725,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -768,9 +773,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/llmapi/mpi_session.html b/latest/_modules/tensorrt_llm/llmapi/mpi_session.html index 345d4d77d5..82524b9394 100644 --- a/latest/_modules/tensorrt_llm/llmapi/mpi_session.html +++ b/latest/_modules/tensorrt_llm/llmapi/mpi_session.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1153,10 +1161,12 @@
+ +
@@ -1178,12 +1188,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1231,9 +1236,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/baichuan/model.html b/latest/_modules/tensorrt_llm/models/baichuan/model.html index 029dc2274d..1a00e90e9c 100644 --- a/latest/_modules/tensorrt_llm/models/baichuan/model.html +++ b/latest/_modules/tensorrt_llm/models/baichuan/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -798,10 +806,12 @@
+ +
@@ -823,12 +833,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -876,9 +881,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/bert/model.html b/latest/_modules/tensorrt_llm/models/bert/model.html index f84f9cf3c0..24810fdf93 100644 --- a/latest/_modules/tensorrt_llm/models/bert/model.html +++ b/latest/_modules/tensorrt_llm/models/bert/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1102,10 +1110,12 @@
+ +
@@ -1127,12 +1137,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1180,9 +1185,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/bloom/model.html b/latest/_modules/tensorrt_llm/models/bloom/model.html index 8fe337eb08..40ff2319a4 100644 --- a/latest/_modules/tensorrt_llm/models/bloom/model.html +++ b/latest/_modules/tensorrt_llm/models/bloom/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -710,10 +718,12 @@
+ +
@@ -735,12 +745,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -788,9 +793,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/chatglm/config.html b/latest/_modules/tensorrt_llm/models/chatglm/config.html index 57eb5e0c9b..9fd0e354cc 100644 --- a/latest/_modules/tensorrt_llm/models/chatglm/config.html +++ b/latest/_modules/tensorrt_llm/models/chatglm/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -727,10 +735,12 @@
+ +
@@ -752,12 +762,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -805,9 +810,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/chatglm/model.html b/latest/_modules/tensorrt_llm/models/chatglm/model.html index 974479e772..03a3321585 100644 --- a/latest/_modules/tensorrt_llm/models/chatglm/model.html +++ b/latest/_modules/tensorrt_llm/models/chatglm/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -926,10 +934,12 @@
+ +
@@ -951,12 +961,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1004,9 +1009,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/clip/model.html b/latest/_modules/tensorrt_llm/models/clip/model.html index 98fd47952e..224ad085fc 100644 --- a/latest/_modules/tensorrt_llm/models/clip/model.html +++ b/latest/_modules/tensorrt_llm/models/clip/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -755,10 +763,12 @@
+ +
@@ -780,12 +790,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -833,9 +838,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/cogvlm/config.html b/latest/_modules/tensorrt_llm/models/cogvlm/config.html index f7d349985e..c2cda288af 100644 --- a/latest/_modules/tensorrt_llm/models/cogvlm/config.html +++ b/latest/_modules/tensorrt_llm/models/cogvlm/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -586,10 +594,12 @@
+ +
@@ -611,12 +621,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -664,9 +669,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/cogvlm/model.html b/latest/_modules/tensorrt_llm/models/cogvlm/model.html index d9afa6de56..555348076d 100644 --- a/latest/_modules/tensorrt_llm/models/cogvlm/model.html +++ b/latest/_modules/tensorrt_llm/models/cogvlm/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -839,10 +847,12 @@
+ +
@@ -864,12 +874,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -917,9 +922,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/commandr/model.html b/latest/_modules/tensorrt_llm/models/commandr/model.html index 56cda556a9..dfbdd75869 100644 --- a/latest/_modules/tensorrt_llm/models/commandr/model.html +++ b/latest/_modules/tensorrt_llm/models/commandr/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -737,10 +745,12 @@
+ +
@@ -762,12 +772,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -815,9 +820,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/dbrx/config.html b/latest/_modules/tensorrt_llm/models/dbrx/config.html index ebfca7311f..f85fdcdde3 100644 --- a/latest/_modules/tensorrt_llm/models/dbrx/config.html +++ b/latest/_modules/tensorrt_llm/models/dbrx/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -601,10 +609,12 @@
+ +
@@ -626,12 +636,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -679,9 +684,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/dbrx/model.html b/latest/_modules/tensorrt_llm/models/dbrx/model.html index 95727293f7..d6bf0f0218 100644 --- a/latest/_modules/tensorrt_llm/models/dbrx/model.html +++ b/latest/_modules/tensorrt_llm/models/dbrx/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -727,10 +735,12 @@
+ +
@@ -752,12 +762,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -805,9 +810,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/deepseek_v1/model.html b/latest/_modules/tensorrt_llm/models/deepseek_v1/model.html index b71b01bb1f..0d431eefa4 100644 --- a/latest/_modules/tensorrt_llm/models/deepseek_v1/model.html +++ b/latest/_modules/tensorrt_llm/models/deepseek_v1/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -821,10 +829,12 @@
+ +
@@ -846,12 +856,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -899,9 +904,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/deepseek_v2/model.html b/latest/_modules/tensorrt_llm/models/deepseek_v2/model.html index e7d87fb611..0c5ac511e8 100644 --- a/latest/_modules/tensorrt_llm/models/deepseek_v2/model.html +++ b/latest/_modules/tensorrt_llm/models/deepseek_v2/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -903,10 +911,12 @@
+ +
@@ -928,12 +938,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -981,9 +986,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/dit/model.html b/latest/_modules/tensorrt_llm/models/dit/model.html index e1742c8c8b..78f320cc67 100644 --- a/latest/_modules/tensorrt_llm/models/dit/model.html +++ b/latest/_modules/tensorrt_llm/models/dit/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -939,10 +947,12 @@
+ +
@@ -964,12 +974,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1017,9 +1022,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/eagle/model.html b/latest/_modules/tensorrt_llm/models/eagle/model.html index 3f1bcf780b..9fc1def7d9 100644 --- a/latest/_modules/tensorrt_llm/models/eagle/model.html +++ b/latest/_modules/tensorrt_llm/models/eagle/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1875,10 +1883,12 @@
+ +
@@ -1900,12 +1910,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1953,9 +1958,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/enc_dec/model.html b/latest/_modules/tensorrt_llm/models/enc_dec/model.html index 71fb11104d..8c3ff84216 100644 --- a/latest/_modules/tensorrt_llm/models/enc_dec/model.html +++ b/latest/_modules/tensorrt_llm/models/enc_dec/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -2782,10 +2790,12 @@
+ +
@@ -2807,12 +2817,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -2860,9 +2865,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/falcon/config.html b/latest/_modules/tensorrt_llm/models/falcon/config.html index 23d343afa1..c1aa3689d0 100644 --- a/latest/_modules/tensorrt_llm/models/falcon/config.html +++ b/latest/_modules/tensorrt_llm/models/falcon/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -662,10 +670,12 @@
+ +
@@ -687,12 +697,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -740,9 +745,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/falcon/model.html b/latest/_modules/tensorrt_llm/models/falcon/model.html index e617d8f85b..2be41eb6dd 100644 --- a/latest/_modules/tensorrt_llm/models/falcon/model.html +++ b/latest/_modules/tensorrt_llm/models/falcon/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -824,10 +832,12 @@
+ +
@@ -849,12 +859,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -902,9 +907,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/gemma/config.html b/latest/_modules/tensorrt_llm/models/gemma/config.html index 9ae58a208b..a8587c4155 100644 --- a/latest/_modules/tensorrt_llm/models/gemma/config.html +++ b/latest/_modules/tensorrt_llm/models/gemma/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -752,10 +760,12 @@
+ +
@@ -777,12 +787,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -830,9 +835,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/gemma/model.html b/latest/_modules/tensorrt_llm/models/gemma/model.html index 6bf6be27bc..1f68dc75a0 100644 --- a/latest/_modules/tensorrt_llm/models/gemma/model.html +++ b/latest/_modules/tensorrt_llm/models/gemma/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -947,10 +955,12 @@
+ +
@@ -972,12 +982,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1025,9 +1030,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/gpt/config.html b/latest/_modules/tensorrt_llm/models/gpt/config.html index 794f35c2d8..3d7a1d560c 100644 --- a/latest/_modules/tensorrt_llm/models/gpt/config.html +++ b/latest/_modules/tensorrt_llm/models/gpt/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -871,10 +879,12 @@
+ +
@@ -896,12 +906,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -949,9 +954,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/gpt/model.html b/latest/_modules/tensorrt_llm/models/gpt/model.html index 2626cb8790..a2ab4fc197 100644 --- a/latest/_modules/tensorrt_llm/models/gpt/model.html +++ b/latest/_modules/tensorrt_llm/models/gpt/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -974,10 +982,12 @@
+ +
@@ -999,12 +1009,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1052,9 +1057,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/gptj/config.html b/latest/_modules/tensorrt_llm/models/gptj/config.html index d6d4f4998d..d0ad25e278 100644 --- a/latest/_modules/tensorrt_llm/models/gptj/config.html +++ b/latest/_modules/tensorrt_llm/models/gptj/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -600,10 +608,12 @@
+ +
@@ -625,12 +635,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -678,9 +683,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/gptj/model.html b/latest/_modules/tensorrt_llm/models/gptj/model.html index 391902d7d9..d38a902e12 100644 --- a/latest/_modules/tensorrt_llm/models/gptj/model.html +++ b/latest/_modules/tensorrt_llm/models/gptj/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -750,10 +758,12 @@
+ +
@@ -775,12 +785,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -828,9 +833,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/gptneox/model.html b/latest/_modules/tensorrt_llm/models/gptneox/model.html index 406b65b14c..0809638485 100644 --- a/latest/_modules/tensorrt_llm/models/gptneox/model.html +++ b/latest/_modules/tensorrt_llm/models/gptneox/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -692,10 +700,12 @@
+ +
@@ -717,12 +727,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -770,9 +775,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/llama/config.html b/latest/_modules/tensorrt_llm/models/llama/config.html index 43ee90e66e..ba3fd2fc80 100644 --- a/latest/_modules/tensorrt_llm/models/llama/config.html +++ b/latest/_modules/tensorrt_llm/models/llama/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -826,10 +834,12 @@
+ +
@@ -851,12 +861,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -904,9 +909,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/llama/model.html b/latest/_modules/tensorrt_llm/models/llama/model.html index db2e1bcb78..409a25337c 100644 --- a/latest/_modules/tensorrt_llm/models/llama/model.html +++ b/latest/_modules/tensorrt_llm/models/llama/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1174,10 +1182,12 @@
+ +
@@ -1199,12 +1209,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1252,9 +1257,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/mamba/model.html b/latest/_modules/tensorrt_llm/models/mamba/model.html index a894470341..15f6bebc91 100644 --- a/latest/_modules/tensorrt_llm/models/mamba/model.html +++ b/latest/_modules/tensorrt_llm/models/mamba/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1019,10 +1027,12 @@
+ +
@@ -1044,12 +1054,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1097,9 +1102,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/medusa/config.html b/latest/_modules/tensorrt_llm/models/medusa/config.html index 7060c82ffb..8003605511 100644 --- a/latest/_modules/tensorrt_llm/models/medusa/config.html +++ b/latest/_modules/tensorrt_llm/models/medusa/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -659,10 +667,12 @@
+ +
@@ -684,12 +694,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -737,9 +742,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/medusa/model.html b/latest/_modules/tensorrt_llm/models/medusa/model.html index e4b377846c..9e5fc3e20d 100644 --- a/latest/_modules/tensorrt_llm/models/medusa/model.html +++ b/latest/_modules/tensorrt_llm/models/medusa/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -809,10 +817,12 @@
+ +
@@ -834,12 +844,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -887,9 +892,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/mllama/model.html b/latest/_modules/tensorrt_llm/models/mllama/model.html index 85737f1f0c..a4024ce3fe 100644 --- a/latest/_modules/tensorrt_llm/models/mllama/model.html +++ b/latest/_modules/tensorrt_llm/models/mllama/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -2120,10 +2128,12 @@
+ +
@@ -2145,12 +2155,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -2198,9 +2203,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/mmdit_sd3/model.html b/latest/_modules/tensorrt_llm/models/mmdit_sd3/model.html index 696af416b4..fa1ff61f9e 100644 --- a/latest/_modules/tensorrt_llm/models/mmdit_sd3/model.html +++ b/latest/_modules/tensorrt_llm/models/mmdit_sd3/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1186,10 +1194,12 @@
+ +
@@ -1211,12 +1221,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1264,9 +1269,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/modeling_utils.html b/latest/_modules/tensorrt_llm/models/modeling_utils.html index c041d480c5..ea7bd83373 100644 --- a/latest/_modules/tensorrt_llm/models/modeling_utils.html +++ b/latest/_modules/tensorrt_llm/models/modeling_utils.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -2595,10 +2603,12 @@
+ +
@@ -2620,12 +2630,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -2673,9 +2678,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/mpt/model.html b/latest/_modules/tensorrt_llm/models/mpt/model.html index 7f493dc8b1..21d4d28661 100644 --- a/latest/_modules/tensorrt_llm/models/mpt/model.html +++ b/latest/_modules/tensorrt_llm/models/mpt/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -724,10 +732,12 @@
+ +
@@ -749,12 +759,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -802,9 +807,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/multimodal_encoders/config.html b/latest/_modules/tensorrt_llm/models/multimodal_encoders/config.html index 22df3b9efb..590da65c85 100644 --- a/latest/_modules/tensorrt_llm/models/multimodal_encoders/config.html +++ b/latest/_modules/tensorrt_llm/models/multimodal_encoders/config.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -658,10 +666,12 @@
+ +
@@ -683,12 +693,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -736,9 +741,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/multimodal_encoders/model.html b/latest/_modules/tensorrt_llm/models/multimodal_encoders/model.html index 28bdeeb20b..406b42f929 100644 --- a/latest/_modules/tensorrt_llm/models/multimodal_encoders/model.html +++ b/latest/_modules/tensorrt_llm/models/multimodal_encoders/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -726,10 +734,12 @@
+ +
@@ -751,12 +761,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -804,9 +809,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/opt/model.html b/latest/_modules/tensorrt_llm/models/opt/model.html index 942d48e38e..63af1be9ea 100644 --- a/latest/_modules/tensorrt_llm/models/opt/model.html +++ b/latest/_modules/tensorrt_llm/models/opt/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -729,10 +737,12 @@
+ +
@@ -754,12 +764,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -807,9 +812,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/phi/model.html b/latest/_modules/tensorrt_llm/models/phi/model.html index de79fdb51e..207f89934e 100644 --- a/latest/_modules/tensorrt_llm/models/phi/model.html +++ b/latest/_modules/tensorrt_llm/models/phi/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -771,10 +779,12 @@
+ +
@@ -796,12 +806,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -849,9 +854,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/phi3/model.html b/latest/_modules/tensorrt_llm/models/phi3/model.html index 113747d83a..3c5c81ef91 100644 --- a/latest/_modules/tensorrt_llm/models/phi3/model.html +++ b/latest/_modules/tensorrt_llm/models/phi3/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -867,10 +875,12 @@
+ +
@@ -892,12 +902,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -945,9 +950,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/recurrentgemma/model.html b/latest/_modules/tensorrt_llm/models/recurrentgemma/model.html index a0674e1e6c..b30c785215 100644 --- a/latest/_modules/tensorrt_llm/models/recurrentgemma/model.html +++ b/latest/_modules/tensorrt_llm/models/recurrentgemma/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1172,10 +1180,12 @@
+ +
@@ -1197,12 +1207,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1250,9 +1255,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/models/redrafter/model.html b/latest/_modules/tensorrt_llm/models/redrafter/model.html index f021db638d..3d3fab6184 100644 --- a/latest/_modules/tensorrt_llm/models/redrafter/model.html +++ b/latest/_modules/tensorrt_llm/models/redrafter/model.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -859,10 +867,12 @@
+ +
@@ -884,12 +894,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -937,9 +942,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/plugin/plugin.html b/latest/_modules/tensorrt_llm/plugin/plugin.html index 91fee392dd..ad11f22df6 100644 --- a/latest/_modules/tensorrt_llm/plugin/plugin.html +++ b/latest/_modules/tensorrt_llm/plugin/plugin.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1316,10 +1324,12 @@
+ +
@@ -1341,12 +1351,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1394,9 +1399,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/quantization/mode.html b/latest/_modules/tensorrt_llm/quantization/mode.html index 17609b8dda..c1e98c4869 100644 --- a/latest/_modules/tensorrt_llm/quantization/mode.html +++ b/latest/_modules/tensorrt_llm/quantization/mode.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1017,10 +1025,12 @@
+ +
@@ -1042,12 +1052,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1095,9 +1100,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/quantization/quantize_by_modelopt.html b/latest/_modules/tensorrt_llm/quantization/quantize_by_modelopt.html index adca519a32..7bfd70fca9 100644 --- a/latest/_modules/tensorrt_llm/quantization/quantize_by_modelopt.html +++ b/latest/_modules/tensorrt_llm/quantization/quantize_by_modelopt.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1815,10 +1823,12 @@
+ +
@@ -1840,12 +1850,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1893,9 +1898,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/runtime/enc_dec_model_runner.html b/latest/_modules/tensorrt_llm/runtime/enc_dec_model_runner.html index d5999f6b46..0ec7f6a011 100644 --- a/latest/_modules/tensorrt_llm/runtime/enc_dec_model_runner.html +++ b/latest/_modules/tensorrt_llm/runtime/enc_dec_model_runner.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1086,10 +1094,12 @@
+ +
@@ -1111,12 +1121,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1164,9 +1169,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/runtime/generation.html b/latest/_modules/tensorrt_llm/runtime/generation.html index 97e296022d..6557aa0a04 100644 --- a/latest/_modules/tensorrt_llm/runtime/generation.html +++ b/latest/_modules/tensorrt_llm/runtime/generation.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -5426,10 +5434,12 @@
+ +
@@ -5451,12 +5461,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -5504,9 +5509,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/runtime/kv_cache_manager.html b/latest/_modules/tensorrt_llm/runtime/kv_cache_manager.html index 0420092b4d..40eaeb8c27 100644 --- a/latest/_modules/tensorrt_llm/runtime/kv_cache_manager.html +++ b/latest/_modules/tensorrt_llm/runtime/kv_cache_manager.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1030,10 +1038,12 @@
+ +
@@ -1055,12 +1065,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1108,9 +1113,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/runtime/model_runner.html b/latest/_modules/tensorrt_llm/runtime/model_runner.html index f179b8447e..59a2a61109 100644 --- a/latest/_modules/tensorrt_llm/runtime/model_runner.html +++ b/latest/_modules/tensorrt_llm/runtime/model_runner.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1549,10 +1557,12 @@
+ +
@@ -1574,12 +1584,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1627,9 +1632,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/runtime/model_runner_cpp.html b/latest/_modules/tensorrt_llm/runtime/model_runner_cpp.html index 7263ad6852..f64aba9a01 100644 --- a/latest/_modules/tensorrt_llm/runtime/model_runner_cpp.html +++ b/latest/_modules/tensorrt_llm/runtime/model_runner_cpp.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1762,10 +1770,12 @@
+ +
@@ -1787,12 +1797,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1840,9 +1845,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/runtime/multimodal_model_runner.html b/latest/_modules/tensorrt_llm/runtime/multimodal_model_runner.html index b70490623b..f982b88414 100644 --- a/latest/_modules/tensorrt_llm/runtime/multimodal_model_runner.html +++ b/latest/_modules/tensorrt_llm/runtime/multimodal_model_runner.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -3344,10 +3352,12 @@
+ +
@@ -3369,12 +3379,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -3422,9 +3427,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/runtime/session.html b/latest/_modules/tensorrt_llm/runtime/session.html index a69b564cd1..c4e0f21c64 100644 --- a/latest/_modules/tensorrt_llm/runtime/session.html +++ b/latest/_modules/tensorrt_llm/runtime/session.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -890,10 +898,12 @@
+ +
@@ -915,12 +925,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -968,9 +973,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_modules/tensorrt_llm/sampling_params.html b/latest/_modules/tensorrt_llm/sampling_params.html index 50ab29bbb1..c9a093e93d 100644 --- a/latest/_modules/tensorrt_llm/sampling_params.html +++ b/latest/_modules/tensorrt_llm/sampling_params.html @@ -31,7 +31,7 @@ - + @@ -43,6 +43,8 @@ + + @@ -58,20 +60,24 @@ + + + - + + @@ -79,6 +85,8 @@ + +
@@ -1085,10 +1093,12 @@
+ +
@@ -1110,12 +1120,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1163,9 +1168,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/_sources/_cpp_gen/executor.rst.txt b/latest/_sources/_cpp_gen/executor.rst.txt index 39b9a6f5a4..d3ca9cd473 100644 --- a/latest/_sources/_cpp_gen/executor.rst.txt +++ b/latest/_sources/_cpp_gen/executor.rst.txt @@ -4,24 +4,6 @@ Executor .. Here are files in the cpp/include/executor .. We manually add subsection to enable detailed description in the future .. It is also doable to automatically generate this file and list all the modules in the conf.py -transferAgent.h -_______________ - -.. doxygenfile:: transferAgent.h - :project: TensorRT-LLM - -types.h -_______ - -.. doxygenfile:: types.h - :project: TensorRT-LLM - -cacheCommunicator.h -___________________ - -.. doxygenfile:: cacheCommunicator.h - :project: TensorRT-LLM - disaggServerUtil.h __________________ @@ -34,6 +16,24 @@ ________ .. doxygenfile:: tensor.h :project: TensorRT-LLM +transferAgent.h +_______________ + +.. doxygenfile:: transferAgent.h + :project: TensorRT-LLM + +serialization.h +_______________ + +.. doxygenfile:: serialization.h + :project: TensorRT-LLM + +types.h +_______ + +.. doxygenfile:: types.h + :project: TensorRT-LLM + executor.h __________ @@ -46,9 +46,9 @@ ______________________ .. doxygenfile:: dataTransceiverState.h :project: TensorRT-LLM -serialization.h -_______________ +cacheCommunicator.h +___________________ -.. doxygenfile:: serialization.h +.. doxygenfile:: cacheCommunicator.h :project: TensorRT-LLM diff --git a/latest/_sources/_cpp_gen/runtime.rst.txt b/latest/_sources/_cpp_gen/runtime.rst.txt index b8dd953966..536188f7ce 100644 --- a/latest/_sources/_cpp_gen/runtime.rst.txt +++ b/latest/_sources/_cpp_gen/runtime.rst.txt @@ -4,46 +4,10 @@ Runtime .. Here are files in the cpp/include/runtime .. We manually add subsection to enable detailed description in the future .. It is also doable to automatically generate this file and list all the modules in the conf.py -iTensor.h -_________ +lookaheadBuffers.h +__________________ -.. doxygenfile:: iTensor.h - :project: TensorRT-LLM - -cudaEvent.h -___________ - -.. doxygenfile:: cudaEvent.h - :project: TensorRT-LLM - -virtualMemory.h -_______________ - -.. doxygenfile:: virtualMemory.h - :project: TensorRT-LLM - -speculativeDecodingModule.h -___________________________ - -.. doxygenfile:: speculativeDecodingModule.h - :project: TensorRT-LLM - -common.h -________ - -.. doxygenfile:: common.h - :project: TensorRT-LLM - -samplingConfig.h -________________ - -.. doxygenfile:: samplingConfig.h - :project: TensorRT-LLM - -tllmLogger.h -____________ - -.. doxygenfile:: tllmLogger.h +.. doxygenfile:: lookaheadBuffers.h :project: TensorRT-LLM lookaheadModule.h @@ -52,88 +16,28 @@ _________________ .. doxygenfile:: lookaheadModule.h :project: TensorRT-LLM +iBuffer.h +_________ + +.. doxygenfile:: iBuffer.h + :project: TensorRT-LLM + modelConfig.h _____________ .. doxygenfile:: modelConfig.h :project: TensorRT-LLM -iGptDecoderBatched.h -____________________ - -.. doxygenfile:: iGptDecoderBatched.h - :project: TensorRT-LLM - -cudaStream.h -____________ - -.. doxygenfile:: cudaStream.h - :project: TensorRT-LLM - -loraCache.h -___________ - -.. doxygenfile:: loraCache.h - :project: TensorRT-LLM - -medusaModule.h -______________ - -.. doxygenfile:: medusaModule.h - :project: TensorRT-LLM - -decoderState.h -______________ - -.. doxygenfile:: decoderState.h - :project: TensorRT-LLM - -lookaheadBuffers.h -__________________ - -.. doxygenfile:: lookaheadBuffers.h - :project: TensorRT-LLM - -eagleModule.h -_____________ - -.. doxygenfile:: eagleModule.h - :project: TensorRT-LLM - -runtimeDefaults.h -_________________ - -.. doxygenfile:: runtimeDefaults.h - :project: TensorRT-LLM - decodingOutput.h ________________ .. doxygenfile:: decodingOutput.h :project: TensorRT-LLM -decodingInput.h -_______________ +promptTuningParams.h +____________________ -.. doxygenfile:: decodingInput.h - :project: TensorRT-LLM - -worldConfig.h -_____________ - -.. doxygenfile:: worldConfig.h - :project: TensorRT-LLM - -gptDecoderBatched.h -___________________ - -.. doxygenfile:: gptDecoderBatched.h - :project: TensorRT-LLM - -explicitDraftTokensBuffers.h -____________________________ - -.. doxygenfile:: explicitDraftTokensBuffers.h +.. doxygenfile:: promptTuningParams.h :project: TensorRT-LLM bufferManager.h @@ -142,46 +46,22 @@ _______________ .. doxygenfile:: bufferManager.h :project: TensorRT-LLM -loraModule.h -____________ - -.. doxygenfile:: loraModule.h - :project: TensorRT-LLM - -eagleBuffers.h -______________ - -.. doxygenfile:: eagleBuffers.h - :project: TensorRT-LLM - -speculativeDecodingMode.h -_________________________ - -.. doxygenfile:: speculativeDecodingMode.h - :project: TensorRT-LLM - -promptTuningParams.h -____________________ - -.. doxygenfile:: promptTuningParams.h - :project: TensorRT-LLM - -gptDecoder.h -____________ - -.. doxygenfile:: gptDecoder.h - :project: TensorRT-LLM - -memoryCounters.h -________________ - -.. doxygenfile:: memoryCounters.h - :project: TensorRT-LLM - -ipcNvlsMemory.h +gptJsonConfig.h _______________ -.. doxygenfile:: ipcNvlsMemory.h +.. doxygenfile:: gptJsonConfig.h + :project: TensorRT-LLM + +runtimeDefaults.h +_________________ + +.. doxygenfile:: runtimeDefaults.h + :project: TensorRT-LLM + +loraCache.h +___________ + +.. doxygenfile:: loraCache.h :project: TensorRT-LLM rawEngine.h @@ -190,22 +70,46 @@ ___________ .. doxygenfile:: rawEngine.h :project: TensorRT-LLM -ipcUtils.h -__________ +gptDecoder.h +____________ -.. doxygenfile:: ipcUtils.h +.. doxygenfile:: gptDecoder.h :project: TensorRT-LLM -iBuffer.h -_________ +eagleBuffers.h +______________ -.. doxygenfile:: iBuffer.h +.. doxygenfile:: eagleBuffers.h :project: TensorRT-LLM -gptJsonConfig.h +medusaModule.h +______________ + +.. doxygenfile:: medusaModule.h + :project: TensorRT-LLM + +virtualMemory.h _______________ -.. doxygenfile:: gptJsonConfig.h +.. doxygenfile:: virtualMemory.h + :project: TensorRT-LLM + +explicitDraftTokensBuffers.h +____________________________ + +.. doxygenfile:: explicitDraftTokensBuffers.h + :project: TensorRT-LLM + +iTensor.h +_________ + +.. doxygenfile:: iTensor.h + :project: TensorRT-LLM + +common.h +________ + +.. doxygenfile:: common.h :project: TensorRT-LLM loraCachePageManagerConfig.h @@ -214,3 +118,99 @@ ____________________________ .. doxygenfile:: loraCachePageManagerConfig.h :project: TensorRT-LLM +worldConfig.h +_____________ + +.. doxygenfile:: worldConfig.h + :project: TensorRT-LLM + +loraModule.h +____________ + +.. doxygenfile:: loraModule.h + :project: TensorRT-LLM + +speculativeDecodingMode.h +_________________________ + +.. doxygenfile:: speculativeDecodingMode.h + :project: TensorRT-LLM + +cudaEvent.h +___________ + +.. doxygenfile:: cudaEvent.h + :project: TensorRT-LLM + +decodingInput.h +_______________ + +.. doxygenfile:: decodingInput.h + :project: TensorRT-LLM + +speculativeDecodingModule.h +___________________________ + +.. doxygenfile:: speculativeDecodingModule.h + :project: TensorRT-LLM + +iGptDecoderBatched.h +____________________ + +.. doxygenfile:: iGptDecoderBatched.h + :project: TensorRT-LLM + +eagleModule.h +_____________ + +.. doxygenfile:: eagleModule.h + :project: TensorRT-LLM + +tllmLogger.h +____________ + +.. doxygenfile:: tllmLogger.h + :project: TensorRT-LLM + +gptDecoderBatched.h +___________________ + +.. doxygenfile:: gptDecoderBatched.h + :project: TensorRT-LLM + +cudaStream.h +____________ + +.. doxygenfile:: cudaStream.h + :project: TensorRT-LLM + +ipcNvlsMemory.h +_______________ + +.. doxygenfile:: ipcNvlsMemory.h + :project: TensorRT-LLM + +samplingConfig.h +________________ + +.. doxygenfile:: samplingConfig.h + :project: TensorRT-LLM + +decoderState.h +______________ + +.. doxygenfile:: decoderState.h + :project: TensorRT-LLM + +ipcUtils.h +__________ + +.. doxygenfile:: ipcUtils.h + :project: TensorRT-LLM + +memoryCounters.h +________________ + +.. doxygenfile:: memoryCounters.h + :project: TensorRT-LLM + diff --git a/latest/_sources/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md.txt b/latest/_sources/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md.txt index 5d3a84cdb2..9f894a8d7e 100644 --- a/latest/_sources/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md.txt +++ b/latest/_sources/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md.txt @@ -25,7 +25,7 @@ TensorRT LLM distributes the pre-built container on [NGC Catalog](https://catalo You can launch the container using the following command: ```bash -docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 +docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 ``` diff --git a/latest/_sources/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md.txt b/latest/_sources/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md.txt index 8bdf42e4c7..0b25c379e2 100644 --- a/latest/_sources/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md.txt +++ b/latest/_sources/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md.txt @@ -47,7 +47,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash ``` diff --git a/latest/_sources/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md.txt b/latest/_sources/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md.txt index c8f055b39e..6c4ece7658 100644 --- a/latest/_sources/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md.txt +++ b/latest/_sources/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md.txt @@ -43,7 +43,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash ``` diff --git a/latest/_sources/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md.txt b/latest/_sources/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md.txt index c680708a0b..5c7c0cf230 100644 --- a/latest/_sources/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md.txt +++ b/latest/_sources/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md.txt @@ -39,7 +39,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash ``` diff --git a/latest/_sources/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md.txt b/latest/_sources/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md.txt index 93589d2047..b1e641c8f6 100644 --- a/latest/_sources/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md.txt +++ b/latest/_sources/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md.txt @@ -38,7 +38,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash ``` diff --git a/latest/_sources/developer-guide/api-change.md.txt b/latest/_sources/developer-guide/api-change.md.txt index 4c17c80739..61f4cfeff2 100644 --- a/latest/_sources/developer-guide/api-change.md.txt +++ b/latest/_sources/developer-guide/api-change.md.txt @@ -34,7 +34,7 @@ TensorRT LLM classifies APIs into two categories: All API schemas are: - Stored as YAML files in the codebase - Protected by unit tests in `tests/unittest/api_stability/` -- Automatically validated to ensure consistency +- Automatically validated to ensure consistency ## API Change Principles @@ -44,22 +44,26 @@ All API schemas are: Argument names should describe what the argument represents, not how it is used internally. -✅ **Good**: `max_new_tokens` (clear meaning) +✅ **Good**: `max_new_tokens` (clear meaning) + ❌ **Bad**: `num` (ambiguous) **Reflect Argument Type and Granularity** - For **boolean** knobs, prefix with verbs like `enable_` and so on. + Examples: `enable_cache`, `enable_flash_attention` -- For **numerical threshold** knobs, suffix with `_limit`, `_size`, `_count`, `_len_` or `_ratio` +- For **numerical threshold** knobs, suffix with `_limit`, `_size`, `_count`, `_len_` or `_ratio` + Examples: `max_seq_len`, `prefill_batch_size` **Avoid Redundant Prefixes** Example (in `MoeConfig`): -✅ **Good**: `backend` +✅ **Good**: `backend` + ❌ **Bad**: `moe_backend` (redundant since it's already in `MoeConfig`) **Use Specific Names for Narrow Scenarios** @@ -68,7 +72,8 @@ When adding knobs for specific use cases, make the name convey the restriction c Example (argument to the LLM class): -✅ **Good**: `rope_scaling_factor` → clearly indicates it's for RoPE +✅ **Good**: `rope_scaling_factor` → clearly indicates it's for RoPE + ❌ **Bad**: `scaling_factor` → too generic and prone to misuse ### 2. Hierarchical Configuration @@ -77,13 +82,16 @@ Organize complex or hierarchical arguments into **dedicated configuration datacl **Guidelines** -- Use the `XxxConfig` suffix consistently +- Use the `XxxConfig` suffix consistently + Examples: `ModelConfig`, `ParallelConfig`, `MoeConfig` - -- **Reflect conceptual hierarchy** + +- **Reflect conceptual hierarchy** + The dataclass name should represent a coherent functional unit, not an arbitrary grouping - -- **Avoid over-nesting** + +- **Avoid over-nesting** + Use only one level of configuration hierarchy whenever possible (e.g., `LlmArgs → ParallelConfig`) to balance readability and modularity ### 3. Prefer `LlmArgs` Over Environment Variables @@ -154,7 +162,7 @@ garbage_collection_gen0_threshold: int = Field( Add the field to the appropriate schema file: -- **Non-committed arguments**: `tests/unittest/api_stability/references/llm_args.yaml` +- **Non-committed arguments**: `tests/unittest/api_stability/references/llm.yaml` ```yaml garbage_collection_gen0_threshold: type: int @@ -162,7 +170,7 @@ Add the field to the appropriate schema file: status: beta # Must match the status in code ``` -- **Committed arguments**: `tests/unittest/api_stability/references_committed/llm_args.yaml` +- **Committed arguments**: `tests/unittest/api_stability/references_committed/llm.yaml` ```yaml garbage_collection_gen0_threshold: type: int @@ -196,16 +204,16 @@ For non-committed APIs, use the `@set_api_status` decorator: ```python @set_api_status("beta") def generate_with_streaming( - self, - prompts: List[str], + self, + prompts: List[str], **kwargs ) -> Iterator[GenerationOutput]: """Generate text with streaming output. - + Args: prompts: Input prompts for generation **kwargs: Additional generation parameters - + Returns: Iterator of generation outputs """ diff --git a/latest/_sources/examples/curl_chat_client.rst.txt b/latest/_sources/examples/curl_chat_client.rst.txt index df501f79cb..1ba2f49063 100644 --- a/latest/_sources/examples/curl_chat_client.rst.txt +++ b/latest/_sources/examples/curl_chat_client.rst.txt @@ -2,7 +2,7 @@ Curl Chat Client ================ Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/curl_chat_client.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/curl_chat_client.sh. .. literalinclude:: ../../../examples/serve/curl_chat_client.sh :lines: 1-11 diff --git a/latest/_sources/examples/curl_chat_client_for_multimodal.rst.txt b/latest/_sources/examples/curl_chat_client_for_multimodal.rst.txt index dda4f7a270..486b1f6591 100644 --- a/latest/_sources/examples/curl_chat_client_for_multimodal.rst.txt +++ b/latest/_sources/examples/curl_chat_client_for_multimodal.rst.txt @@ -2,7 +2,7 @@ Curl Chat Client For Multimodal =============================== Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/curl_chat_client_for_multimodal.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/curl_chat_client_for_multimodal.sh. .. literalinclude:: ../../../examples/serve/curl_chat_client_for_multimodal.sh :lines: 1-88 diff --git a/latest/_sources/examples/curl_completion_client.rst.txt b/latest/_sources/examples/curl_completion_client.rst.txt index 6cafc38a25..71baea84b0 100644 --- a/latest/_sources/examples/curl_completion_client.rst.txt +++ b/latest/_sources/examples/curl_completion_client.rst.txt @@ -2,7 +2,7 @@ Curl Completion Client ====================== Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/curl_completion_client.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/curl_completion_client.sh. .. literalinclude:: ../../../examples/serve/curl_completion_client.sh :lines: 1-10 diff --git a/latest/_sources/examples/deepseek_r1_reasoning_parser.rst.txt b/latest/_sources/examples/deepseek_r1_reasoning_parser.rst.txt index bc281e405b..8e270b1ba7 100644 --- a/latest/_sources/examples/deepseek_r1_reasoning_parser.rst.txt +++ b/latest/_sources/examples/deepseek_r1_reasoning_parser.rst.txt @@ -2,7 +2,7 @@ Deepseek R1 Reasoning Parser ============================ Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/deepseek_r1_reasoning_parser.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/deepseek_r1_reasoning_parser.sh. .. literalinclude:: ../../../examples/serve/deepseek_r1_reasoning_parser.sh :lines: 1-10 diff --git a/latest/_sources/examples/genai_perf_client.rst.txt b/latest/_sources/examples/genai_perf_client.rst.txt index e179b4bb68..dbdabecabc 100644 --- a/latest/_sources/examples/genai_perf_client.rst.txt +++ b/latest/_sources/examples/genai_perf_client.rst.txt @@ -2,7 +2,7 @@ Genai Perf Client ================= Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/genai_perf_client.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/genai_perf_client.sh. .. literalinclude:: ../../../examples/serve/genai_perf_client.sh :lines: 1-16 diff --git a/latest/_sources/examples/genai_perf_client_for_multimodal.rst.txt b/latest/_sources/examples/genai_perf_client_for_multimodal.rst.txt index d8108fb7e2..a6db846c2f 100644 --- a/latest/_sources/examples/genai_perf_client_for_multimodal.rst.txt +++ b/latest/_sources/examples/genai_perf_client_for_multimodal.rst.txt @@ -2,7 +2,7 @@ Genai Perf Client For Multimodal ================================ Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/genai_perf_client_for_multimodal.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/genai_perf_client_for_multimodal.sh. .. literalinclude:: ../../../examples/serve/genai_perf_client_for_multimodal.sh :lines: 1-19 diff --git a/latest/_sources/examples/llm_guided_decoding.rst.txt b/latest/_sources/examples/llm_guided_decoding.rst.txt index 7183afaefc..e5ccdf712f 100644 --- a/latest/_sources/examples/llm_guided_decoding.rst.txt +++ b/latest/_sources/examples/llm_guided_decoding.rst.txt @@ -1,6 +1,6 @@ Generate text with guided decoding ================================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_guided_decoding.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_guided_decoding.py. .. literalinclude:: ../../../examples/llm-api/llm_guided_decoding.py :lines: 4-47 diff --git a/latest/_sources/examples/llm_inference.rst.txt b/latest/_sources/examples/llm_inference.rst.txt index 7ab13621d6..840b845158 100644 --- a/latest/_sources/examples/llm_inference.rst.txt +++ b/latest/_sources/examples/llm_inference.rst.txt @@ -1,6 +1,6 @@ Generate text ============= -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_inference.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_inference.py. .. literalinclude:: ../../../examples/llm-api/llm_inference.py :lines: 4-35 diff --git a/latest/_sources/examples/llm_inference_async.rst.txt b/latest/_sources/examples/llm_inference_async.rst.txt index d400f732a9..20ccc45806 100644 --- a/latest/_sources/examples/llm_inference_async.rst.txt +++ b/latest/_sources/examples/llm_inference_async.rst.txt @@ -1,6 +1,6 @@ Generate text asynchronously ============================ -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_inference_async.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_inference_async.py. .. literalinclude:: ../../../examples/llm-api/llm_inference_async.py :lines: 4-43 diff --git a/latest/_sources/examples/llm_inference_async_streaming.rst.txt b/latest/_sources/examples/llm_inference_async_streaming.rst.txt index d4859f92ff..094826e23b 100644 --- a/latest/_sources/examples/llm_inference_async_streaming.rst.txt +++ b/latest/_sources/examples/llm_inference_async_streaming.rst.txt @@ -1,6 +1,6 @@ Generate text in streaming ========================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_inference_async_streaming.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_inference_async_streaming.py. .. literalinclude:: ../../../examples/llm-api/llm_inference_async_streaming.py :lines: 4-64 diff --git a/latest/_sources/examples/llm_inference_distributed.rst.txt b/latest/_sources/examples/llm_inference_distributed.rst.txt index 45e4fbb34f..dfae6bf4ec 100644 --- a/latest/_sources/examples/llm_inference_distributed.rst.txt +++ b/latest/_sources/examples/llm_inference_distributed.rst.txt @@ -1,6 +1,6 @@ Distributed LLM Generation ========================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_inference_distributed.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_inference_distributed.py. .. literalinclude:: ../../../examples/llm-api/llm_inference_distributed.py :lines: 4-44 diff --git a/latest/_sources/examples/llm_kv_cache_connector.rst.txt b/latest/_sources/examples/llm_kv_cache_connector.rst.txt index 05c13e4ed1..5d5f1fa309 100644 --- a/latest/_sources/examples/llm_kv_cache_connector.rst.txt +++ b/latest/_sources/examples/llm_kv_cache_connector.rst.txt @@ -1,6 +1,6 @@ KV Cache Connector ================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_kv_cache_connector.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_kv_cache_connector.py. .. literalinclude:: ../../../examples/llm-api/llm_kv_cache_connector.py :lines: 4-247 diff --git a/latest/_sources/examples/llm_kv_cache_offloading.rst.txt b/latest/_sources/examples/llm_kv_cache_offloading.rst.txt index c736de385c..00623dd82b 100644 --- a/latest/_sources/examples/llm_kv_cache_offloading.rst.txt +++ b/latest/_sources/examples/llm_kv_cache_offloading.rst.txt @@ -1,6 +1,6 @@ KV Cache Offloading =================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_kv_cache_offloading.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_kv_cache_offloading.py. .. literalinclude:: ../../../examples/llm-api/llm_kv_cache_offloading.py :lines: 4-134 diff --git a/latest/_sources/examples/llm_logits_processor.rst.txt b/latest/_sources/examples/llm_logits_processor.rst.txt index 59c47ddd49..0f1fc7a596 100644 --- a/latest/_sources/examples/llm_logits_processor.rst.txt +++ b/latest/_sources/examples/llm_logits_processor.rst.txt @@ -1,6 +1,6 @@ Control generated text using logits processor ============================================= -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_logits_processor.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_logits_processor.py. .. literalinclude:: ../../../examples/llm-api/llm_logits_processor.py :lines: 4-128 diff --git a/latest/_sources/examples/llm_mgmn_llm_distributed.rst.txt b/latest/_sources/examples/llm_mgmn_llm_distributed.rst.txt index a711a2c992..bfbbd3feee 100644 --- a/latest/_sources/examples/llm_mgmn_llm_distributed.rst.txt +++ b/latest/_sources/examples/llm_mgmn_llm_distributed.rst.txt @@ -1,6 +1,6 @@ Run LLM-API with pytorch backend on Slurm ========================================= -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_mgmn_llm_distributed.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_mgmn_llm_distributed.sh. .. literalinclude:: ../../../examples/llm-api/llm_mgmn_llm_distributed.sh :lines: 1-10,14-55 diff --git a/latest/_sources/examples/llm_mgmn_trtllm_bench.rst.txt b/latest/_sources/examples/llm_mgmn_trtllm_bench.rst.txt index 47d286fdfa..5b1851c1f1 100644 --- a/latest/_sources/examples/llm_mgmn_trtllm_bench.rst.txt +++ b/latest/_sources/examples/llm_mgmn_trtllm_bench.rst.txt @@ -1,6 +1,6 @@ Run trtllm-bench with pytorch backend on Slurm ============================================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_mgmn_trtllm_bench.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_mgmn_trtllm_bench.sh. .. literalinclude:: ../../../examples/llm-api/llm_mgmn_trtllm_bench.sh :lines: 1-10,14-95 diff --git a/latest/_sources/examples/llm_mgmn_trtllm_serve.rst.txt b/latest/_sources/examples/llm_mgmn_trtllm_serve.rst.txt index 6b14f0d61e..01b5ca1a47 100644 --- a/latest/_sources/examples/llm_mgmn_trtllm_serve.rst.txt +++ b/latest/_sources/examples/llm_mgmn_trtllm_serve.rst.txt @@ -1,6 +1,6 @@ Run trtllm-serve with pytorch backend on Slurm ============================================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_mgmn_trtllm_serve.sh. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_mgmn_trtllm_serve.sh. .. literalinclude:: ../../../examples/llm-api/llm_mgmn_trtllm_serve.sh :lines: 1-10,14-56 diff --git a/latest/_sources/examples/llm_multilora.rst.txt b/latest/_sources/examples/llm_multilora.rst.txt index 60ed84cf0a..fb4e601f34 100644 --- a/latest/_sources/examples/llm_multilora.rst.txt +++ b/latest/_sources/examples/llm_multilora.rst.txt @@ -1,6 +1,6 @@ Generate text with multiple LoRA adapters ========================================= -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_multilora.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_multilora.py. .. literalinclude:: ../../../examples/llm-api/llm_multilora.py :lines: 4-89 diff --git a/latest/_sources/examples/llm_runtime.rst.txt b/latest/_sources/examples/llm_runtime.rst.txt index f8dae0cf18..bd0ecd57f8 100644 --- a/latest/_sources/examples/llm_runtime.rst.txt +++ b/latest/_sources/examples/llm_runtime.rst.txt @@ -1,6 +1,6 @@ Runtime Configuration Examples ============================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_runtime.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_runtime.py. .. literalinclude:: ../../../examples/llm-api/llm_runtime.py :lines: 4-96 diff --git a/latest/_sources/examples/llm_sampling.rst.txt b/latest/_sources/examples/llm_sampling.rst.txt index 9bf131085a..439d5e0199 100644 --- a/latest/_sources/examples/llm_sampling.rst.txt +++ b/latest/_sources/examples/llm_sampling.rst.txt @@ -1,6 +1,6 @@ Sampling Techniques Showcase ============================ -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_sampling.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_sampling.py. .. literalinclude:: ../../../examples/llm-api/llm_sampling.py :lines: 4-229 diff --git a/latest/_sources/examples/llm_sparse_attention.rst.txt b/latest/_sources/examples/llm_sparse_attention.rst.txt index 4d3d72b221..be70d09706 100644 --- a/latest/_sources/examples/llm_sparse_attention.rst.txt +++ b/latest/_sources/examples/llm_sparse_attention.rst.txt @@ -1,8 +1,8 @@ Sparse Attention ================ -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_sparse_attention.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_sparse_attention.py. .. literalinclude:: ../../../examples/llm-api/llm_sparse_attention.py - :lines: 4-208 + :lines: 4-209 :language: python :linenos: diff --git a/latest/_sources/examples/llm_speculative_decoding.rst.txt b/latest/_sources/examples/llm_speculative_decoding.rst.txt index 412f190b9c..8ce53a7a92 100644 --- a/latest/_sources/examples/llm_speculative_decoding.rst.txt +++ b/latest/_sources/examples/llm_speculative_decoding.rst.txt @@ -1,6 +1,6 @@ Speculative Decoding ==================== -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/llm-api/llm_speculative_decoding.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/llm-api/llm_speculative_decoding.py. .. literalinclude:: ../../../examples/llm-api/llm_speculative_decoding.py :lines: 4-95 diff --git a/latest/_sources/examples/openai_chat_client.rst.txt b/latest/_sources/examples/openai_chat_client.rst.txt index 0ae2d28e94..f466331fcc 100644 --- a/latest/_sources/examples/openai_chat_client.rst.txt +++ b/latest/_sources/examples/openai_chat_client.rst.txt @@ -2,7 +2,7 @@ OpenAI Chat Client ================== Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/openai_chat_client.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/openai_chat_client.py. .. literalinclude:: ../../../examples/serve/openai_chat_client.py :lines: 2-21 diff --git a/latest/_sources/examples/openai_chat_client_for_multimodal.rst.txt b/latest/_sources/examples/openai_chat_client_for_multimodal.rst.txt index 6789409f58..ea728f4b92 100644 --- a/latest/_sources/examples/openai_chat_client_for_multimodal.rst.txt +++ b/latest/_sources/examples/openai_chat_client_for_multimodal.rst.txt @@ -2,7 +2,7 @@ OpenAI Chat Client for Multimodal ================================= Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/openai_chat_client_for_multimodal.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/openai_chat_client_for_multimodal.py. .. literalinclude:: ../../../examples/serve/openai_chat_client_for_multimodal.py :lines: 2-129 diff --git a/latest/_sources/examples/openai_completion_client.rst.txt b/latest/_sources/examples/openai_completion_client.rst.txt index 7e15368b43..5e6c5935bf 100644 --- a/latest/_sources/examples/openai_completion_client.rst.txt +++ b/latest/_sources/examples/openai_completion_client.rst.txt @@ -2,7 +2,7 @@ OpenAI Completion Client ======================== Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/openai_completion_client.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/openai_completion_client.py. .. literalinclude:: ../../../examples/serve/openai_completion_client.py :lines: 2-15 diff --git a/latest/_sources/examples/openai_completion_client_for_lora.rst.txt b/latest/_sources/examples/openai_completion_client_for_lora.rst.txt index 1840a24c56..43c28642c7 100644 --- a/latest/_sources/examples/openai_completion_client_for_lora.rst.txt +++ b/latest/_sources/examples/openai_completion_client_for_lora.rst.txt @@ -2,7 +2,7 @@ Openai Completion Client For Lora ================================= Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/openai_completion_client_for_lora.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/openai_completion_client_for_lora.py. .. literalinclude:: ../../../examples/serve/openai_completion_client_for_lora.py :lines: 1-30 diff --git a/latest/_sources/examples/openai_completion_client_json_schema.rst.txt b/latest/_sources/examples/openai_completion_client_json_schema.rst.txt index 125b3aa7ca..9c1c3d9f09 100644 --- a/latest/_sources/examples/openai_completion_client_json_schema.rst.txt +++ b/latest/_sources/examples/openai_completion_client_json_schema.rst.txt @@ -2,7 +2,7 @@ OpenAI Completion Client with JSON Schema ========================================= Refer to the `trtllm-serve documentation `_ for starting a server. -Source https://github.com/NVIDIA/TensorRT-LLM/blob/31116825b39f4e6a6a1e127001f5204b73d1dc32/examples/serve/openai_completion_client_json_schema.py. +Source https://github.com/NVIDIA/TensorRT-LLM/blob/2128f73d58508a1a0b37119bd851edb19ab88635/examples/serve/openai_completion_client_json_schema.py. .. literalinclude:: ../../../examples/serve/openai_completion_client_json_schema.py :lines: 2-52 diff --git a/latest/_sources/features/multi-modality.md.txt b/latest/_sources/features/multi-modality.md.txt index b9f2029412..87a7b3bfcd 100644 --- a/latest/_sources/features/multi-modality.md.txt +++ b/latest/_sources/features/multi-modality.md.txt @@ -35,7 +35,7 @@ The following examples demonstrate how to use TensorRT LLM's multimodal support Quickly try out TensorRT LLM's multimodal support using our `LLM-API` and a ready-to-run [example](source:examples/llm-api/quickstart_multimodal.py): ```bash -python3 quickstart_multimodal.py --model_dir Efficient-Large-Model/NVILA-8B --modality image +python3 quickstart_multimodal.py --model_dir Efficient-Large-Model/NVILA-8B --modality image --disable_kv_cache_reuse ``` ### OpenAI-Compatible Server via [`trtllm-serve`](../../source/commands/trtllm-serve/trtllm-serve.rst) diff --git a/latest/_sources/installation/linux.md.txt b/latest/_sources/installation/linux.md.txt index 4f3fd0129e..2aae24e6af 100644 --- a/latest/_sources/installation/linux.md.txt +++ b/latest/_sources/installation/linux.md.txt @@ -59,3 +59,15 @@ There are some known limitations when you pip install pre-built TensorRT LLM whe when OMPI was not configured --with-slurm and we weren't able to discover a SLURM installation in the usual places. ``` + +2. Prevent `pip` from replacing existing PyTorch installation + + On certain systems, particularly Ubuntu 22.04, users installing TensorRT LLM would find that their existing, CUDA 13.0 compatible PyTorch installation (e.g., `torch==2.9.0+cu130`) was being uninstalled by `pip`. It was then replaced by a CUDA 12.8 version (`torch==2.9.0`), causing the TensorRT LLM installation to be unusable and leading to runtime errors. + + The solution is to create a `pip` constraints file, locking `torch` to the currently installed version. Here is an example of how this can be done manually: + + ```bash + CURRENT_TORCH_VERSION=$(python3 -c "import torch; print(torch.__version__)") + echo "torch==$CURRENT_TORCH_VERSION" > /tmp/torch-constraint.txt + pip3 install --upgrade pip setuptools && pip3 install tensorrt_llm -c /tmp/torch-constraint.txt + ``` diff --git a/latest/_sources/llm-api/reference.rst.txt b/latest/_sources/llm-api/reference.rst.txt index 1eb25571f7..0aebe88699 100644 --- a/latest/_sources/llm-api/reference.rst.txt +++ b/latest/_sources/llm-api/reference.rst.txt @@ -288,7 +288,7 @@ API Reference :special-members: __init__ :member-order: groupwise :inherited-members: - :exclude-members: update_forward_refs,model_construct,model_dump,schema_json,parse_raw,schema,model_fields,model_post_init,model_rebuild,model_config,model_json_schema,construct,from_orm,dict,model_fields_set,model_validate,model_computed_fields,model_extra,copy,model_validate_json,model_validate_strings,parse_file,parse_obj,model_dump_json,validate,model_parametrized_name,json,model_copy + :exclude-members: validate,json,model_extra,model_validate_strings,model_parametrized_name,parse_file,model_computed_fields,schema_json,model_dump_json,model_validate_json,update_forward_refs,dict,model_validate,model_fields,copy,model_post_init,schema,model_json_schema,model_copy,model_rebuild,from_orm,model_fields_set,construct,model_construct,model_dump,parse_obj,model_config,parse_raw .. autoclass:: tensorrt_llm.llmapi.TrtLlmArgs :members: @@ -297,7 +297,7 @@ API Reference :special-members: __init__ :member-order: groupwise :inherited-members: - :exclude-members: update_forward_refs,model_construct,model_dump,schema_json,parse_raw,schema,model_fields,model_post_init,model_rebuild,model_config,model_json_schema,construct,from_orm,dict,model_fields_set,model_validate,model_computed_fields,model_extra,copy,model_validate_json,model_validate_strings,parse_file,parse_obj,model_dump_json,validate,model_parametrized_name,json,model_copy + :exclude-members: validate,json,model_extra,model_validate_strings,model_parametrized_name,parse_file,model_computed_fields,schema_json,model_dump_json,model_validate_json,update_forward_refs,dict,model_validate,model_fields,copy,model_post_init,schema,model_json_schema,model_copy,model_rebuild,from_orm,model_fields_set,construct,model_construct,model_dump,parse_obj,model_config,parse_raw .. autoclass:: tensorrt_llm.llmapi.AutoDecodingConfig :members: diff --git a/latest/_sources/quick-start-guide.md.txt b/latest/_sources/quick-start-guide.md.txt index 8bb856306f..c22aeb82ef 100644 --- a/latest/_sources/quick-start-guide.md.txt +++ b/latest/_sources/quick-start-guide.md.txt @@ -10,7 +10,7 @@ This is the starting point to try out TensorRT LLM. Specifically, this Quick Sta The [TensorRT LLM container](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags) maintained by NVIDIA contains all of the required dependencies pre-installed. You can start the container on a machine with NVIDIA GPUs via: ```bash -docker run --rm -it --ipc host --gpus all --ulimit memlock=-1 --ulimit stack=67108864 -p 8000:8000 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 +docker run --rm -it --ipc host --gpus all --ulimit memlock=-1 --ulimit stack=67108864 -p 8000:8000 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 ``` diff --git a/latest/_static/styles/nvidia-sphinx-theme.css b/latest/_static/styles/nvidia-sphinx-theme.css index 70478d60fc..4753a2d266 100644 --- a/latest/_static/styles/nvidia-sphinx-theme.css +++ b/latest/_static/styles/nvidia-sphinx-theme.css @@ -8,5 +8,5 @@ * disclosure or distribution of this material and related documentation * without an express license agreement from NVIDIA CORPORATION or * its affiliates is strictly prohibited. - */html:not([data-theme]),html[data-theme=light]{--nv-color-green:#76b900;--nv-color-green-2:#004831;--pst-color-background:#fff;--pst-color-on-background:#fff;--pst-color-shadow:#ccc;--pst-color-heading:#000;--pst-color-text-base:#1a1a1a;--pst-color-text-muted:#666;--pst-color-surface:#f7f7f7;--pst-color-on-surface:#333;--pst-color-primary:var(--nv-color-green-2);--pst-color-table-row-hover-bg:var(--nv-color-green);--pst-color-link:var(--pst-color-text-base);--pst-color-link-hover:var(--pst-color-text-base);--pst-color-inline-code:var(--pst-color-primary);--pst-color-inline-code-links:var(--pst-color-primary);--pst-color-secondary:var(--pst-color-primary);--pst-color-secondary-bg:var(--nv-color-green);--pst-color-accent:var(--nv-color-green)}html[data-theme=dark]{--nv-color-green:#76b900;--nv-color-green-2:#004831;--pst-color-background:#111;--pst-color-on-background:#000;--pst-color-shadow:#000;--pst-color-heading:#fff;--pst-color-text-base:#eee;--pst-color-text-muted:#999;--pst-color-surface:#1a1a1a;--pst-color-on-surface:#ddd;--pst-color-primary:var(--nv-color-green);--pst-color-table-row-hover-bg:var(--nv-color-green-2);--pst-color-link:var(--pst-color-text-base);--pst-color-link-hover:var(--pst-color-text-base);--pst-color-inline-code:var(--pst-color-primary);--pst-color-inline-code-links:var(--pst-color-primary);--pst-color-secondary:var(--pst-color-primary);--pst-color-secondary-bg:var(--nv-color-green-2);--pst-color-accent:var(--nv-color-green)}@supports not (font-variation-settings:normal){@font-face{font-family:NVIDIA;font-style:normal;font-weight:300;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff2) format("woff2")}@font-face{font-family:NVIDIA;font-style:normal;font-weight:400;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff2) format("woff2")}@font-face{font-family:NVIDIA;font-style:normal;font-weight:500;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff2) format("woff2")}@font-face{font-family:NVIDIA;font-style:normal;font-weight:700;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff2) format("woff2")}}@supports(font-variation-settings:normal){@font-face{font-family:NVIDIA;font-stretch:25% 151%;font-style:normal;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2) format("woff2 supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2) format("woff2-variations")}@font-face{font-family:NVIDIA;font-stretch:25% 151%;font-style:italic;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2) format("woff2 supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2) format("woff2-variations")}}@supports not (font-variation-settings:normal){@font-face{font-family:RobotoMono;font-style:normal;font-weight:300;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Light.ttf) format("truetype")}@font-face{font-family:RobotoMono;font-style:normal;font-weight:400;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Regular.ttf) format("truetype")}@font-face{font-family:RobotoMono;font-style:normal;font-weight:500;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Medium.ttf) format("truetype")}@font-face{font-family:RobotoMono;font-style:normal;font-weight:700;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Bold.ttf) format("truetype")}}@supports(font-variation-settings:normal){@font-face{font-family:RobotoMono;font-stretch:25% 151%;font-style:normal;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf) format("truetype supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf) format("truetype-variations")}@font-face{font-family:RobotoMono;font-stretch:25% 151%;font-style:italic;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf) format("truetype supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf) format("truetype-variations")}}html{--pst-font-size-h1:2.25rem;--pst-font-size-h2:1.75rem;--pst-font-size-h3:1.5rem;--pst-font-size-h4:1.25rem;--pst-font-size-h5:1.125rem;--pst-font-size-h6:1rem;--pst-font-weight-heading:700;--pst-font-family-base:NVIDIA,Arial,Helvetica,Sans-Serif;--pst-font-family-heading:var(--pst-font-family-base);--pst-font-family-monospace:RobotoMono,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;--pst-header-height:3rem;overflow-y:scroll}.bd-article h1,.bd-article h2,.bd-article h3,.bd-article h4,.bd-article h5,.bd-article h6{color:var(--pst-color-heading);word-wrap:break-word}.bd-article a{text-decoration-color:var(--nv-color-green)}.bd-article a.headerlink{color:var(--pst-color-primary)}pre{color:unset}html[data-theme] .highlight span.linenos{padding-left:0;padding-right:1rem}#pst-back-to-top:hover{background-color:var(--pst-color-secondary);color:var(--pst-color-secondary-text)}.footer-brand img{height:var(--pst-header-height)}.bd-footer{color:var(--pst-color-text-muted);font-size:var(--pst-sidebar-font-size-mobile)}@media(min-width:960px){.bd-footer{font-size:var(--pst-sidebar-font-size)}}.bd-footer a{color:var(--pst-color-text-muted);text-decoration:none}.bd-footer a:hover{color:var(--pst-color-link-hover);text-decoration:underline;text-decoration-skip-ink:none;text-decoration-thickness:max(3px,.1875rem,.12em)}.bd-header .navbar-header-items__start{flex-basis:fit-content}.bd-header .navbar-header-items__end{flex-grow:1}@media(min-width:960px){.bd-sidebar-primary .navbar-brand{display:none}}@media(max-width:960px){.bd-sidebar-primary .navbar-brand{flex-direction:column;height:unset;max-height:unset}.bd-sidebar-primary .navbar-brand img{display:unset}}.bd-sidebar-primary .sidebar-header-items .sidebar-header-items__end{margin:auto}.bd-sidebar-primary.hide-on-wide{display:inherit}.navbar-brand{font-weight:700;padding:0}@media(max-width:960px){.navbar-brand img{display:none}}.prev-next-area a .prev-next-info{max-width:100%}.bd-sidebar-primary div#rtd-footer-container:empty,.search-button-field span{display:none}.bd-sidebar-primary{flex-basis:20%;min-width:var(--pst-sidebar-secondary)}.bd-sidebar-secondary{flex-basis:25%;min-width:var(--pst-sidebar-secondary)}.bd-sidebar .nav-item__separator{display:none}.toc-entry a.nav-link.active,nav.bd-links .current>a{box-shadow:inset max(3px,.1875rem,.12em) 0 0 var(--nv-color-green)}.cpp .sig-name-template,.py .sig-param .n~.n,.sig{color:var(--pst-color-text-muted);font-weight:400}.cpp .sig-param.n,.py .sig-param .n{color:var(--pst-color-text-base);font-weight:700}dl[class]:not(.option-list,.field-list,.footnote,.glossary,.simple) dl[class]:not(.option-list,.field-list,.footnote,.glossary,.simple){margin-bottom:1.8rem}.sig{background-color:var(--pst-color-surface);border:1px solid var(--pst-color-on-surface);border-radius:.25rem;line-height:1.25rem;margin-bottom:.5rem;padding:.1rem .25rem}.sig .k{font-style:italic}.sig.c .k,.sig.c .kt,.sig.c .m,.sig.c .s,.sig.c .sc,.sig.cpp .k,.sig.cpp .kt,.sig.cpp .m,.sig.cpp .s,.sig.cpp .sc{color:var(--pst-color-text-muted)}.sig-name{font-size:1rem}.sig-param,.sig-param.n{font-style:normal}.caption-number:after{content:": ";margin-left:-.25rem}ol.arabic{list-style:decimal}ol.arabic ol{list-style:lower-alpha}ol.arabic ol ol{list-style:lower-roman}ol.arabic li::marker{font-weight:700} + */html:not([data-theme]),html[data-theme=light]{--nv-color-green:#76b900;--nv-color-green-2:#004831;--pst-color-background:#fff;--pst-color-on-background:#fff;--pst-color-shadow:#ccc;--pst-color-heading:#000;--pst-color-text-base:#1a1a1a;--pst-color-text-muted:#666;--pst-color-surface:#f7f7f7;--pst-color-on-surface:#333;--pst-color-primary:var(--nv-color-green-2);--pst-color-table-row-hover-bg:var(--nv-color-green);--pst-color-link:var(--pst-color-text-base);--pst-color-link-hover:var(--pst-color-text-base);--pst-color-inline-code:var(--pst-color-primary);--pst-color-inline-code-links:var(--pst-color-primary);--pst-color-secondary:var(--pst-color-primary);--pst-color-secondary-bg:var(--nv-color-green);--pst-color-accent:var(--nv-color-green)}html[data-theme=dark]{--nv-color-green:#76b900;--nv-color-green-2:#004831;--pst-color-background:#111;--pst-color-on-background:#000;--pst-color-shadow:#000;--pst-color-heading:#fff;--pst-color-text-base:#eee;--pst-color-text-muted:#999;--pst-color-surface:#1a1a1a;--pst-color-on-surface:#ddd;--pst-color-primary:var(--nv-color-green);--pst-color-table-row-hover-bg:var(--nv-color-green-2);--pst-color-link:var(--pst-color-text-base);--pst-color-link-hover:var(--pst-color-text-base);--pst-color-inline-code:var(--pst-color-primary);--pst-color-inline-code-links:var(--pst-color-primary);--pst-color-secondary:var(--pst-color-primary);--pst-color-secondary-bg:var(--nv-color-green-2);--pst-color-accent:var(--nv-color-green)}@supports not (font-variation-settings:normal){@font-face{font-family:NVIDIA;font-style:normal;font-weight:300;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff2) format("woff2")}@font-face{font-family:NVIDIA;font-style:normal;font-weight:400;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff2) format("woff2")}@font-face{font-family:NVIDIA;font-style:normal;font-weight:500;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff2) format("woff2")}@font-face{font-family:NVIDIA;font-style:normal;font-weight:700;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff) format("woff"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff2) format("woff2")}}@supports(font-variation-settings:normal){@font-face{font-family:NVIDIA;font-stretch:25% 151%;font-style:normal;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2) format("woff2 supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2) format("woff2-variations")}@font-face{font-family:NVIDIA;font-stretch:25% 151%;font-style:italic;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2) format("woff2 supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2) format("woff2-variations")}}@supports not (font-variation-settings:normal){@font-face{font-family:RobotoMono;font-style:normal;font-weight:300;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Light.ttf) format("truetype")}@font-face{font-family:RobotoMono;font-style:normal;font-weight:400;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Regular.ttf) format("truetype")}@font-face{font-family:RobotoMono;font-style:normal;font-weight:500;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Medium.ttf) format("truetype")}@font-face{font-family:RobotoMono;font-style:normal;font-weight:700;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Bold.ttf) format("truetype")}}@supports(font-variation-settings:normal){@font-face{font-family:RobotoMono;font-stretch:25% 151%;font-style:normal;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf) format("truetype supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf) format("truetype-variations")}@font-face{font-family:RobotoMono;font-stretch:25% 151%;font-style:italic;font-weight:100 1000;src:url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf) format("truetype supports variations"),url(https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf) format("truetype-variations")}}html{--pst-font-size-h1:2.25rem;--pst-font-size-h2:1.75rem;--pst-font-size-h3:1.5rem;--pst-font-size-h4:1.25rem;--pst-font-size-h5:1.125rem;--pst-font-size-h6:1rem;--pst-font-weight-heading:700;--pst-font-family-base:NVIDIA,Arial,Helvetica,Sans-Serif;--pst-font-family-heading:var(--pst-font-family-base);--pst-font-family-monospace:RobotoMono,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;--pst-header-height:3rem;overflow-y:scroll}.bd-article h1,.bd-article h2,.bd-article h3,.bd-article h4,.bd-article h5,.bd-article h6{color:var(--pst-color-heading);word-wrap:break-word}.bd-article a{text-decoration-color:var(--nv-color-green)}.bd-article a.headerlink{color:var(--pst-color-primary)}pre{color:unset}html[data-theme] .highlight span.linenos{padding-left:0;padding-right:1rem}#pst-back-to-top:hover{background-color:var(--pst-color-secondary);color:var(--pst-color-secondary-text)}abbr[title]{text-decoration:none}.footer-brand img{height:var(--pst-header-height)}.bd-footer{color:var(--pst-color-text-muted);font-size:var(--pst-sidebar-font-size-mobile)}@media(min-width:960px){.bd-footer{font-size:var(--pst-sidebar-font-size)}}.bd-footer a{color:var(--pst-color-text-muted);text-decoration:none}.bd-footer a:hover{color:var(--pst-color-link-hover);text-decoration:underline;text-decoration-skip-ink:none;text-decoration-thickness:max(3px,.1875rem,.12em)}.bd-header .navbar-header-items__start{flex-basis:fit-content}.bd-header .navbar-header-items__end{flex-grow:1}@media(min-width:960px){.bd-sidebar-primary .navbar-brand{display:none}}@media(max-width:960px){.bd-sidebar-primary .navbar-brand{flex-direction:column;height:unset;max-height:unset}.bd-sidebar-primary .navbar-brand img{display:unset;height:auto}}.bd-sidebar-primary .sidebar-header-items .sidebar-header-items__end{margin:auto}.bd-sidebar-primary.hide-on-wide{display:inherit}.navbar-brand{font-weight:700;padding:0}@media(max-width:960px){.navbar-brand img{display:none}}.prev-next-area a .prev-next-info{max-width:100%}.bd-sidebar-primary div#rtd-footer-container:empty,.search-button-field span{display:none}.bd-sidebar-primary{flex-basis:20%;min-width:var(--pst-sidebar-secondary)}.bd-sidebar-secondary{flex-basis:25%;min-width:var(--pst-sidebar-secondary)}.bd-sidebar .nav-item__separator{display:none}.toc-entry a.nav-link.active,nav.bd-links .current>a{box-shadow:inset max(3px,.1875rem,.12em) 0 0 var(--nv-color-green)}.cpp .sig-name-template,.py .sig-param .n~.n,.sig{color:var(--pst-color-text-muted);font-weight:400}.cpp .sig-param.n,.py .sig-param .n{color:var(--pst-color-text-base);font-weight:700}dl[class]:not(.option-list,.field-list,.footnote,.glossary,.simple) dl[class]:not(.option-list,.field-list,.footnote,.glossary,.simple){margin-bottom:1.8rem}.sig{background-color:var(--pst-color-surface);border:1px solid var(--pst-color-on-surface);border-radius:.25rem;line-height:1.25rem;margin-bottom:.5rem;padding:.1rem .25rem}.sig .k{font-style:italic}.sig.c .k,.sig.c .kt,.sig.c .m,.sig.c .s,.sig.c .sc,.sig.cpp .k,.sig.cpp .kt,.sig.cpp .m,.sig.cpp .s,.sig.cpp .sc{color:var(--pst-color-text-muted)}.sig-name{font-size:1rem}.sig-param,.sig-param.n{font-style:normal}.caption-number:after{content:": ";margin-left:-.25rem}ol.arabic{list-style:decimal}ol.arabic ol{list-style:lower-alpha}ol.arabic ol ol{list-style:lower-roman}ol.arabic li::marker{font-weight:700} /*# sourceMappingURL=nvidia-sphinx-theme.css.map*/ \ No newline at end of file diff --git a/latest/_static/styles/nvidia-sphinx-theme.css.map b/latest/_static/styles/nvidia-sphinx-theme.css.map index e1c36d16bb..2cf5cd99be 100644 --- a/latest/_static/styles/nvidia-sphinx-theme.css.map +++ b/latest/_static/styles/nvidia-sphinx-theme.css.map @@ -1 +1 @@ -{"version":3,"file":"styles/nvidia-sphinx-theme.css","mappings":"AAAA;;;;;;;;;;EAAA,CCyFE,8CAVI,gsBAUJ,sBAVI,+rBCpEN,+CACE,WACE,mBAIA,kBADA,gBAFA,yQAGA,CAEF,WACE,mBAIA,kBADA,gBAFA,yQAGA,CAEF,WACE,mBAIA,kBADA,gBAFA,yQAGA,CAEF,WACE,mBAIA,kBADA,gBAFA,yQAGA,EAKJ,0CACE,WACE,mBAIA,sBACA,kBAFA,qBAFA,0TAIA,CAEF,WACE,mBAIA,qBAAqB,CACrB,iBAAgB,CAFhB,oBAAoB,CAFpB,wUAIiB,CAAjB,CAKJ,+CACE,WACE,uBAGA,kBADA,gBADA,yIAEA,CAEF,WACE,uBAGA,kBADA,gBADA,2IAEA,CAEF,WACE,uBAGA,kBADA,gBADA,0IAEA,CAEF,WACE,uBAGA,kBADA,gBADA,wIAEA,EAKJ,0CACE,WACE,uBAIA,sBACA,kBAFA,qBAFA,wTAIA,CAEF,WACE,uBAIA,qBAAqB,CACrB,iBAAgB,CAFhB,oBAAoB,CAFpB,sUAIiB,CAAjB,CAIJ,KACE,0BAA2B,CAC3B,0BAA2B,CAC3B,yBAA0B,CAC1B,0BAA2B,CAC3B,2BAA4B,CAC5B,uBAAwB,CAExB,6BAA8B,CAE9B,wDAAyD,CACzD,qDAAsD,CACtD,yHAAyH,CC9GzH,wBAAwB,CCAxB,iBF8G0H,CE1G1H,0FAOE,+BAGA,qBAIF,cACE,4CAEA,yBACE,+BAKN,IACE,YAIF,yCACE,eACA,mBAIF,uBACE,4CACA,sCC1CF,kBAEE,gCAGF,WAEE,kCACA,8CAEA,wBALF,WAMI,wCAGF,aACE,kCACA,qBAEA,mBACE,kCACA,0BACA,8BACA,kDCrBJ,uCAEE,uBAEF,qCAEE,YCHA,wBAHF,kCAKI,cAGF,wBARF,kCAUI,sBACA,aACA,iBAEA,sCAEE,eAKN,qEAEE,YAGF,iCAEE,gBC7BJ,cAKE,gBAHA,SAGA,CAEA,wBACE,kBAEE,cCVN,kCACE,eCAA,6EAEE,aCHJ,oBAEE,eACA,uCAGF,sBAEE,eACA,uCAGF,iCAEE,aAGF,qDAGE,mECnBF,kDAEE,kCADA,eACA,CAIF,oCAEE,iCADA,eACA,CAMF,wIAEE,qBAGF,KAUE,0CACA,6CACA,qBARA,oBAGA,oBAMA,qBAGA,QACE,kBAMA,kHAKE,kCAKN,UAEE,eAYA,wBACE,kBCnEJ,sBACI,aACA,oBCFJ,UACE,mBACA,aACE,uBACA,gBACE,uBAIJ,qBACE","sources":["webpack:///./src/nvidia_sphinx_theme/assets/styles/index.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/variables/_color.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/variables/_fonts.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/variables/_layout.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/base/_base.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/sections/_footer.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/sections/_header.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/sections/_sidebar-primary.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/header/_header-logo.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/_prev-next.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/_search.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/_sidebar.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/content/_api.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/content/_figures.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/content/_lists.scss"],"sourcesContent":["/*!\n * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n * SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n *\n * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n * property and proprietary rights in and to this material, related\n * documentation and any modifications thereto. Any use, reproduction,\n * disclosure or distribution of this material and related documentation\n * without an express license agreement from NVIDIA CORPORATION or\n * its affiliates is strictly prohibited.\n */\n\n@import \"variables/color\";\n@import \"variables/fonts\";\n@import \"variables/layout\";\n\n@import \"base/base\";\n\n@import \"sections/footer\";\n@import \"sections/header\";\n@import \"sections/sidebar-primary\";\n\n@import \"components/header/header-logo\";\n@import \"components/prev-next\";\n@import \"components/readthedocs-switcher\";\n@import \"components/search\";\n@import \"components/sidebar\";\n\n@import \"content/api\";\n@import \"content/figures\";\n@import \"content/lists\";\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n$nv-colors: (\n \"green\": #76b900,\n \"green-2\": #004831,\n);\n\n$pst-colors: (\n \"background\": (\n \"light\": #ffffff,\n \"dark\": #111111,\n ),\n \"on-background\": (\n \"light\": #ffffff,\n \"dark\": #000000,\n ),\n \"shadow\": (\n \"light\": #cccccc,\n \"dark\": #000000,\n ),\n \"heading\": (\n \"light\": #000000,\n \"dark\": #ffffff,\n ),\n \"text-base\": (\n \"light\": #1a1a1a,\n \"dark\": #eeeeee,\n ),\n \"text-muted\": (\n \"light\": #666666,\n \"dark\": #999999,\n ),\n \"surface\": (\n \"light\": #f7f7f7,\n \"dark\": #1a1a1a,\n ),\n \"on-surface\": (\n \"light\": #333333,\n \"dark\": #dddddd,\n ),\n \"primary\": (\n \"light\": var(--nv-color-green-2),\n \"dark\": var(--nv-color-green),\n ),\n \"table-row-hover-bg\": (\n \"light\": var(--nv-color-green),\n \"dark\": var(--nv-color-green-2),\n ),\n \"link\": var(--pst-color-text-base),\n \"link-hover\": var(--pst-color-text-base),\n \"inline-code\": var(--pst-color-primary),\n \"inline-code-links\": var(--pst-color-primary),\n \"secondary\": var(--pst-color-primary),\n \"secondary-bg\": (\n \"light\": var(--nv-color-green),\n \"dark\": var(--nv-color-green-2),\n ),\n \"accent\": var(--nv-color-green),\n);\n\n$prefix-map: (\n \"nv\": $nv-colors,\n \"pst\": $pst-colors,\n);\n\n@mixin theme-colors($mode) {\n @each $prefix, $colors in $prefix-map {\n @each $name, $value in $colors {\n @if type-of($value) == map {\n $value: map-get($value, $mode)\n }\n --#{$prefix}-color-#{$name}: #{$value};\n }\n }\n}\n\nhtml:not([data-theme]) {\n @include theme-colors(light);\n}\n\n@each $mode in (light, dark) {\n html[data-theme='#{$mode}'] {\n @include theme-colors($mode);\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// Set up for old browsers\n@supports not (font-variation-settings: normal) {\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff2\") format(\"woff2\");\n font-weight: 300;\n font-style: normal;\n }\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff2\") format(\"woff2\");\n font-weight: 400;\n font-style: normal;\n }\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff2\") format(\"woff2\");\n font-weight: 500;\n font-style: normal;\n }\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff2\") format(\"woff2\");\n font-weight: 700;\n font-style: normal;\n }\n}\n\n// Set up for modern browsers, all weights\n@supports (font-variation-settings: normal) {\n @font-face {\n font-family: 'NVIDIA';\n src: url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2') format('woff2 supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2') format('woff2-variations');\n font-weight: 100 1000;\n font-stretch: 25% 151%;\n font-style: normal;\n }\n @font-face{\n font-family:'NVIDIA';\n src:url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2') format('woff2 supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2') format('woff2-variations');\n font-weight:100 1000;\n font-stretch:25% 151%;\n font-style:italic;\n }\n}\n\n/* Set up for old browsers*/\n@supports not (font-variation-settings: normal) {\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Light.ttf\") format(\"truetype\");\n font-weight: 300;\n font-style: normal;\n }\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Regular.ttf\") format(\"truetype\");\n font-weight: 400;\n font-style: normal;\n }\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Medium.ttf\") format(\"truetype\");\n font-weight: 500;\n font-style: normal;\n }\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Bold.ttf\") format(\"truetype\");\n font-weight: 700;\n font-style: normal;\n }\n}\n\n/* Set up for modern browsers, all weights */\n@supports (font-variation-settings: normal) {\n @font-face {\n font-family: 'RobotoMono';\n src: url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf') format('truetype supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf') format('truetype-variations');\n font-weight: 100 1000;\n font-stretch: 25% 151%;\n font-style: normal;\n }\n @font-face{\n font-family: 'RobotoMono';\n src: url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf') format('truetype supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf') format('truetype-variations');\n font-weight:100 1000;\n font-stretch:25% 151%;\n font-style:italic;\n }\n}\n\nhtml {\n --pst-font-size-h1: 2.25rem;\n --pst-font-size-h2: 1.75rem;\n --pst-font-size-h3: 1.5rem;\n --pst-font-size-h4: 1.25rem;\n --pst-font-size-h5: 1.125rem;\n --pst-font-size-h6: 1rem;\n\n --pst-font-weight-heading: 700;\n\n --pst-font-family-base: NVIDIA,Arial,Helvetica,Sans-Serif;\n --pst-font-family-heading: var(--pst-font-family-base);\n --pst-font-family-monospace: RobotoMono,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\nhtml {\n // reduce header height to match existing\n --pst-header-height: 3rem;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\nhtml {\n // prevent shifting the content between pages when one would not normally need a scrollbar\n overflow-y: scroll;\n}\n\n.bd-article {\n h1,\n h2,\n h3,\n h4,\n h5,\n h6 {\n // patch for variable name in base theme\n color: var(--pst-color-heading);\n\n // ensure long headings are visible\n word-wrap: break-word;\n }\n\n // change only links in .bd-article to avoid affecting nav, prev/next page links\n a {\n text-decoration-color: var(--nv-color-green); // same for light/dark mode\n\n &.headerlink {\n color: var(--pst-color-primary);\n }\n }\n}\n\npre {\n color: unset; // let pygments set the font colors (applies to text not in a span with a class)\n}\n\n// ensure the padding overrides pygments\nhtml[data-theme] .highlight span.linenos {\n padding-left: 0;\n padding-right: 1rem;\n}\n\n// maintain theme color on hover\n#pst-back-to-top:hover {\n background-color: var(--pst-color-secondary);\n color: var(--pst-color-secondary-text);\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.footer-brand img {\n // match header logo size\n height: var(--pst-header-height);\n}\n\n.bd-footer {\n // match sidebar style\n color: var(--pst-color-text-muted);\n font-size: var(--pst-sidebar-font-size-mobile);\n\n @media (min-width: 960px) {\n font-size: var(--pst-sidebar-font-size);\n }\n\n a {\n color: var(--pst-color-text-muted);\n text-decoration: none;\n\n &:hover {\n color: var(--pst-color-link-hover);\n text-decoration: underline;\n text-decoration-skip-ink: none;\n text-decoration-thickness: max(3px,.1875rem,.12em);\n }\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.bd-header {\n .navbar-header-items__start {\n // prevent long project titles overlapping extra links\n flex-basis: fit-content;\n }\n .navbar-header-items__end {\n // expand to fill header\n flex-grow: 1;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.bd-sidebar-primary {\n .navbar-brand {\n // style sidebar branding\n\n @media (min-width: 960px) {\n // hide logo in sidebar on wide screens\n display: none;\n }\n\n @media (max-width: 960px) {\n // display a larger logo and title in the sidebar on mobile\n flex-direction: column;\n height: unset;\n max-height: unset;\n\n img {\n // show the logo mobile\n display: unset;\n }\n }\n }\n\n .sidebar-header-items .sidebar-header-items__end {\n // center align theme switcher in mobile sidebar\n margin: auto;\n }\n\n &.hide-on-wide {\n // prevent the sidebar being hidden\n display: inherit;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.navbar-brand {\n // use the margin already in the logo\n padding: 0;\n\n // make project title bold\n font-weight: bold;\n\n @media (max-width: 960px) {\n img {\n // hide logo when primary sidebar disappears\n display: none;\n }\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.prev-next-area a .prev-next-info {\n max-width: 100%;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.search-button-field {\n span {\n // hide text and keyboard icons\n display: none;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.bd-sidebar-primary {\n // make the two sidebars the same size\n flex-basis: 20%;\n min-width: var(--pst-sidebar-secondary);\n}\n\n.bd-sidebar-secondary {\n // make the two sidebars the same size\n flex-basis: 25%;\n min-width: var(--pst-sidebar-secondary);\n}\n\n.bd-sidebar .nav-item__separator {\n // hide sidebar header item separators\n display: none;\n}\n\nnav.bd-links .current > a,\n.toc-entry a.nav-link.active {\n // use nvgreen to highlight active sections\n box-shadow: inset max(3px,.1875rem,.12em) 0 0 var(--nv-color-green);\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// styling that should be applied to the whole signature\n%api-signature-style {\n font-weight: normal;\n color: var(--pst-color-text-muted);\n}\n\n// styling to apply to arguments only\n%api-signature-argument-style {\n font-weight: bold;\n color: var(--pst-color-text-base);\n}\n\n// api selector, see: https://github.com/pydata/pydata-sphinx-theme/blob/v0.15.4/src/pydata_sphinx_theme/assets/styles/content/_api.scss#L136\n$api-selector: 'dl[class]:not(.option-list, .field-list, .footnote, .glossary, .simple)';\n\n#{$api-selector} #{$api-selector} {\n // for nested api objects, set the margin to match paragraph margin (1.15rem) + line height space (0.65rem)\n margin-bottom: 1.8rem;\n}\n\n.sig {\n @extend %api-signature-style;\n\n // match heading line height\n line-height: 1.25rem;\n\n // margin required on api signatures now that they match inline code (see below)\n margin-bottom: 0.5rem;\n\n // match inline code\n background-color: var(--pst-color-surface);\n border: 1px solid var(--pst-color-on-surface);\n border-radius: 0.25rem;\n padding: 0.1rem 0.25rem;\n\n // make keywords in signatures italicized\n .k {\n font-style: italic;\n }\n\n // override un-muted coloring of some elements by pydata\n &.c,\n &.cpp {\n .k,\n .kt,\n .m,\n .s,\n .sc {\n color: var(--pst-color-text-muted);\n }\n }\n}\n\n.sig-name {\n // override basic.css which increases the size of object names\n font-size: 1rem;\n}\n\n.cpp .sig-name-template {\n // use default style on template parameters\n @extend %api-signature-style;\n}\n\n.sig-param {\n font-style: normal;\n\n // override basic.css which will make parameter names italicized\n &.n {\n font-style: normal;\n }\n}\n\n// set parameter names bold and unmuted color\n.cpp .sig-param.n {\n @extend %api-signature-argument-style;\n}\n\n// python signatures do not have a nice structure so we set all names as arguments, and then undo some\n.py .sig-param .n {\n @extend %api-signature-argument-style;\n\n // revert styling on sibling names\n ~ .n {\n @extend %api-signature-style;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// append ':' to figure captions numbers\n.caption-number:after {\n content: ': ';\n margin-left: -0.25rem;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// set ordered list styles as 1. / a. / i.\nol.arabic {\n list-style: decimal;\n ol {\n list-style: lower-alpha;\n ol {\n list-style: lower-roman;\n }\n }\n\n li::marker {\n font-weight: 700;\n }\n}\n"],"names":[],"sourceRoot":""} \ No newline at end of file +{"version":3,"file":"styles/nvidia-sphinx-theme.css","mappings":"AAAA;;;;;;;;;;EAAA,CC2FE,8CAVI,gsBAUJ,sBAVI,+rBCtEN,+CACE,WACE,mBAIA,kBADA,gBAFA,yQAGA,CAEF,WACE,mBAIA,kBADA,gBAFA,yQAGA,CAEF,WACE,mBAIA,kBADA,gBAFA,yQAGA,CAEF,WACE,mBAIA,kBADA,gBAFA,yQAGA,EAKJ,0CACE,WACE,mBAIA,sBACA,kBAFA,qBAFA,0TAIA,CAEF,WACE,mBAIA,qBAAqB,CACrB,iBAAgB,CAFhB,oBAAoB,CAFpB,wUAIiB,CAAjB,CAKJ,+CACE,WACE,uBAGA,kBADA,gBADA,yIAEA,CAEF,WACE,uBAGA,kBADA,gBADA,2IAEA,CAEF,WACE,uBAGA,kBADA,gBADA,0IAEA,CAEF,WACE,uBAGA,kBADA,gBADA,wIAEA,EAKJ,0CACE,WACE,uBAIA,sBACA,kBAFA,qBAFA,wTAIA,CAEF,WACE,uBAIA,qBAAqB,CACrB,iBAAgB,CAFhB,oBAAoB,CAFpB,sUAIiB,CAAjB,CAIJ,KACE,0BAA2B,CAC3B,0BAA2B,CAC3B,yBAA0B,CAC1B,0BAA2B,CAC3B,2BAA4B,CAC5B,uBAAwB,CAExB,6BAA8B,CAE9B,wDAAyD,CACzD,qDAAsD,CACtD,yHAAyH,CC9GzH,wBAAwB,CCAxB,iBF8G0H,CE1G1H,0FAOE,+BAGA,qBAIF,cACE,4CAEA,yBACE,+BAKN,IACE,YAIF,yCACE,eACA,mBAIF,uBACE,4CACA,sCAIF,YACE,qBC/CF,kBAEE,gCAGF,WAEE,kCACA,8CAEA,wBALF,WAMI,wCAGF,aACE,kCACA,qBAEA,mBACE,kCACA,0BACA,8BACA,kDCrBJ,uCAEE,uBAEF,qCAEE,YCHA,wBAHF,kCAKI,cAGF,wBARF,kCAUI,sBACA,aACA,iBAEA,sCAEE,cAGA,aAKN,qEAEE,YAGF,iCAEE,gBChCJ,cAKE,gBAHA,SAGA,CAEA,wBACE,kBAEE,cCVN,kCACE,eCAA,6EAEE,aCHJ,oBAEE,eACA,uCAGF,sBAEE,eACA,uCAGF,iCAEE,aAGF,qDAGE,mECnBF,kDAEE,kCADA,eACA,CAIF,oCAEE,iCADA,eACA,CAMF,wIAEE,qBAGF,KAUE,0CACA,6CACA,qBARA,oBAGA,oBAMA,qBAGA,QACE,kBAMA,kHAKE,kCAKN,UAEE,eAYA,wBACE,kBCnEJ,sBACI,aACA,oBCFJ,UACE,mBACA,aACE,uBACA,gBACE,uBAIJ,qBACE","sources":["webpack:///./src/nvidia_sphinx_theme/assets/styles/index.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/variables/_color.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/variables/_fonts.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/variables/_layout.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/base/_base.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/sections/_footer.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/sections/_header.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/sections/_sidebar-primary.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/header/_header-logo.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/_prev-next.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/_search.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/components/_sidebar.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/content/_api.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/content/_figures.scss","webpack:///./src/nvidia_sphinx_theme/assets/styles/content/_lists.scss"],"sourcesContent":["/*!\n * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n * SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n *\n * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n * property and proprietary rights in and to this material, related\n * documentation and any modifications thereto. Any use, reproduction,\n * disclosure or distribution of this material and related documentation\n * without an express license agreement from NVIDIA CORPORATION or\n * its affiliates is strictly prohibited.\n */\n\n@use \"variables/color\";\n@use \"variables/fonts\";\n@use \"variables/layout\";\n\n@use \"base/base\";\n\n@use \"sections/footer\";\n@use \"sections/header\";\n@use \"sections/sidebar-primary\";\n\n@use \"components/header/header-logo\";\n@use \"components/prev-next\";\n@use \"components/readthedocs-switcher\";\n@use \"components/search\";\n@use \"components/sidebar\";\n\n@use \"content/api\";\n@use \"content/figures\";\n@use \"content/lists\";\n","@use \"sass:map\";\n@use \"sass:meta\";\n// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n$nv-colors: (\n \"green\": #76b900,\n \"green-2\": #004831,\n);\n\n$pst-colors: (\n \"background\": (\n \"light\": #ffffff,\n \"dark\": #111111,\n ),\n \"on-background\": (\n \"light\": #ffffff,\n \"dark\": #000000,\n ),\n \"shadow\": (\n \"light\": #cccccc,\n \"dark\": #000000,\n ),\n \"heading\": (\n \"light\": #000000,\n \"dark\": #ffffff,\n ),\n \"text-base\": (\n \"light\": #1a1a1a,\n \"dark\": #eeeeee,\n ),\n \"text-muted\": (\n \"light\": #666666,\n \"dark\": #999999,\n ),\n \"surface\": (\n \"light\": #f7f7f7,\n \"dark\": #1a1a1a,\n ),\n \"on-surface\": (\n \"light\": #333333,\n \"dark\": #dddddd,\n ),\n \"primary\": (\n \"light\": var(--nv-color-green-2),\n \"dark\": var(--nv-color-green),\n ),\n \"table-row-hover-bg\": (\n \"light\": var(--nv-color-green),\n \"dark\": var(--nv-color-green-2),\n ),\n \"link\": var(--pst-color-text-base),\n \"link-hover\": var(--pst-color-text-base),\n \"inline-code\": var(--pst-color-primary),\n \"inline-code-links\": var(--pst-color-primary),\n \"secondary\": var(--pst-color-primary),\n \"secondary-bg\": (\n \"light\": var(--nv-color-green),\n \"dark\": var(--nv-color-green-2),\n ),\n \"accent\": var(--nv-color-green),\n);\n\n$prefix-map: (\n \"nv\": $nv-colors,\n \"pst\": $pst-colors,\n);\n\n@mixin theme-colors($mode) {\n @each $prefix, $colors in $prefix-map {\n @each $name, $value in $colors {\n @if meta.type-of($value) == map {\n $value: map.get($value, $mode)\n }\n --#{$prefix}-color-#{$name}: #{$value};\n }\n }\n}\n\nhtml:not([data-theme]) {\n @include theme-colors(light);\n}\n\n@each $mode in (light, dark) {\n html[data-theme='#{$mode}'] {\n @include theme-colors($mode);\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// Set up for old browsers\n@supports not (font-variation-settings: normal) {\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Lt.woff2\") format(\"woff2\");\n font-weight: 300;\n font-style: normal;\n }\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Rg.woff2\") format(\"woff2\");\n font-weight: 400;\n font-style: normal;\n }\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Md.woff2\") format(\"woff2\");\n font-weight: 500;\n font-style: normal;\n }\n @font-face {\n font-family: \"NVIDIA\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff\") format(\"woff\"),\n url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/NVIDIASans_W_Bd.woff2\") format(\"woff2\");\n font-weight: 700;\n font-style: normal;\n }\n}\n\n// Set up for modern browsers, all weights\n@supports (font-variation-settings: normal) {\n @font-face {\n font-family: 'NVIDIA';\n src: url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2') format('woff2 supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_W_Wght.woff2') format('woff2-variations');\n font-weight: 100 1000;\n font-stretch: 25% 151%;\n font-style: normal;\n }\n @font-face{\n font-family:'NVIDIA';\n src:url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2') format('woff2 supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/nvidia-sans/GLOBAL/var/NVIDIASansVF_Wght_W_Italic.woff2') format('woff2-variations');\n font-weight:100 1000;\n font-stretch:25% 151%;\n font-style:italic;\n }\n}\n\n/* Set up for old browsers*/\n@supports not (font-variation-settings: normal) {\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Light.ttf\") format(\"truetype\");\n font-weight: 300;\n font-style: normal;\n }\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Regular.ttf\") format(\"truetype\");\n font-weight: 400;\n font-style: normal;\n }\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Medium.ttf\") format(\"truetype\");\n font-weight: 500;\n font-style: normal;\n }\n @font-face {\n font-family: \"RobotoMono\";\n src: url(\"https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/static/RobotoMono-Bold.ttf\") format(\"truetype\");\n font-weight: 700;\n font-style: normal;\n }\n}\n\n/* Set up for modern browsers, all weights */\n@supports (font-variation-settings: normal) {\n @font-face {\n font-family: 'RobotoMono';\n src: url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf') format('truetype supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-VariableFont_wght.ttf') format('truetype-variations');\n font-weight: 100 1000;\n font-stretch: 25% 151%;\n font-style: normal;\n }\n @font-face{\n font-family: 'RobotoMono';\n src: url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf') format('truetype supports variations'),\n url('https://images.nvidia.com/etc/designs/nvidiaGDC/clientlibs_base/fonts/Roboto_Mono/RobotoMono-Italic-VariableFont_wght.ttf') format('truetype-variations');\n font-weight:100 1000;\n font-stretch:25% 151%;\n font-style:italic;\n }\n}\n\nhtml {\n --pst-font-size-h1: 2.25rem;\n --pst-font-size-h2: 1.75rem;\n --pst-font-size-h3: 1.5rem;\n --pst-font-size-h4: 1.25rem;\n --pst-font-size-h5: 1.125rem;\n --pst-font-size-h6: 1rem;\n\n --pst-font-weight-heading: 700;\n\n --pst-font-family-base: NVIDIA,Arial,Helvetica,Sans-Serif;\n --pst-font-family-heading: var(--pst-font-family-base);\n --pst-font-family-monospace: RobotoMono,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,Courier,monospace;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\nhtml {\n // reduce header height to match existing\n --pst-header-height: 3rem;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\nhtml {\n // prevent shifting the content between pages when one would not normally need a scrollbar\n overflow-y: scroll;\n}\n\n.bd-article {\n h1,\n h2,\n h3,\n h4,\n h5,\n h6 {\n // patch for variable name in base theme\n color: var(--pst-color-heading);\n\n // ensure long headings are visible\n word-wrap: break-word;\n }\n\n // change only links in .bd-article to avoid affecting nav, prev/next page links\n a {\n text-decoration-color: var(--nv-color-green); // same for light/dark mode\n\n &.headerlink {\n color: var(--pst-color-primary);\n }\n }\n}\n\npre {\n color: unset; // let pygments set the font colors (applies to text not in a span with a class)\n}\n\n// ensure the padding overrides pygments\nhtml[data-theme] .highlight span.linenos {\n padding-left: 0;\n padding-right: 1rem;\n}\n\n// maintain theme color on hover\n#pst-back-to-top:hover {\n background-color: var(--pst-color-secondary);\n color: var(--pst-color-secondary-text);\n}\n\n// fix double underlines on abbreviation elements\nabbr[title] {\n text-decoration: none;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.footer-brand img {\n // match header logo size\n height: var(--pst-header-height);\n}\n\n.bd-footer {\n // match sidebar style\n color: var(--pst-color-text-muted);\n font-size: var(--pst-sidebar-font-size-mobile);\n\n @media (min-width: 960px) {\n font-size: var(--pst-sidebar-font-size);\n }\n\n a {\n color: var(--pst-color-text-muted);\n text-decoration: none;\n\n &:hover {\n color: var(--pst-color-link-hover);\n text-decoration: underline;\n text-decoration-skip-ink: none;\n text-decoration-thickness: max(3px,.1875rem,.12em);\n }\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.bd-header {\n .navbar-header-items__start {\n // prevent long project titles overlapping extra links\n flex-basis: fit-content;\n }\n .navbar-header-items__end {\n // expand to fill header\n flex-grow: 1;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.bd-sidebar-primary {\n .navbar-brand {\n // style sidebar branding\n\n @media (min-width: 960px) {\n // hide logo in sidebar on wide screens\n display: none;\n }\n\n @media (max-width: 960px) {\n // display a larger logo and title in the sidebar on mobile\n flex-direction: column;\n height: unset;\n max-height: unset;\n\n img {\n // show the logo mobile\n display: unset;\n\n // set the height for the logo to ensure safari compatibilty\n height: auto;\n }\n }\n }\n\n .sidebar-header-items .sidebar-header-items__end {\n // center align theme switcher in mobile sidebar\n margin: auto;\n }\n\n &.hide-on-wide {\n // prevent the sidebar being hidden\n display: inherit;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.navbar-brand {\n // use the margin already in the logo\n padding: 0;\n\n // make project title bold\n font-weight: bold;\n\n @media (max-width: 960px) {\n img {\n // hide logo when primary sidebar disappears\n display: none;\n }\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.prev-next-area a .prev-next-info {\n max-width: 100%;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.search-button-field {\n span {\n // hide text and keyboard icons\n display: none;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n.bd-sidebar-primary {\n // make the two sidebars the same size\n flex-basis: 20%;\n min-width: var(--pst-sidebar-secondary);\n}\n\n.bd-sidebar-secondary {\n // make the two sidebars the same size\n flex-basis: 25%;\n min-width: var(--pst-sidebar-secondary);\n}\n\n.bd-sidebar .nav-item__separator {\n // hide sidebar header item separators\n display: none;\n}\n\nnav.bd-links .current > a,\n.toc-entry a.nav-link.active {\n // use nvgreen to highlight active sections\n box-shadow: inset max(3px,.1875rem,.12em) 0 0 var(--nv-color-green);\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// styling that should be applied to the whole signature\n%api-signature-style {\n font-weight: normal;\n color: var(--pst-color-text-muted);\n}\n\n// styling to apply to arguments only\n%api-signature-argument-style {\n font-weight: bold;\n color: var(--pst-color-text-base);\n}\n\n// api selector, see: https://github.com/pydata/pydata-sphinx-theme/blob/v0.15.4/src/pydata_sphinx_theme/assets/styles/content/_api.scss#L136\n$api-selector: 'dl[class]:not(.option-list, .field-list, .footnote, .glossary, .simple)';\n\n#{$api-selector} #{$api-selector} {\n // for nested api objects, set the margin to match paragraph margin (1.15rem) + line height space (0.65rem)\n margin-bottom: 1.8rem;\n}\n\n.sig {\n @extend %api-signature-style;\n\n // match heading line height\n line-height: 1.25rem;\n\n // margin required on api signatures now that they match inline code (see below)\n margin-bottom: 0.5rem;\n\n // match inline code\n background-color: var(--pst-color-surface);\n border: 1px solid var(--pst-color-on-surface);\n border-radius: 0.25rem;\n padding: 0.1rem 0.25rem;\n\n // make keywords in signatures italicized\n .k {\n font-style: italic;\n }\n\n // override un-muted coloring of some elements by pydata\n &.c,\n &.cpp {\n .k,\n .kt,\n .m,\n .s,\n .sc {\n color: var(--pst-color-text-muted);\n }\n }\n}\n\n.sig-name {\n // override basic.css which increases the size of object names\n font-size: 1rem;\n}\n\n.cpp .sig-name-template {\n // use default style on template parameters\n @extend %api-signature-style;\n}\n\n.sig-param {\n font-style: normal;\n\n // override basic.css which will make parameter names italicized\n &.n {\n font-style: normal;\n }\n}\n\n// set parameter names bold and unmuted color\n.cpp .sig-param.n {\n @extend %api-signature-argument-style;\n}\n\n// python signatures do not have a nice structure so we set all names as arguments, and then undo some\n.py .sig-param .n {\n @extend %api-signature-argument-style;\n\n // revert styling on sibling names\n ~ .n {\n @extend %api-signature-style;\n }\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// append ':' to figure captions numbers\n.caption-number:after {\n content: ': ';\n margin-left: -0.25rem;\n}\n","// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\n// SPDX-License-Identifier: LicenseRef-NvidiaProprietary\n//\n// NVIDIA CORPORATION, its affiliates and licensors retain all intellectual\n// property and proprietary rights in and to this material, related\n// documentation and any modifications thereto. Any use, reproduction,\n// disclosure or distribution of this material and related documentation\n// without an express license agreement from NVIDIA CORPORATION or\n// its affiliates is strictly prohibited.\n\n// set ordered list styles as 1. / a. / i.\nol.arabic {\n list-style: decimal;\n ol {\n list-style: lower-alpha;\n ol {\n list-style: lower-roman;\n }\n }\n\n li::marker {\n font-weight: 700;\n }\n}\n"],"names":[],"sourceRoot":""} \ No newline at end of file diff --git a/latest/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.html b/latest/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.html index c68dc9ad00..ed5e6463c6 100644 --- a/latest/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.html +++ b/latest/blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -1054,10 +1062,12 @@ trtllm-bench -m deepseek-ai/DeepSe
+ +
@@ -1079,12 +1089,7 @@ trtllm-bench -m deepseek-ai/DeepSe - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1132,9 +1137,9 @@ trtllm-bench -m deepseek-ai/DeepSe
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/Falcon180B-H200.html b/latest/blogs/Falcon180B-H200.html index 1d45e0944e..72818cc020 100644 --- a/latest/blogs/Falcon180B-H200.html +++ b/latest/blogs/Falcon180B-H200.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -704,10 +712,12 @@ ISL = Input Sequence Length
+ +
@@ -729,12 +739,7 @@ ISL = Input Sequence Length - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -782,9 +787,9 @@ ISL = Input Sequence Length
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/H100vsA100.html b/latest/blogs/H100vsA100.html index 56005bc36a..cda500897d 100644 --- a/latest/blogs/H100vsA100.html +++ b/latest/blogs/H100vsA100.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -676,10 +684,12 @@
+ +
@@ -701,12 +711,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -754,9 +759,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/H200launch.html b/latest/blogs/H200launch.html index 1c2979ae9f..fc93a07b46 100644 --- a/latest/blogs/H200launch.html +++ b/latest/blogs/H200launch.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -668,10 +676,12 @@ TensorRT LLM v0.5.0, TensorRT v9.1.0.4 | H200, H100 FP8.

+ +
@@ -693,12 +703,7 @@ TensorRT LLM v0.5.0, TensorRT v9.1.0.4 | H200, H100 FP8.

- Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -746,9 +751,9 @@ TensorRT LLM v0.5.0, TensorRT v9.1.0.4 | H200, H100 FP8.

-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/XQA-kernel.html b/latest/blogs/XQA-kernel.html index 3382936851..612a0e080b 100644 --- a/latest/blogs/XQA-kernel.html +++ b/latest/blogs/XQA-kernel.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -635,10 +643,12 @@ ISL = Input Sequence Length
+ +
@@ -660,12 +670,7 @@ ISL = Input Sequence Length - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -713,9 +718,9 @@ ISL = Input Sequence Length
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/quantization-in-TRT-LLM.html b/latest/blogs/quantization-in-TRT-LLM.html index 5ffbbf6b64..b956b8f578 100644 --- a/latest/blogs/quantization-in-TRT-LLM.html +++ b/latest/blogs/quantization-in-TRT-LLM.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -770,10 +778,12 @@
+ +
@@ -795,12 +805,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -848,9 +853,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog10_ADP_Balance_Strategy.html b/latest/blogs/tech_blog/blog10_ADP_Balance_Strategy.html index 0825ea17c2..1907c5cf75 100644 --- a/latest/blogs/tech_blog/blog10_ADP_Balance_Strategy.html +++ b/latest/blogs/tech_blog/blog10_ADP_Balance_Strategy.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -61,22 +63,26 @@ + + + - + + @@ -84,6 +90,8 @@ + +
@@ -1017,10 +1025,12 @@ The Pareto frontier analysis provides critical insights for real-world deploymen
+ +
@@ -1042,12 +1052,7 @@ The Pareto frontier analysis provides critical insights for real-world deploymen - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1095,9 +1100,9 @@ The Pareto frontier analysis provides critical insights for real-world deploymen
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog11_GPT_OSS_Eagle3.html b/latest/blogs/tech_blog/blog11_GPT_OSS_Eagle3.html index c36b2f7eb3..a35435ea3c 100644 --- a/latest/blogs/tech_blog/blog11_GPT_OSS_Eagle3.html +++ b/latest/blogs/tech_blog/blog11_GPT_OSS_Eagle3.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -710,10 +718,12 @@ cat > /config/models/eagle/eagl
+ +
@@ -735,12 +745,7 @@ cat > /config/models/eagle/eagl - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -788,9 +793,9 @@ cat > /config/models/eagle/eagl
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html b/latest/blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html index 2c3aae7045..5ca9cc72b2 100644 --- a/latest/blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html +++ b/latest/blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -61,22 +63,26 @@ + + + - + + @@ -84,6 +90,8 @@ + +
@@ -931,10 +939,12 @@
+ +
@@ -956,12 +966,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1009,9 +1014,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html b/latest/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html index 3c2801e1f5..c1bfa8ce8f 100644 --- a/latest/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html +++ b/latest/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -998,10 +1006,12 @@ is a certainty-based, training-free approach to accelerate Chain-of-Thought (CoT
+ +
@@ -1023,12 +1033,7 @@ is a certainty-based, training-free approach to accelerate Chain-of-Thought (CoT - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1076,9 +1081,9 @@ is a certainty-based, training-free approach to accelerate Chain-of-Thought (CoT
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html b/latest/blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html index 87786c1dc5..40f6b15cf1 100644 --- a/latest/blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html +++ b/latest/blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -817,10 +825,12 @@ However, since Q is in BF16 format, FMHA will also be performed in BF16 format,
+ +
@@ -842,12 +852,7 @@ However, since Q is in BF16 format, FMHA will also be performed in BF16 format, - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -895,9 +900,9 @@ However, since Q is in BF16 format, FMHA will also be performed in BF16 format,
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html b/latest/blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html index c2432b879c..0834a231d8 100644 --- a/latest/blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html +++ b/latest/blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -1111,10 +1119,12 @@
+ +
@@ -1136,12 +1146,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1189,9 +1194,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html b/latest/blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html index aa4d21135d..84e2f76322 100644 --- a/latest/blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html +++ b/latest/blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -61,22 +63,26 @@ + + + - + + @@ -84,6 +90,8 @@ + +
@@ -856,10 +864,12 @@ trtllm-bench --model nvidia/DeepSe
+ +
@@ -881,12 +891,7 @@ trtllm-bench --model nvidia/DeepSe - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -934,9 +939,9 @@ trtllm-bench --model nvidia/DeepSe
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html b/latest/blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html index 887b0a2cc1..18bb723ba1 100644 --- a/latest/blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html +++ b/latest/blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -833,10 +841,12 @@ Running the shared and routed experts in 2 streams combined with other multi-str
+ +
@@ -858,12 +868,7 @@ Running the shared and routed experts in 2 streams combined with other multi-str - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -911,9 +916,9 @@ Running the shared and routed experts in 2 streams combined with other multi-str
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.html b/latest/blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.html index aae396af04..ddb60e010c 100644 --- a/latest/blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.html +++ b/latest/blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -61,22 +63,26 @@ + + + - + + @@ -84,6 +90,8 @@ + +
@@ -1377,10 +1385,12 @@ Based on our current performance analysis, when you plan to apply large-scale EP
+ +
@@ -1402,12 +1412,7 @@ Based on our current performance analysis, when you plan to apply large-scale EP - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1455,9 +1460,9 @@ Based on our current performance analysis, when you plan to apply large-scale EP
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog5_Disaggregated_Serving_in_TensorRT-LLM.html b/latest/blogs/tech_blog/blog5_Disaggregated_Serving_in_TensorRT-LLM.html index b2d683c81a..6445696743 100644 --- a/latest/blogs/tech_blog/blog5_Disaggregated_Serving_in_TensorRT-LLM.html +++ b/latest/blogs/tech_blog/blog5_Disaggregated_Serving_in_TensorRT-LLM.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -61,22 +63,26 @@ + + + - + + @@ -84,6 +90,8 @@ + +
@@ -891,10 +899,12 @@ trtllm-serve disaggregated -c
+ +
@@ -916,12 +926,7 @@ trtllm-serve disaggregated -cManage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -969,9 +974,9 @@ trtllm-serve disaggregated -c
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog6_Llama4_maverick_eagle_guide.html b/latest/blogs/tech_blog/blog6_Llama4_maverick_eagle_guide.html index 88799d490f..5b8231a4fc 100644 --- a/latest/blogs/tech_blog/blog6_Llama4_maverick_eagle_guide.html +++ b/latest/blogs/tech_blog/blog6_Llama4_maverick_eagle_guide.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -734,10 +742,12 @@ + +
@@ -759,12 +769,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -812,9 +817,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog7_NGram_performance_Analysis_And_Auto_Enablement.html b/latest/blogs/tech_blog/blog7_NGram_performance_Analysis_And_Auto_Enablement.html index 2e8d770caa..818625796f 100644 --- a/latest/blogs/tech_blog/blog7_NGram_performance_Analysis_And_Auto_Enablement.html +++ b/latest/blogs/tech_blog/blog7_NGram_performance_Analysis_And_Auto_Enablement.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -856,10 +864,12 @@ N-Gram with k
+ +
@@ -881,12 +891,7 @@ N-Gram with kManage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -934,9 +939,9 @@ N-Gram with k
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.html b/latest/blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.html index 00fb7ac5d0..f20fa3e2b5 100644 --- a/latest/blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.html +++ b/latest/blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -901,10 +909,12 @@ always defer defer+madvise + +
@@ -926,12 +936,7 @@ always defer defer+madviseManage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -979,9 +984,9 @@ always defer defer+madvise
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/blogs/tech_blog/blog9_Deploying_GPT_OSS_on_TRTLLM.html b/latest/blogs/tech_blog/blog9_Deploying_GPT_OSS_on_TRTLLM.html index 8080641cea..69d652b815 100644 --- a/latest/blogs/tech_blog/blog9_Deploying_GPT_OSS_on_TRTLLM.html +++ b/latest/blogs/tech_blog/blog9_Deploying_GPT_OSS_on_TRTLLM.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -926,10 +934,12 @@ others according to your needs.

+ +
@@ -951,12 +961,7 @@ others according to your needs.

- Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1004,9 +1009,9 @@ others according to your needs.

-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/commands/trtllm-bench.html b/latest/commands/trtllm-bench.html index 394eb24af9..30a4ea067e 100644 --- a/latest/commands/trtllm-bench.html +++ b/latest/commands/trtllm-bench.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -1256,10 +1264,12 @@
+ +
@@ -1281,12 +1291,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1334,9 +1339,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/commands/trtllm-build.html b/latest/commands/trtllm-build.html index 12c0544fce..6f37e92153 100644 --- a/latest/commands/trtllm-build.html +++ b/latest/commands/trtllm-build.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -933,10 +941,12 @@
+ +
@@ -958,12 +968,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1011,9 +1016,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/commands/trtllm-eval.html b/latest/commands/trtllm-eval.html index 0b30acb437..779e0ced41 100644 --- a/latest/commands/trtllm-eval.html +++ b/latest/commands/trtllm-eval.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -1353,10 +1361,12 @@ trtllm-eval --model meta-llama/Lla
+ +
@@ -1378,12 +1388,7 @@ trtllm-eval --model meta-llama/Lla - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1431,9 +1436,9 @@ trtllm-eval --model meta-llama/Lla
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/commands/trtllm-serve/index.html b/latest/commands/trtllm-serve/index.html index 066ef22a4b..a674de9032 100644 --- a/latest/commands/trtllm-serve/index.html +++ b/latest/commands/trtllm-serve/index.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -562,10 +570,12 @@
+ +
@@ -587,12 +597,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -640,9 +645,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/commands/trtllm-serve/run-benchmark-with-trtllm-serve.html b/latest/commands/trtllm-serve/run-benchmark-with-trtllm-serve.html index 85e851fb83..3f38600afc 100644 --- a/latest/commands/trtllm-serve/run-benchmark-with-trtllm-serve.html +++ b/latest/commands/trtllm-serve/run-benchmark-with-trtllm-serve.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -533,7 +541,7 @@ A complete reference for the API is available in the #

TensorRT LLM distributes the pre-built container on NGC Catalog.

You can launch the container using the following command:

-
docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2
+
docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3
@@ -923,10 +931,12 @@ trtllm-serve ${m + +
@@ -948,12 +958,7 @@ trtllm-serve ${m - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1001,9 +1006,9 @@ trtllm-serve ${m
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/commands/trtllm-serve/trtllm-serve.html b/latest/commands/trtllm-serve/trtllm-serve.html index 87f2c332dc..3c0d92f60a 100644 --- a/latest/commands/trtllm-serve/trtllm-serve.html +++ b/latest/commands/trtllm-serve/trtllm-serve.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -1004,7 +1012,7 @@ Since the statistics are stored in an internal queue and removed once retrieved,

[Experimental] Specify the parser for tool models.

Options:
-

qwen3

+

qwen3 | qwen3_coder

@@ -1152,10 +1160,12 @@ Please refer to Performance Benchmarking with `trtllm-serve <
+ +
@@ -1177,12 +1187,7 @@ Please refer to Performance Benchmarking with `trtllm-serve <Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1230,9 +1235,9 @@ Please refer to Performance Benchmarking with `trtllm-serve <
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.html b/latest/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.html index 3b8bb3fbd5..9f414e5dac 100644 --- a/latest/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.html +++ b/latest/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -580,7 +588,7 @@ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash @@ -1097,10 +1105,12 @@ chmod +x bench.sh + +
@@ -1122,12 +1132,7 @@ chmod +x bench.sh - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1175,9 +1180,9 @@ chmod +x bench.sh
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.html b/latest/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.html index 499f8a4dd9..e9d57b4a7c 100644 --- a/latest/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.html +++ b/latest/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -568,7 +576,7 @@ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash
@@ -1057,10 +1065,12 @@ chmod +x bench.sh + +
@@ -1082,12 +1092,7 @@ chmod +x bench.sh - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1135,9 +1140,9 @@ chmod +x bench.sh
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.html b/latest/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.html index af0ad1f012..b7bd83c97a 100644 --- a/latest/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.html +++ b/latest/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -547,7 +555,7 @@ Python3 and python3-pip (Optional, for accuracy evaluation only)

-p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash
@@ -975,10 +983,12 @@ chmod +x bench.sh + +
@@ -1000,12 +1010,7 @@ chmod +x bench.sh - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1053,9 +1058,9 @@ chmod +x bench.sh
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.html b/latest/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.html index 2eec555457..de6298d782 100644 --- a/latest/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.html +++ b/latest/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -549,7 +557,7 @@ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc2 \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc3 \ /bin/bash
@@ -1003,10 +1011,12 @@ chmod +x bench.sh + +
@@ -1028,12 +1038,7 @@ chmod +x bench.sh - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1081,9 +1086,9 @@ chmod +x bench.sh
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.html b/latest/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.html index baf4b89dd5..d3d78ea757 100644 --- a/latest/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.html +++ b/latest/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -846,10 +854,12 @@ chmod +x bench.sh
+ +
@@ -871,12 +881,7 @@ chmod +x bench.sh - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -924,9 +929,9 @@ chmod +x bench.sh
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/deployment-guide/index.html b/latest/deployment-guide/index.html index d5cc1daf66..67eb76028a 100644 --- a/latest/deployment-guide/index.html +++ b/latest/deployment-guide/index.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -683,10 +691,12 @@
+ +
@@ -708,12 +718,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -761,9 +766,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/developer-guide/api-change.html b/latest/developer-guide/api-change.html index c2ad02e695..7de118c98a 100644 --- a/latest/developer-guide/api-change.html +++ b/latest/developer-guide/api-change.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -560,36 +568,41 @@

1. Knob Naming#

Use Semantic Clarity

Argument names should describe what the argument represents, not how it is used internally.

-

✅ Good: max_new_tokens (clear meaning)
-❌ Bad: num (ambiguous)

+

✅ Good: max_new_tokens (clear meaning)

+

❌ Bad: num (ambiguous)

Reflect Argument Type and Granularity

-
    -

  • For boolean knobs, prefix with verbs like enable_ and so on. -Examples: enable_cache, enable_flash_attention

    • -

  • For numerical threshold knobs, suffix with _limit, _size, _count, _len_ or _ratio
    -Examples: max_seq_len, prefill_batch_size

    • +
      +

    • For boolean knobs, prefix with verbs like enable_ and so on.

      +

      Examples: enable_cache, enable_flash_attention

      +
      • +

    • For numerical threshold knobs, suffix with _limit, _size, _count, _len_ or _ratio

      +

      Examples: max_seq_len, prefill_batch_size

      +

    Avoid Redundant Prefixes

    Example (in MoeConfig):

    -

    ✅ Good: backend
    -❌ Bad: moe_backend (redundant since it’s already in MoeConfig)

    +

    ✅ Good: backend

    +

    ❌ Bad: moe_backend (redundant since it’s already in MoeConfig)

    Use Specific Names for Narrow Scenarios

    When adding knobs for specific use cases, make the name convey the restriction clearly via a prefix. It’s acceptable to rename later when the knob becomes more generic or is moved into a dedicated config.

    Example (argument to the LLM class):

    -

    ✅ Good: rope_scaling_factor → clearly indicates it’s for RoPE
    -❌ Bad: scaling_factor → too generic and prone to misuse

    +

    ✅ Good: rope_scaling_factor → clearly indicates it’s for RoPE

    +

    ❌ Bad: scaling_factor → too generic and prone to misuse

2. Hierarchical Configuration#

Organize complex or hierarchical arguments into dedicated configuration dataclasses with intuitive and consistent naming.

Guidelines

-
    -

  • Use the XxxConfig suffix consistently
    -Examples: ModelConfig, ParallelConfig, MoeConfig

    • -

  • Reflect conceptual hierarchy
    -The dataclass name should represent a coherent functional unit, not an arbitrary grouping

    • -

  • Avoid over-nesting
    -Use only one level of configuration hierarchy whenever possible (e.g., LlmArgs → ParallelConfig) to balance readability and modularity

    • +
      +

    • Use the XxxConfig suffix consistently

      +

      Examples: ModelConfig, ParallelConfig, MoeConfig

      +
      • +

    • Reflect conceptual hierarchy

      +

      The dataclass name should represent a coherent functional unit, not an arbitrary grouping

      +
      • +

    • Avoid over-nesting

      +

      Use only one level of configuration hierarchy whenever possible (e.g., LlmArgs → ParallelConfig) to balance readability and modularity

      +
@@ -671,7 +684,7 @@ Use only one level of configuration hierarchy whenever possible (e.g., 2. Update the API schema#

Add the field to the appropriate schema file:

    -

  • Non-committed arguments: tests/unittest/api_stability/references/llm_args.yaml

    +

  • Non-committed arguments: tests/unittest/api_stability/references/llm.yaml

    garbage_collection_gen0_threshold:
   type: int
   default: 20000
@@ -679,7 +692,7 @@ Use only one level of configuration hierarchy whenever possible (e.g.,
    • -

  • Committed arguments: tests/unittest/api_stability/references_committed/llm_args.yaml

    +

  • Committed arguments: tests/unittest/api_stability/references_committed/llm.yaml

    garbage_collection_gen0_threshold:
   type: int
   default: 20000
@@ -715,16 +728,16 @@ Use only one level of configuration hierarchy whenever possible (e.g., For non-committed APIs, use the @set_api_status decorator:
    
 
    @set_api_status("beta")
 def generate_with_streaming(
-    self, 
-    prompts: List[str], 
+    self,
+    prompts: List[str],
     **kwargs
 ) -> Iterator[GenerationOutput]:
     """Generate text with streaming output.
-    
+
     Args:
         prompts: Input prompts for generation
         **kwargs: Additional generation parameters
-        
+
     Returns:
         Iterator of generation outputs
     """
@@ -946,10 +959,12 @@ python -m pytest <
     
    
   
    
   
+
   
   
 
 
+
   
    
 
    
   
@@ -971,12 +986,7 @@ python -m pytest <
   
   
   
-  Manage My Privacy
-   | 
-  
-  
-  
-  Do Not Sell or Share My Data
+  Your Privacy Choices
    | 
   
   
@@ -1024,9 +1034,9 @@ python -m pytest <
         
    
 
    
   
-  
    Last updated on November 05, 2025.
    
+  
    Last updated on November 20, 2025.
    
   
-  
    This page is generated by TensorRT-LLM commit 3111682.
    
+  
    This page is generated by TensorRT-LLM commit 2128f73.
    
   
 
    
       
diff --git a/latest/developer-guide/ci-overview.html b/latest/developer-guide/ci-overview.html
index e53d793ba0..31315e11c9 100644
--- a/latest/developer-guide/ci-overview.html
+++ b/latest/developer-guide/ci-overview.html
@@ -32,7 +32,7 @@
 
 
     
-    
+    
     
     
     
@@ -44,6 +44,8 @@
   
 
 
+
+
     
     
     
@@ -59,22 +61,26 @@
     
+
     
+
     
     
     
     
 
+
   
   
-  
+  
 
 
   
+
   
   
   
@@ -82,6 +88,8 @@
   
   
   
+
+
   
   
    
   
@@ -707,10 +715,12 @@ selective keeps CI turnaround fast and conserves hardware resources.
    + +
    @@ -732,12 +742,7 @@ selective keeps CI turnaround fast and conserves hardware resources.

    - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -785,9 +790,9 @@ selective keeps CI turnaround fast and conserves hardware resources.

    -

    Last updated on November 05, 2025.

    +

    Last updated on November 20, 2025.

    -

    This page is generated by TensorRT-LLM commit 3111682.

    +

    This page is generated by TensorRT-LLM commit 2128f73.

    diff --git a/latest/developer-guide/dev-containers.html b/latest/developer-guide/dev-containers.html index 7c91a0adb4..ac50b8b96a 100644 --- a/latest/developer-guide/dev-containers.html +++ b/latest/developer-guide/dev-containers.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
    @@ -669,10 +677,12 @@ initialization script will create one with the contents listed above.

    + +
    @@ -694,12 +704,7 @@ initialization script will create one with the contents listed above.

    - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -747,9 +752,9 @@ initialization script will create one with the contents listed above.

    -

    Last updated on November 05, 2025.

    +

    Last updated on November 20, 2025.

    -

    This page is generated by TensorRT-LLM commit 3111682.

    +

    This page is generated by TensorRT-LLM commit 2128f73.

    diff --git a/latest/developer-guide/kv-transfer.html b/latest/developer-guide/kv-transfer.html index 638785f16f..9b30101cce 100644 --- a/latest/developer-guide/kv-transfer.html +++ b/latest/developer-guide/kv-transfer.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
    @@ -667,10 +675,12 @@
    + +
    @@ -692,12 +702,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -745,9 +750,9 @@
    -

    Last updated on November 05, 2025.

    +

    Last updated on November 20, 2025.

    -

    This page is generated by TensorRT-LLM commit 3111682.

    +

    This page is generated by TensorRT-LLM commit 2128f73.

    diff --git a/latest/developer-guide/overview.html b/latest/developer-guide/overview.html index 7839ec5bbd..d8f93dd695 100644 --- a/latest/developer-guide/overview.html +++ b/latest/developer-guide/overview.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
    @@ -635,10 +643,12 @@
    + +
    @@ -660,12 +670,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -713,9 +718,9 @@
    -

    Last updated on November 05, 2025.

    +

    Last updated on November 20, 2025.

    -

    This page is generated by TensorRT-LLM commit 3111682.

    +

    This page is generated by TensorRT-LLM commit 2128f73.

    diff --git a/latest/developer-guide/perf-analysis.html b/latest/developer-guide/perf-analysis.html index b5b6768052..5447b6c1e4 100644 --- a/latest/developer-guide/perf-analysis.html +++ b/latest/developer-guide/perf-analysis.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
    @@ -682,10 +690,12 @@ python3 benchmarks/cpp/prepare_dataset.py
    + +
    @@ -707,12 +717,7 @@ python3 benchmarks/cpp/prepare_dataset.py - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -760,9 +765,9 @@ python3 benchmarks/cpp/prepare_dataset.py
    -

    Last updated on November 05, 2025.

    +

    Last updated on November 20, 2025.

    -

    This page is generated by TensorRT-LLM commit 3111682.

    +

    This page is generated by TensorRT-LLM commit 2128f73.

    diff --git a/latest/developer-guide/perf-benchmarking.html b/latest/developer-guide/perf-benchmarking.html index 49c19fe605..3f9ade7bfa 100644 --- a/latest/developer-guide/perf-benchmarking.html +++ b/latest/developer-guide/perf-benchmarking.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
    @@ -1087,10 +1095,12 @@ when the checkpoint precision is
    + +
    @@ -1112,12 +1122,7 @@ when the checkpoint precision is Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1165,9 +1170,9 @@ when the checkpoint precision is
    -

    Last updated on November 05, 2025.

    +

    Last updated on November 20, 2025.

    -

    This page is generated by TensorRT-LLM commit 3111682.

    +

    This page is generated by TensorRT-LLM commit 2128f73.

    diff --git a/latest/developer-guide/perf-overview.html b/latest/developer-guide/perf-overview.html index 102a022ef3..a65e07b878 100644 --- a/latest/developer-guide/perf-overview.html +++ b/latest/developer-guide/perf-overview.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
    @@ -1352,7 +1360,7 @@ nvidia/Qwen3-235B-A22B-FP8

Preparing a Dataset#

-

In order to prepare a dataset, you can use the provided script. +

In order to prepare a dataset, you can use the provided script. To generate a synthetic dataset, run the following command:

python benchmarks/cpp/prepare_dataset.py --tokenizer=$model_name --stdout token-norm-dist --num-requests=$num_requests --input-mean=$isl --output-mean=$osl --input-stdev=0 --output-stdev=0 > $dataset_file
@@ -1432,7 +1440,7 @@ remain in the system longer and therefore require less requests to achieve stead

Running the Benchmark#

To run the benchmark with the generated data set, simply use the trtllm-bench throughput subcommand. The benchmarker will run an offline maximum throughput scenario such that all requests are queued in rapid succession. You simply need to provide -a model name (HuggingFace reference or path to a local model), a generated dataset, and a file containing any desired extra options to the LLM APIs (details in tensorrt_llm/llmapi/llm_args.py:LlmArgs).

+a model name (HuggingFace reference or path to a local model), a generated dataset, and a file containing any desired extra options to the LLM APIs (details in tensorrt_llm/llmapi/llm_args.py:LlmArgs).

For dense / non-MoE models:

trtllm-bench --tp $tp_size --pp $pp_size --model $model_name throughput --dataset $dataset_file --backend pytorch --extra_llm_api_options $llm_options
@@ -1610,10 +1618,12 @@ using the --kv_cach
+ +
@@ -1635,12 +1645,7 @@ using the --kv_cach - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1688,9 +1693,9 @@ using the --kv_cach
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/curl_chat_client.html b/latest/examples/curl_chat_client.html index 7efbc5c8f6..ef1f3ea393 100644 --- a/latest/examples/curl_chat_client.html +++ b/latest/examples/curl_chat_client.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

Curl Chat Client#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#! /usr/bin/env bash
  2
  3curl http://localhost:8000/v1/chat/completions \
@@ -574,10 +582,12 @@
+ +
@@ -599,12 +609,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -652,9 +657,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/curl_chat_client_for_multimodal.html b/latest/examples/curl_chat_client_for_multimodal.html index e9bfbd2497..76a86c268c 100644 --- a/latest/examples/curl_chat_client_for_multimodal.html +++ b/latest/examples/curl_chat_client_for_multimodal.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

Curl Chat Client For Multimodal#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#! /usr/bin/env bash
  2
  3# SINGLE IMAGE INFERENCE
@@ -651,10 +659,12 @@
+ +
@@ -676,12 +686,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -729,9 +734,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/curl_completion_client.html b/latest/examples/curl_completion_client.html index 0e1e24e5d1..3f6a5c50a9 100644 --- a/latest/examples/curl_completion_client.html +++ b/latest/examples/curl_completion_client.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

Curl Completion Client#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#! /usr/bin/env bash
  2
  3curl http://localhost:8000/v1/completions \
@@ -573,10 +581,12 @@
+ +
@@ -598,12 +608,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -651,9 +656,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/customization.html b/latest/examples/customization.html index 3e57b1265f..162d0ed41e 100644 --- a/latest/examples/customization.html +++ b/latest/examples/customization.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -690,10 +698,12 @@
+ +
@@ -715,12 +725,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -768,9 +773,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/deepseek_r1_reasoning_parser.html b/latest/examples/deepseek_r1_reasoning_parser.html index 2ac1161c9f..166c34a10f 100644 --- a/latest/examples/deepseek_r1_reasoning_parser.html +++ b/latest/examples/deepseek_r1_reasoning_parser.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

Deepseek R1 Reasoning Parser#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#! /usr/bin/env bash
  2
  3trtllm-serve \
@@ -573,10 +581,12 @@
+ +
@@ -598,12 +608,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -651,9 +656,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/dynamo_k8s_example.html b/latest/examples/dynamo_k8s_example.html index ebe12d8954..e6eb6a5862 100644 --- a/latest/examples/dynamo_k8s_example.html +++ b/latest/examples/dynamo_k8s_example.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -570,10 +578,12 @@ for more details.

+ +
@@ -595,12 +605,7 @@ for more details.

- Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -648,9 +653,9 @@ for more details.

-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/genai_perf_client.html b/latest/examples/genai_perf_client.html index 0e4e390f0b..8b32681ce0 100644 --- a/latest/examples/genai_perf_client.html +++ b/latest/examples/genai_perf_client.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

Genai Perf Client#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#! /usr/bin/env bash
  2
  3genai-perf profile \
@@ -579,10 +587,12 @@
+ +
@@ -604,12 +614,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -657,9 +662,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/genai_perf_client_for_multimodal.html b/latest/examples/genai_perf_client_for_multimodal.html index a86994e98f..727eaf5636 100644 --- a/latest/examples/genai_perf_client_for_multimodal.html +++ b/latest/examples/genai_perf_client_for_multimodal.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

Genai Perf Client For Multimodal#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#! /usr/bin/env bash
  2
  3genai-perf profile \
@@ -582,10 +590,12 @@
+ +
@@ -607,12 +617,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -660,9 +665,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/index.html b/latest/examples/index.html index 4404e1a0ce..17b2c3ed1e 100644 --- a/latest/examples/index.html +++ b/latest/examples/index.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -580,10 +588,12 @@
+ +
@@ -605,12 +615,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -658,9 +663,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/kvcacheconfig.html b/latest/examples/kvcacheconfig.html index 985b238cc5..284bdc450b 100644 --- a/latest/examples/kvcacheconfig.html +++ b/latest/examples/kvcacheconfig.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -575,10 +583,12 @@
+ +
@@ -600,12 +610,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -653,9 +658,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/kvcacheretentionconfig.html b/latest/examples/kvcacheretentionconfig.html index 36d0785c47..006422be18 100644 --- a/latest/examples/kvcacheretentionconfig.html +++ b/latest/examples/kvcacheretentionconfig.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -598,10 +606,12 @@
+ +
@@ -623,12 +633,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -676,9 +681,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_api_examples.html b/latest/examples/llm_api_examples.html index a63f76e16c..7660b640ea 100644 --- a/latest/examples/llm_api_examples.html +++ b/latest/examples/llm_api_examples.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -615,10 +623,12 @@
+ +
@@ -640,12 +650,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -693,9 +698,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_guided_decoding.html b/latest/examples/llm_guided_decoding.html index c1f5e9bc7b..76de28edb5 100644 --- a/latest/examples/llm_guided_decoding.html +++ b/latest/examples/llm_guided_decoding.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Generate text with guided decoding#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1from tensorrt_llm import LLM, SamplingParams
  2from tensorrt_llm.llmapi import GuidedDecodingParams
  3
@@ -606,10 +614,12 @@
+ +
@@ -631,12 +641,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -684,9 +689,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_inference.html b/latest/examples/llm_inference.html index be3ae87f0b..c5bd05680a 100644 --- a/latest/examples/llm_inference.html +++ b/latest/examples/llm_inference.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Generate text#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1from tensorrt_llm import LLM, SamplingParams
  2
  3
@@ -594,10 +602,12 @@
+ +
@@ -619,12 +629,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -672,9 +677,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_inference_async.html b/latest/examples/llm_inference_async.html index acf7aa7da1..bef53a3e91 100644 --- a/latest/examples/llm_inference_async.html +++ b/latest/examples/llm_inference_async.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Generate text asynchronously#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1import asyncio
  2
  3from tensorrt_llm import LLM, SamplingParams
@@ -602,10 +610,12 @@
+ +
@@ -627,12 +637,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -680,9 +685,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_inference_async_streaming.html b/latest/examples/llm_inference_async_streaming.html index 35523d6665..c9ce26a8c3 100644 --- a/latest/examples/llm_inference_async_streaming.html +++ b/latest/examples/llm_inference_async_streaming.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Generate text in streaming#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1import asyncio
  2
  3from tensorrt_llm import LLM, SamplingParams
@@ -623,10 +631,12 @@
+ +
@@ -648,12 +658,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -701,9 +706,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_inference_distributed.html b/latest/examples/llm_inference_distributed.html index cd7b7e73f3..3af9b4b703 100644 --- a/latest/examples/llm_inference_distributed.html +++ b/latest/examples/llm_inference_distributed.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Distributed LLM Generation#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1from tensorrt_llm import LLM, SamplingParams
  2
  3
@@ -603,10 +611,12 @@
+ +
@@ -628,12 +638,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -681,9 +686,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_kv_cache_connector.html b/latest/examples/llm_kv_cache_connector.html index 51c434a8cd..324afda46c 100644 --- a/latest/examples/llm_kv_cache_connector.html +++ b/latest/examples/llm_kv_cache_connector.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

KV Cache Connector#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

  1
   2import os
   3import sys
@@ -806,10 +814,12 @@
+ +
@@ -831,12 +841,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -884,9 +889,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_kv_cache_offloading.html b/latest/examples/llm_kv_cache_offloading.html index af977c3392..f4c4fe07bd 100644 --- a/latest/examples/llm_kv_cache_offloading.html +++ b/latest/examples/llm_kv_cache_offloading.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

KV Cache Offloading#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

  1'''
   2This script demonstrates the effectiveness of KV cache host offloading in TensorRT-LLM.
   3
@@ -693,10 +701,12 @@
+ +
@@ -718,12 +728,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -771,9 +776,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_logits_processor.html b/latest/examples/llm_logits_processor.html index a92d6a5468..0b14913d01 100644 --- a/latest/examples/llm_logits_processor.html +++ b/latest/examples/llm_logits_processor.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Control generated text using logits processor#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

  1from typing import List, Optional
   2
   3import torch
@@ -687,10 +695,12 @@
+ +
@@ -712,12 +722,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -765,9 +770,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_mgmn_llm_distributed.html b/latest/examples/llm_mgmn_llm_distributed.html index e3d1708288..0cfd12b2a1 100644 --- a/latest/examples/llm_mgmn_llm_distributed.html +++ b/latest/examples/llm_mgmn_llm_distributed.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Run LLM-API with pytorch backend on Slurm#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#!/bin/bash
  2#SBATCH -A <account>    # parameter
  3#SBATCH -p <partition>  # parameter
@@ -614,10 +622,12 @@
+ +
@@ -639,12 +649,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -692,9 +697,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_mgmn_trtllm_bench.html b/latest/examples/llm_mgmn_trtllm_bench.html index b29f423de5..b9fb318c97 100644 --- a/latest/examples/llm_mgmn_trtllm_bench.html +++ b/latest/examples/llm_mgmn_trtllm_bench.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Run trtllm-bench with pytorch backend on Slurm#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#!/bin/bash
  2#SBATCH -A <account>
  3#SBATCH -p <partition>
@@ -654,10 +662,12 @@
+ +
@@ -679,12 +689,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -732,9 +737,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_mgmn_trtllm_serve.html b/latest/examples/llm_mgmn_trtllm_serve.html index ff9ff9357b..04cdb337cd 100644 --- a/latest/examples/llm_mgmn_trtllm_serve.html +++ b/latest/examples/llm_mgmn_trtllm_serve.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Run trtllm-serve with pytorch backend on Slurm#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1#!/bin/bash
  2#SBATCH -A <account>
  3#SBATCH -p <partition>
@@ -615,10 +623,12 @@
+ +
@@ -640,12 +650,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -693,9 +698,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_multilora.html b/latest/examples/llm_multilora.html index a37ee95eba..b0962efb50 100644 --- a/latest/examples/llm_multilora.html +++ b/latest/examples/llm_multilora.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Generate text with multiple LoRA adapters#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1
  2import argparse
  3from typing import Optional
@@ -648,10 +656,12 @@
+ +
@@ -673,12 +683,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -726,9 +731,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_runtime.html b/latest/examples/llm_runtime.html index 68edc2a63d..ca1e9d2eec 100644 --- a/latest/examples/llm_runtime.html +++ b/latest/examples/llm_runtime.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Runtime Configuration Examples#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1
  2import argparse
  3
@@ -655,10 +663,12 @@
+ +
@@ -680,12 +690,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -733,9 +738,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_sampling.html b/latest/examples/llm_sampling.html index 50591a9e8c..2e81e93761 100644 --- a/latest/examples/llm_sampling.html +++ b/latest/examples/llm_sampling.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Sampling Techniques Showcase#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

  1"""
   2This example demonstrates various sampling techniques available in TensorRT-LLM.
   3It showcases different sampling parameters and their effects on text generation.
@@ -788,10 +796,12 @@
+ +
@@ -813,12 +823,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -866,9 +871,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_sparse_attention.html b/latest/examples/llm_sparse_attention.html index 9cf6c1d844..3c4fa2803c 100644 --- a/latest/examples/llm_sparse_attention.html +++ b/latest/examples/llm_sparse_attention.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,212 +515,213 @@

Sparse Attention#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

  1"""
   2This example demonstrates how to use sparse attention with TensorRT-LLM.
   3
   4Supported sparse attention algorithms:
   5- RocketKV
-  6
-  7Usage:
-  8```bash
-  9python llm_sparse_attention.py --algo RocketKV --attention_backend TRTLLM --window_size 32 --kernel_size 63 --prompt_budget 2048
- 10```
- 11"""
- 12import argparse
- 13import json
- 14
- 15from tensorrt_llm import LLM, SamplingParams
- 16from tensorrt_llm.llmapi import (CudaGraphConfig, DeepSeekSparseAttentionConfig,
- 17                                 KvCacheConfig, MoeConfig,
- 18                                 RocketSparseAttentionConfig)
- 19
+  6- DSA
+  7
+  8Usage:
+  9```bash
+ 10python llm_sparse_attention.py --algo ROCKETKV --attention_backend TRTLLM --window_size 32 --kernel_size 63 --prompt_budget 2048
+ 11```
+ 12"""
+ 13import argparse
+ 14import json
+ 15
+ 16from tensorrt_llm import LLM, SamplingParams
+ 17from tensorrt_llm.llmapi import (CudaGraphConfig, DeepSeekSparseAttentionConfig,
+ 18                                 KvCacheConfig, MoeConfig,
+ 19                                 RocketSparseAttentionConfig)
  20
- 21def read_input(input_file):
- 22    results = []
- 23    with open(input_file, 'r') as f:
- 24        for line in f:
- 25            ret = json.loads(line)
- 26            results.append(ret)
- 27    return results
- 28
+ 21
+ 22def read_input(input_file):
+ 23    results = []
+ 24    with open(input_file, 'r') as f:
+ 25        for line in f:
+ 26            ret = json.loads(line)
+ 27            results.append(ret)
+ 28    return results
  29
- 30def parse_arguments():
- 31    parser = argparse.ArgumentParser()
- 32    parser.add_argument(
- 33        '--model_path',
- 34        type=str,
- 35        default=
- 36        "/home/scratch.trt_llm_data/llm-models/llama-3.1-model/Llama-3.1-8B-Instruct"
- 37    )
- 38    parser.add_argument(
- 39        '--input_file',
- 40        type=str,
- 41        default="tests/unittest/_torch/multi_gpu/test_star_attention_input.jsonl"
- 42    )
- 43    # Build config
- 44    parser.add_argument('--algo',
- 45                        type=str,
- 46                        default='ROCKETKV',
- 47                        choices=['ROCKETKV', 'DSA'])
- 48    parser.add_argument('--attention_backend',
- 49                        type=str,
- 50                        default='TRTLLM',
- 51                        choices=['VANILLA', 'TRTLLM'])
- 52    parser.add_argument('--window_size',
- 53                        type=int,
- 54                        default=32,
- 55                        help="The window size for RocketKV.")
- 56    parser.add_argument('--kernel_size',
- 57                        type=int,
- 58                        default=63,
- 59                        help="The kernel size for RocketKV.")
- 60    parser.add_argument('--prompt_budget',
- 61                        type=int,
- 62                        default=2048,
- 63                        help="The prompt budget for RocketKV.")
- 64    parser.add_argument('--index_max_chunk_size',
- 65                        type=int,
- 66                        default=32768,
- 67                        help="The maximum chunk size for the indexer.")
- 68    parser.add_argument("--max_seq_len",
- 69                        type=int,
- 70                        default=8192,
- 71                        help="The maximum sequence length.")
- 72    parser.add_argument("--max_batch_size",
- 73                        type=int,
- 74                        default=256,
- 75                        help="The maximum batch size.")
- 76    parser.add_argument("--max_new_tokens",
- 77                        type=int,
- 78                        default=128,
- 79                        help="The maximum new tokens.")
- 80    parser.add_argument(
- 81        "--max_num_tokens",
- 82        type=int,
- 83        default=8192,
- 84        help=
- 85        "The maximum total tokens (context + generation) across all sequences in a batch."
- 86    )
- 87
- 88    # Parallelism
- 89    parser.add_argument('--moe_backend',
- 90                        type=str,
- 91                        default='CUTLASS',
- 92                        choices=[
- 93                            'CUTLASS', 'TRTLLM', 'VANILLA', 'WIDEEP',
- 94                            'DEEPGEMM', 'CUTEDSL', 'TRITON'
- 95                        ])
- 96    parser.add_argument('--tp_size', type=int, default=1)
- 97    parser.add_argument('--moe_ep_size', type=int, default=-1)
- 98    parser.add_argument('--enable_attention_dp',
- 99                        default=False,
-100                        action='store_true')
-101
-102    # KV cache
-103    parser.add_argument('--kv_cache_dtype', type=str, default='auto')
-104    parser.add_argument("--kv_cache_fraction", type=float, default=None)
-105    parser.add_argument('--num_samples', type=int, default=10)
-106
-107    # Runtime
-108    parser.add_argument('--print_iter_log',
-109                        default=False,
-110                        action='store_true',
-111                        help='Print iteration logs during execution')
-112    parser.add_argument('--use_cuda_graph', default=False, action='store_true')
-113    parser.add_argument('--cuda_graph_padding_enabled',
-114                        default=False,
-115                        action='store_true')
-116    parser.add_argument('--cuda_graph_batch_sizes',
-117                        nargs='+',
-118                        type=int,
-119                        default=None)
-120    args = parser.parse_args()
-121    return args
-122
+ 30
+ 31def parse_arguments():
+ 32    parser = argparse.ArgumentParser()
+ 33    parser.add_argument(
+ 34        '--model_path',
+ 35        type=str,
+ 36        default=
+ 37        "/home/scratch.trt_llm_data/llm-models/llama-3.1-model/Llama-3.1-8B-Instruct"
+ 38    )
+ 39    parser.add_argument(
+ 40        '--input_file',
+ 41        type=str,
+ 42        default="tests/unittest/_torch/multi_gpu/test_star_attention_input.jsonl"
+ 43    )
+ 44    # Build config
+ 45    parser.add_argument('--algo',
+ 46                        type=str,
+ 47                        default='ROCKETKV',
+ 48                        choices=['ROCKETKV', 'DSA'])
+ 49    parser.add_argument('--attention_backend',
+ 50                        type=str,
+ 51                        default='TRTLLM',
+ 52                        choices=['VANILLA', 'TRTLLM'])
+ 53    parser.add_argument('--window_size',
+ 54                        type=int,
+ 55                        default=32,
+ 56                        help="The window size for RocketKV.")
+ 57    parser.add_argument('--kernel_size',
+ 58                        type=int,
+ 59                        default=63,
+ 60                        help="The kernel size for RocketKV.")
+ 61    parser.add_argument('--prompt_budget',
+ 62                        type=int,
+ 63                        default=2048,
+ 64                        help="The prompt budget for RocketKV.")
+ 65    parser.add_argument('--index_max_chunk_size',
+ 66                        type=int,
+ 67                        default=32768,
+ 68                        help="The maximum chunk size for the indexer.")
+ 69    parser.add_argument("--max_seq_len",
+ 70                        type=int,
+ 71                        default=10240,
+ 72                        help="The maximum sequence length.")
+ 73    parser.add_argument("--max_batch_size",
+ 74                        type=int,
+ 75                        default=256,
+ 76                        help="The maximum batch size.")
+ 77    parser.add_argument("--max_new_tokens",
+ 78                        type=int,
+ 79                        default=128,
+ 80                        help="The maximum new tokens.")
+ 81    parser.add_argument(
+ 82        "--max_num_tokens",
+ 83        type=int,
+ 84        default=81920,
+ 85        help=
+ 86        "The maximum total tokens (context + generation) across all sequences in a batch."
+ 87    )
+ 88
+ 89    # Parallelism
+ 90    parser.add_argument('--moe_backend',
+ 91                        type=str,
+ 92                        default='CUTLASS',
+ 93                        choices=[
+ 94                            'CUTLASS', 'TRTLLM', 'VANILLA', 'WIDEEP',
+ 95                            'DEEPGEMM', 'CUTEDSL', 'TRITON'
+ 96                        ])
+ 97    parser.add_argument('--tp_size', type=int, default=1)
+ 98    parser.add_argument('--moe_ep_size', type=int, default=-1)
+ 99    parser.add_argument('--enable_attention_dp',
+100                        default=False,
+101                        action='store_true')
+102
+103    # KV cache
+104    parser.add_argument('--kv_cache_dtype', type=str, default='auto')
+105    parser.add_argument("--kv_cache_fraction", type=float, default=0.7)
+106    parser.add_argument('--num_samples', type=int, default=10)
+107
+108    # Runtime
+109    parser.add_argument('--print_iter_log',
+110                        default=False,
+111                        action='store_true',
+112                        help='Print iteration logs during execution')
+113    parser.add_argument('--use_cuda_graph', default=False, action='store_true')
+114    parser.add_argument('--cuda_graph_padding_enabled',
+115                        default=False,
+116                        action='store_true')
+117    parser.add_argument('--cuda_graph_batch_sizes',
+118                        nargs='+',
+119                        type=int,
+120                        default=None)
+121    args = parser.parse_args()
+122    return args
 123
-124def run_llm(args, sparse_attention_config):
-125    data = read_input(args.input_file)
-126    num_samples = args.num_samples if args.num_samples is not None else len(
-127        data)
-128    data = data[:num_samples]
-129
-130    kv_cache_config = KvCacheConfig(
-131        enable_block_reuse=
-132        False,  # sparse attention does not support kv cache reuse now
-133        free_gpu_memory_fraction=args.kv_cache_fraction,
-134        dtype=args.kv_cache_dtype,
-135    )
-136
-137    cuda_graph_config = CudaGraphConfig(
-138        batch_sizes=args.cuda_graph_batch_sizes,
-139        enable_padding=args.cuda_graph_padding_enabled,
-140    ) if args.use_cuda_graph else None
-141
-142    llm = LLM(
-143        model=args.model_path,
-144        backend='pytorch',
-145        kv_cache_config=kv_cache_config,
-146        attn_backend=args.attention_backend,
-147        sparse_attention_config=sparse_attention_config,
-148        max_batch_size=args.max_batch_size,
-149        max_seq_len=args.max_seq_len,
-150        max_num_tokens=args.max_num_tokens,
-151        tensor_parallel_size=args.tp_size,
-152        moe_expert_parallel_size=args.moe_ep_size,
-153        enable_attention_dp=args.enable_attention_dp,
-154        cuda_graph_config=cuda_graph_config,
-155        print_iter_log=args.print_iter_log,
-156        enable_iter_perf_stats=args.print_iter_log,
-157        moe_config=MoeConfig(backend=args.moe_backend),
-158    )
-159
-160    prompts = []
-161    reference = []
-162    for sample in data:
-163        prompts.append(
-164            {'prompt': sample['input_context'] + sample['input_query']})
-165        reference.append(sample['outputs'])
-166
-167    sampling_params = SamplingParams(add_special_tokens=False,
-168                                     max_tokens=args.max_new_tokens,
-169                                     temperature=0.8,
-170                                     top_p=0.95)
-171
-172    outputs = llm.generate(prompts, sampling_params)
-173    for idx, output in enumerate(outputs):
-174        print(
-175            f'Generated text: {output.outputs[0].text!r}, ref: {reference[idx]}'
-176        )
-177
+124
+125def run_llm(args, sparse_attention_config):
+126    data = read_input(args.input_file)
+127    num_samples = args.num_samples if args.num_samples is not None else len(
+128        data)
+129    data = data[:num_samples]
+130
+131    kv_cache_config = KvCacheConfig(
+132        enable_block_reuse=
+133        False,  # sparse attention does not support kv cache reuse now
+134        free_gpu_memory_fraction=args.kv_cache_fraction,
+135        dtype=args.kv_cache_dtype,
+136    )
+137
+138    cuda_graph_config = CudaGraphConfig(
+139        batch_sizes=args.cuda_graph_batch_sizes,
+140        enable_padding=args.cuda_graph_padding_enabled,
+141    ) if args.use_cuda_graph else None
+142
+143    llm = LLM(
+144        model=args.model_path,
+145        backend='pytorch',
+146        kv_cache_config=kv_cache_config,
+147        attn_backend=args.attention_backend,
+148        sparse_attention_config=sparse_attention_config,
+149        max_batch_size=args.max_batch_size,
+150        max_seq_len=args.max_seq_len,
+151        max_num_tokens=args.max_num_tokens,
+152        tensor_parallel_size=args.tp_size,
+153        moe_expert_parallel_size=args.moe_ep_size,
+154        enable_attention_dp=args.enable_attention_dp,
+155        cuda_graph_config=cuda_graph_config,
+156        print_iter_log=args.print_iter_log,
+157        enable_iter_perf_stats=args.print_iter_log,
+158        moe_config=MoeConfig(backend=args.moe_backend),
+159    )
+160
+161    prompts = []
+162    reference = []
+163    for sample in data:
+164        prompts.append(
+165            {'prompt': sample['input_context'] + sample['input_query']})
+166        reference.append(sample['outputs'])
+167
+168    sampling_params = SamplingParams(add_special_tokens=False,
+169                                     max_tokens=args.max_new_tokens,
+170                                     temperature=0.8,
+171                                     top_p=0.95)
+172
+173    outputs = llm.generate(prompts, sampling_params)
+174    for idx, output in enumerate(outputs):
+175        print(
+176            f'Generated text: {output.outputs[0].text!r}, ref: {reference[idx]}'
+177        )
 178
-179def run_RocketKV(args):
-180    sparse_attention_config = RocketSparseAttentionConfig(
-181        window_size=args.window_size,
-182        kernel_size=args.kernel_size,
-183        prompt_budget=args.prompt_budget,
-184    )
-185    run_llm(args, sparse_attention_config)
-186
+179
+180def run_RocketKV(args):
+181    sparse_attention_config = RocketSparseAttentionConfig(
+182        window_size=args.window_size,
+183        kernel_size=args.kernel_size,
+184        prompt_budget=args.prompt_budget,
+185    )
+186    run_llm(args, sparse_attention_config)
 187
-188def run_DSA(args):
-189    sparse_attention_config = DeepSeekSparseAttentionConfig(
-190        indexer_max_chunk_size=args.index_max_chunk_size, )
-191    run_llm(args, sparse_attention_config)
-192
+188
+189def run_DSA(args):
+190    sparse_attention_config = DeepSeekSparseAttentionConfig(
+191        indexer_max_chunk_size=args.index_max_chunk_size, )
+192    run_llm(args, sparse_attention_config)
 193
-194def main():
-195    args = parse_arguments()
-196    if args.algo == 'ROCKETKV':
-197        run_RocketKV(args)
-198    elif args.algo == 'DSA':
-199        run_DSA(args)
-200    else:
-201        raise ValueError(f"Invalid algorithm: {args.algo}")
-202
+194
+195def main():
+196    args = parse_arguments()
+197    if args.algo == 'ROCKETKV':
+198        run_RocketKV(args)
+199    elif args.algo == 'DSA':
+200        run_DSA(args)
+201    else:
+202        raise ValueError(f"Invalid algorithm: {args.algo}")
 203
-204if __name__ == "__main__":
-205    main()
+204
+205if __name__ == "__main__":
+206    main()
@@ -767,10 +776,12 @@
+ +
@@ -792,12 +803,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -845,9 +851,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/llm_speculative_decoding.html b/latest/examples/llm_speculative_decoding.html index bd563acbd0..fc712b8eaa 100644 --- a/latest/examples/llm_speculative_decoding.html +++ b/latest/examples/llm_speculative_decoding.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -507,7 +515,7 @@

Speculative Decoding#

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1from typing import Optional
  2
  3import click
@@ -654,10 +662,12 @@
+ +
@@ -679,12 +689,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -732,9 +737,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/openai_chat_client.html b/latest/examples/openai_chat_client.html index 01235d8c29..86fe4780fb 100644 --- a/latest/examples/openai_chat_client.html +++ b/latest/examples/openai_chat_client.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

OpenAI Chat Client#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1
  2from openai import OpenAI
  3
@@ -583,10 +591,12 @@
+ +
@@ -608,12 +618,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -661,9 +666,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/openai_chat_client_for_multimodal.html b/latest/examples/openai_chat_client_for_multimodal.html index 267b91f9fb..f455441ee7 100644 --- a/latest/examples/openai_chat_client_for_multimodal.html +++ b/latest/examples/openai_chat_client_for_multimodal.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

OpenAI Chat Client for Multimodal#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

  1
   2import os
   3from pathlib import Path
@@ -691,10 +699,12 @@
+ +
@@ -716,12 +726,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -769,9 +774,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/openai_completion_client.html b/latest/examples/openai_completion_client.html index d7d027d1eb..0e5b0872df 100644 --- a/latest/examples/openai_completion_client.html +++ b/latest/examples/openai_completion_client.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

OpenAI Completion Client#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1
  2from openai import OpenAI
  3
@@ -577,10 +585,12 @@
+ +
@@ -602,12 +612,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -655,9 +660,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/openai_completion_client_for_lora.html b/latest/examples/openai_completion_client_for_lora.html index 67cb74c2ca..7b48cd9636 100644 --- a/latest/examples/openai_completion_client_for_lora.html +++ b/latest/examples/openai_completion_client_for_lora.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

Openai Completion Client For Lora#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1### OpenAI Completion Client
  2
  3import os
@@ -593,10 +601,12 @@
+ +
@@ -618,12 +628,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -671,9 +676,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/openai_completion_client_json_schema.html b/latest/examples/openai_completion_client_json_schema.html index aab08fc04b..e32b959391 100644 --- a/latest/examples/openai_completion_client_json_schema.html +++ b/latest/examples/openai_completion_client_json_schema.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -508,7 +516,7 @@

OpenAI Completion Client with JSON Schema#

Refer to the trtllm-serve documentation for starting a server.

-

Source NVIDIA/TensorRT-LLM.

+

Source NVIDIA/TensorRT-LLM.

 1
  2# This example requires to specify `guided_decoding_backend` as
  3# `xgrammar` or `llguidance` in the extra_llm_api_options.yaml file.
@@ -614,10 +622,12 @@
+ +
@@ -639,12 +649,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -692,9 +697,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/examples/trtllm_serve_examples.html b/latest/examples/trtllm_serve_examples.html index 48ac24bebb..040cf6df52 100644 --- a/latest/examples/trtllm_serve_examples.html +++ b/latest/examples/trtllm_serve_examples.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -571,10 +579,12 @@
+ +
@@ -596,12 +606,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -649,9 +654,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/additional-outputs.html b/latest/features/additional-outputs.html index 27c541ee3c..9d1e2e4d26 100644 --- a/latest/features/additional-outputs.html +++ b/latest/features/additional-outputs.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -641,10 +649,12 @@ and sequence.additi
+ +
@@ -666,12 +676,7 @@ and sequence.additi - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -719,9 +724,9 @@ and sequence.additi
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/attention.html b/latest/features/attention.html index aa7325c0f2..5a50ed43c7 100644 --- a/latest/features/attention.html +++ b/latest/features/attention.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -844,7 +852,7 @@ reach that point).

the different requests by a cache manager during processing. That cache manager keeps track of the sequences, allocates new blocks from a pool and recycles those blocks when required. See the implementation of -KVCacheManager.

+KVCacheManager.

INT8/FP8 KV Caches#

@@ -1060,10 +1068,12 @@ is computed as:

+ +
@@ -1085,12 +1095,7 @@ is computed as:

- Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1138,9 +1143,9 @@ is computed as:

-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/auto_deploy/advanced/benchmarking_with_trtllm_bench.html b/latest/features/auto_deploy/advanced/benchmarking_with_trtllm_bench.html index ce12234ba6..59f84ceaef 100644 --- a/latest/features/auto_deploy/advanced/benchmarking_with_trtllm_bench.html +++ b/latest/features/auto_deploy/advanced/benchmarking_with_trtllm_bench.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -697,10 +705,12 @@
+ +
@@ -722,12 +732,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -775,9 +780,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/auto_deploy/advanced/example_run.html b/latest/features/auto_deploy/advanced/example_run.html index 10f26f8916..8567d5503e 100644 --- a/latest/features/auto_deploy/advanced/example_run.html +++ b/latest/features/auto_deploy/advanced/example_run.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -613,10 +621,12 @@ python build_and_run_ad.py
+ +
@@ -638,12 +648,7 @@ python build_and_run_ad.py Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -691,9 +696,9 @@ python build_and_run_ad.py
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/auto_deploy/advanced/expert_configurations.html b/latest/features/auto_deploy/advanced/expert_configurations.html index 515623c6d4..0b9aa3d6d1 100644 --- a/latest/features/auto_deploy/advanced/expert_configurations.html +++ b/latest/features/auto_deploy/advanced/expert_configurations.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -720,10 +728,12 @@ python build_and_run_ad.py
+ +
@@ -745,12 +755,7 @@ python build_and_run_ad.py Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -798,9 +803,9 @@ python build_and_run_ad.py
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/auto_deploy/advanced/logging.html b/latest/features/auto_deploy/advanced/logging.html index 99bfbe2a01..a3858d8323 100644 --- a/latest/features/auto_deploy/advanced/logging.html +++ b/latest/features/auto_deploy/advanced/logging.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -546,10 +554,12 @@ decreasing verbosity;

+ +
@@ -571,12 +581,7 @@ decreasing verbosity;

- Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -624,9 +629,9 @@ decreasing verbosity;

-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/auto_deploy/advanced/workflow.html b/latest/features/auto_deploy/advanced/workflow.html index 819916b706..25e0b98dac 100644 --- a/latest/features/auto_deploy/advanced/workflow.html +++ b/latest/features/auto_deploy/advanced/workflow.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -559,10 +567,12 @@
+ +
@@ -584,12 +594,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -637,9 +642,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/auto_deploy/auto-deploy.html b/latest/features/auto_deploy/auto-deploy.html index b3e24c168d..18f1daa776 100644 --- a/latest/features/auto_deploy/auto-deploy.html +++ b/latest/features/auto_deploy/auto-deploy.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -656,10 +664,12 @@ We welcome community contributions, see +
@@ -681,12 +691,7 @@ We welcome community contributions, see Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -734,9 +739,9 @@ We welcome community contributions, see
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/auto_deploy/support_matrix.html b/latest/features/auto_deploy/support_matrix.html index 624198c15f..6156140473 100644 --- a/latest/features/auto_deploy/support_matrix.html +++ b/latest/features/auto_deploy/support_matrix.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,20 +61,24 @@ + + + - + + @@ -80,6 +86,8 @@ + +
@@ -722,10 +730,12 @@ In addition, the following models have been officially validated using the defau
+ +
@@ -747,12 +757,7 @@ In addition, the following models have been officially validated using the defau - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -800,9 +805,9 @@ In addition, the following models have been officially validated using the defau
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/checkpoint-loading.html b/latest/features/checkpoint-loading.html index 6b8d6c3c06..4596d00396 100644 --- a/latest/features/checkpoint-loading.html +++ b/latest/features/checkpoint-loading.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -918,10 +926,12 @@ Likewise, if the format shares some components with an already supported framewo
+ +
@@ -943,12 +953,7 @@ Likewise, if the format shares some components with an already supported framewo - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -996,9 +1001,9 @@ Likewise, if the format shares some components with an already supported framewo
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/disagg-serving.html b/latest/features/disagg-serving.html index a826b5af7a..62550c4bff 100644 --- a/latest/features/disagg-serving.html +++ b/latest/features/disagg-serving.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -818,10 +826,12 @@ when routing requests to the generation servers, the disaggregated server will m
+ +
@@ -843,12 +853,7 @@ when routing requests to the generation servers, the disaggregated server will m - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -896,9 +901,9 @@ when routing requests to the generation servers, the disaggregated server will m
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/feature-combination-matrix.html b/latest/features/feature-combination-matrix.html index 79bf402430..5d9b5cee89 100644 --- a/latest/features/feature-combination-matrix.html +++ b/latest/features/feature-combination-matrix.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -835,10 +843,12 @@
+ +
@@ -860,12 +870,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -913,9 +918,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/kvcache.html b/latest/features/kvcache.html index 16f3da8baa..4ba059e3a2 100644 --- a/latest/features/kvcache.html +++ b/latest/features/kvcache.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -667,10 +675,12 @@
+ +
@@ -692,12 +702,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -745,9 +750,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/long-sequence.html b/latest/features/long-sequence.html index 442e83f267..6d09ff062f 100644 --- a/latest/features/long-sequence.html +++ b/latest/features/long-sequence.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -636,10 +644,12 @@
+ +
@@ -661,12 +671,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -714,9 +719,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/lora.html b/latest/features/lora.html index 7b6163116c..c9951e1757 100644 --- a/latest/features/lora.html +++ b/latest/features/lora.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -840,10 +848,12 @@
+ +
@@ -865,12 +875,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -918,9 +923,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/multi-modality.html b/latest/features/multi-modality.html index 1119a63c75..5672c457fe 100644 --- a/latest/features/multi-modality.html +++ b/latest/features/multi-modality.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -538,8 +546,8 @@

The following examples demonstrate how to use TensorRT LLM’s multimodal support in various scenarios, including quick run examples, serving endpoints, and performance benchmarking.

Quick start#

-

Quickly try out TensorRT LLM’s multimodal support using our LLM-API and a ready-to-run example:

-
python3 quickstart_multimodal.py --model_dir Efficient-Large-Model/NVILA-8B --modality image
+
Quickly try out TensorRT LLM’s multimodal support using our LLM-API and a ready-to-run example:

+
python3 quickstart_multimodal.py --model_dir Efficient-Large-Model/NVILA-8B --modality image --disable_kv_cache_reuse
@@ -549,7 +557,7 @@ 
trtllm-serve Qwen/Qwen2-VL-7B-Instruct  --backend pytorch
-

You can then send OpenAI-compatible requests, such as via curl or API clients, to the server endpoint. See curl chat client for multimodal script as an example.

+

You can then send OpenAI-compatible requests, such as via curl or API clients, to the server endpoint. See curl chat client for multimodal script as an example.

Run with trtllm-bench#

@@ -632,10 +640,12 @@
+ +
@@ -657,12 +667,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -710,9 +715,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/overlap-scheduler.html b/latest/features/overlap-scheduler.html index 40063d65d4..5462b96eec 100644 --- a/latest/features/overlap-scheduler.html +++ b/latest/features/overlap-scheduler.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -614,10 +622,12 @@
+ +
@@ -639,12 +649,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -692,9 +697,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/paged-attention-ifb-scheduler.html b/latest/features/paged-attention-ifb-scheduler.html index 3b95d55647..f57cd9aff5 100644 --- a/latest/features/paged-attention-ifb-scheduler.html +++ b/latest/features/paged-attention-ifb-scheduler.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -594,9 +602,9 @@ different types of KV caches: contiguous and pagedThe paged KV cache decomposes the KV cache into blocks that are distributed to the different requests by a cache manager during processing. That cache manager keeps track of the sequences, allocates new blocks from a pool, and recycles those blocks when required. See the simplified implementation of -tensorrt_llm.runtime.KVCacheManager. +tensorrt_llm.runtime.KVCacheManager. A more efficient C++ implementation is included in the -Batch Manager.

+Batch Manager.

@@ -710,10 +718,12 @@ A more efficient C++ implementation is included in the + +
@@ -735,12 +745,7 @@ A more efficient C++ implementation is included in the - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -788,9 +793,9 @@ A more efficient C++ implementation is included in the
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/parallel-strategy.html b/latest/features/parallel-strategy.html index 5de9c77b78..a720a741dc 100644 --- a/latest/features/parallel-strategy.html +++ b/latest/features/parallel-strategy.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -819,10 +827,12 @@
+ +
@@ -844,12 +854,7 @@ - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -897,9 +902,9 @@
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/quantization.html b/latest/features/quantization.html index 1f851197c9..2363663cbc 100644 --- a/latest/features/quantization.html +++ b/latest/features/quantization.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -1001,10 +1009,12 @@ The language component decides which quantization methods are supported by a giv
+ +
@@ -1026,12 +1036,7 @@ The language component decides which quantization methods are supported by a giv - Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -1079,9 +1084,9 @@ The language component decides which quantization methods are supported by a giv
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/ray-orchestrator.html b/latest/features/ray-orchestrator.html index e497eb0539..d68e75b024 100644 --- a/latest/features/ray-orchestrator.html +++ b/latest/features/ray-orchestrator.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -623,10 +631,12 @@ pip install -r
+ +
@@ -648,12 +658,7 @@ pip install -r Manage My Privacy - | - - - - Do Not Sell or Share My Data + Your Privacy Choices | @@ -701,9 +706,9 @@ pip install -r
-

Last updated on November 05, 2025.

+

Last updated on November 20, 2025.

-

This page is generated by TensorRT-LLM commit 3111682.

+

This page is generated by TensorRT-LLM commit 2128f73.

diff --git a/latest/features/sampling.html b/latest/features/sampling.html index d25e8f6f8f..6e611c8c0d 100644 --- a/latest/features/sampling.html +++ b/latest/features/sampling.html @@ -32,7 +32,7 @@ - + @@ -44,6 +44,8 @@ + + @@ -59,22 +61,26 @@ + + + - + + @@ -82,6 +88,8 @@ + +
@@ -514,7 +522,7 @@

To use the feature:

  1. Enable the enable_trtllm_sampler option in the LLM class

    2. -

  2. Pass a SamplingParams object with the desired options to the generate() function

    3. +

  3. Pass a SamplingParams object with the desired options to the generate() function

The following example prepares two identical prompts which will give different results due to the sampling parameters chosen:

from tensorrt_llm import LLM, SamplingParams
@@ -579,7 +587,7 @@
 
To enable guided decoding, you must:

 
    
 
  1. Set the guided_decoding_backend parameter to 'xgrammar' or 'llguidance' in the LLM class
    2. 
-
  2. Create a GuidedDecodingParams object with the desired format specification
    
+
  3. Create a GuidedDecodingParams object with the desired format specification
    
 
      
 
    • Note: Depending on the type of format, a different parameter needs to be chosen to construct the object (json, regex, grammar, structural_tag).
      • 
 
    
@@ -600,14 +608,14 @@
 llm.generate("Generate a JSON response", sampling_params)
-

You can find a more detailed example on guided decoding here.

+

You can find a more detailed example on guided decoding here.

Logits processor#

Logits processors allow you to modify the logits produced by the network before sampling, enabling custom generation behavior and constraints.

To use a custom logits processor:

    -

  1. Create a custom class that inherits from LogitsProcessor and implements the __call__ method

    2. +

  2. Create a custom class that inherits from LogitsProcessor and implements the __call__ method

  3. Pass an instance of this class to the logits_processor parameter of SamplingParams

The following example demonstrates logits processing:

@@ -635,7 +643,7 @@ llm.generate(["Hello, my name is"], sampling_params) -

You can find a more detailed example on logits processors here.

+

You can find a more detailed example on logits processors here.

@@ -708,10 +716,12 @@ + +
@@ -641,10 +658,12 @@ The setup methods depends on your slurm configuration, pls check with your admin + +