mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-27 22:23:25 +08:00
358 lines
9.4 KiB
ReStructuredText
358 lines
9.4 KiB
ReStructuredText
API Reference
|
|
-------------
|
|
|
|
.. note::
|
|
Since version 1.0, we have attached a status label to `LLM`, `LlmArgs` and `TorchLlmArgs` Classes.
|
|
|
|
1. :tag:`stable` - The item is stable and will keep consistent.
|
|
2. :tag:`prototype` - The item is a prototype and is subject to change.
|
|
3. :tag:`beta` - The item is in beta and approaching stability.
|
|
4. :tag:`deprecated` - The item is deprecated and will be removed in a future release.
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.LLM
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.AsyncLLM
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.MultimodalEncoder
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.CompletionOutput
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.RequestOutput
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.GuidedDecodingParams
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.SamplingParams
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.DisaggregatedParams
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.KvCacheConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.KvCacheRetentionConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.CudaGraphConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.MoeConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.LookaheadDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.MedusaDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.EagleDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.MTPDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.SchedulerConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.CapacitySchedulerPolicy
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.BuildConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.QuantConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.QuantAlgo
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.CalibConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.BuildCacheConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.RequestError
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.MpiCommSession
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.ExtendedRuntimePerfKnobConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.BatchingType
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.ContextChunkingPolicy
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.DynamicBatchConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.CacheTransceiverConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.NGramDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.UserProvidedDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.TorchCompileConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.DraftTargetDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.LlmArgs
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.TorchLlmArgs
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
:exclude-members: model_rebuild,model_fields_set,parse_obj,model_post_init,model_fields,validate,from_orm,update_forward_refs,model_dump_json,model_dump,parse_file,model_json_schema,model_parametrized_name,json,model_validate,model_config,model_copy,model_construct,parse_raw,model_validate_json,dict,construct,schema,copy,model_validate_strings,model_computed_fields,model_extra,schema_json
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.TrtLlmArgs
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
:exclude-members: model_rebuild,model_fields_set,parse_obj,model_post_init,model_fields,validate,from_orm,update_forward_refs,model_dump_json,model_dump,parse_file,model_json_schema,model_parametrized_name,json,model_validate,model_config,model_copy,model_construct,parse_raw,model_validate_json,dict,construct,schema,copy,model_validate_strings,model_computed_fields,model_extra,schema_json
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.AutoDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.AttentionDpConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.LoRARequest
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.SaveHiddenStatesDecodingConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.RocketSparseAttentionConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|
|
.. autoclass:: tensorrt_llm.llmapi.DeepSeekSparseAttentionConfig
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
:special-members: __init__
|
|
:member-order: groupwise
|
|
:inherited-members:
|
|
|