kanshan/TensorRT-LLMs
1
0
Fork 0
mirror of https://github.com/NVIDIA/TensorRT-LLM.git synced 2026-01-14 06:27:45 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

[TRTLLM-8413][chore] resolve sampling defaults in OpenAI API backend (#8121)

Browse Source 
Signed-off-by: ixlmar <206748156+ixlmar@users.noreply.github.com>
This commit is contained in:
mpikulski 2025-10-06 15:09:43 +02:00 committed by GitHub
parent 54ab9767b5
commit 98b3af4d4e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tensorrt_llm/serve/openai_protocol.py
View File

@ -220,8 +220,8 @@ class CompletionRequest(OpenAIBaseModel):
stream: Optional[bool] = False
stream_options: Optional[StreamOptions] = None
suffix: Optional[str] = None
temperature: Optional[float] = 1.0
top_p: Optional[float] = 1.0
temperature: Optional[float] = None
top_p: Optional[float] = None
user: Optional[str] = None
lora_request: Optional[LoRARequest] = None
@ -275,8 +275,9 @@ class CompletionRequest(OpenAIBaseModel):
presence_penalty=self.presence_penalty,
seed=self.seed,
stop=self.stop,
temperature=self.temperature,
top_p=self.top_p,
temperature=(self.temperature
if self.temperature is not None else 1.0),
top_p=(self.top_p if self.top_p is not None else 1.0),
# completion-sampling-params
use_beam_search=self.use_beam_search,
@ -510,8 +511,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
stream: Optional[bool] = False
stream_options: Optional[StreamOptions] = None
temperature: Optional[float] = 1.0
top_p: Optional[float] = 1.0
temperature: Optional[float] = None
top_p: Optional[float] = None
tools: Optional[List[ChatCompletionToolsParam]] = None
tool_choice: Optional[Union[Literal["none", "auto"],
ChatCompletionNamedToolChoiceParam]] = "none"
@ -614,13 +615,14 @@ class ChatCompletionRequest(OpenAIBaseModel):
presence_penalty=self.presence_penalty,
seed=self.seed,
stop=self.stop,
temperature=self.temperature,
temperature=(self.temperature
if self.temperature is not None else 1.0),
# chat-completion-sampling-params
best_of=self.best_of,
use_beam_search=self.use_beam_search,
top_k=self.top_k,
top_p=self.top_p,
top_p=(self.top_p if self.top_p is not None else 1.0),
top_p_min=self.top_p_min if self.top_p_min > 0 else None,
min_p=self.min_p,
repetition_penalty=self.repetition_penalty,