mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-14 06:27:45 +08:00
f25c7cefb4
Signed-off-by: Pengyun Lin <81065165+LinPoly@users.noreply.github.com> Signed-off-by: Kaiyu Xie <26294424+kaiyux@users.noreply.github.com> Co-authored-by: Kaiyu Xie <26294424+kaiyux@users.noreply.github.com>
22 lines
430 B
Python
22 lines
430 B
Python
### OpenAI Chat Client
|
|
|
|
from openai import OpenAI
|
|
|
|
client = OpenAI(
|
|
base_url="http://localhost:8000/v1",
|
|
api_key="tensorrt_llm",
|
|
)
|
|
|
|
response = client.chat.completions.create(
|
|
model="TinyLlama-1.1B-Chat-v1.0",
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "you are a helpful assistant"
|
|
}, {
|
|
"role": "user",
|
|
"content": "Where is New York?"
|
|
}],
|
|
max_tokens=20,
|
|
)
|
|
print(response)
|