mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-14 06:27:45 +08:00
c6e2111f4e
* feat: enhance trtllm serve multimodal 1. made the load_image and load_video asynchronous 2. add image_encoded input support to be compatible with genai-perf 3. support text-only on multimodal mdoels(currently, Qwen2-VL & Qwen2.5-VL) Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * add test Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * fix bandit Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * trimming uils Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * trimming for test Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * genai perf command fix Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * command fix Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * refactor chat_utils Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> * stress test genai-perf command Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com> --------- Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com>
115 lines
2.9 KiB
Python
115 lines
2.9 KiB
Python
### OpenAI Chat Client
|
|
|
|
from openai import OpenAI
|
|
|
|
from tensorrt_llm.inputs import encode_base64_content_from_url
|
|
|
|
client = OpenAI(
|
|
base_url="http://localhost:8000/v1",
|
|
api_key="tensorrt_llm",
|
|
)
|
|
|
|
# SINGLE IMAGE INFERENCE
|
|
response = client.chat.completions.create(
|
|
model="Qwen2.5-VL-3B-Instruct",
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "you are a helpful assistant"
|
|
}, {
|
|
"role":
|
|
"user",
|
|
"content": [{
|
|
"type": "text",
|
|
"text": "Describe the natural environment in the image."
|
|
}, {
|
|
"type": "image_url",
|
|
"image_url": {
|
|
"url":
|
|
"https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/seashore.png"
|
|
}
|
|
}]
|
|
}],
|
|
max_tokens=64,
|
|
)
|
|
print(response)
|
|
|
|
# MULTI IMAGE INFERENCE
|
|
response = client.chat.completions.create(
|
|
model="Qwen2.5-VL-3B-Instruct",
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "you are a helpful assistant"
|
|
}, {
|
|
"role":
|
|
"user",
|
|
"content": [{
|
|
"type": "text",
|
|
"text": "Tell me the difference between two images"
|
|
}, {
|
|
"type": "image_url",
|
|
"image_url": {
|
|
"url":
|
|
"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"
|
|
}
|
|
}, {
|
|
"type": "image_url",
|
|
"image_url": {
|
|
"url":
|
|
"https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/seashore.png"
|
|
}
|
|
}]
|
|
}],
|
|
max_tokens=64,
|
|
)
|
|
print(response)
|
|
|
|
# SINGLE VIDEO INFERENCE
|
|
response = client.chat.completions.create(
|
|
model="Qwen2.5-VL-3B-Instruct",
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "you are a helpful assistant"
|
|
}, {
|
|
"role":
|
|
"user",
|
|
"content": [{
|
|
"type": "text",
|
|
"text": "Tell me what you see in the video briefly."
|
|
}, {
|
|
"type": "video_url",
|
|
"video_url": {
|
|
"url":
|
|
"https://huggingface.co/datasets/Efficient-Large-Model/VILA-inference-demos/resolve/main/OAI-sora-tokyo-walk.mp4"
|
|
}
|
|
}]
|
|
}],
|
|
max_tokens=64,
|
|
)
|
|
print(response)
|
|
|
|
# IMAGE EMBED INFERENCE
|
|
image64 = encode_base64_content_from_url(
|
|
"https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/seashore.png"
|
|
)
|
|
response = client.chat.completions.create(
|
|
model="Qwen2.5-VL-3B-Instruct",
|
|
messages=[{
|
|
"role": "system",
|
|
"content": "you are a helpful assistant"
|
|
}, {
|
|
"role":
|
|
"user",
|
|
"content": [{
|
|
"type": "text",
|
|
"text": "Describe the natural environment in the image."
|
|
}, {
|
|
"type": "image_url",
|
|
"image_url": {
|
|
"url": "data:image/png;base64," + image64
|
|
}
|
|
}]
|
|
}],
|
|
max_tokens=64,
|
|
)
|
|
print(response)
|