obscura/vllm
2
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2026-08-22 21:50:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Disagg] return routed_experts on streaming generate responses (#44638)

Browse Source 
Signed-off-by: aoshen02 <[email protected]>
Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]>
Co-authored-by: Roger Wang <[email protected]>
This commit is contained in:
aoshen02
2026-06-21 07:37:10 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Roger Wang
parent b91b7726e0
commit 3e6e33526d
2 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/entrypoints/serve/disagg/protocol.py
+1
View File
@@ -181,6 +181,7 @@ class GenerateResponseStreamChoice(BaseModel):
logprobs: ChatCompletionLogProbs | None = None
finish_reason: str | None = None
token_ids: list[int] | None = None
routed_experts: str | None = None
class GenerateStreamResponse(BaseModel):
vllm/entrypoints/serve/disagg/serving.py
+9
View File
@@ -400,6 +400,14 @@ class ServingTokens(OpenAIServing):
else:
logprobs = None
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
chunk = GenerateStreamResponse(
request_id=request_id,
choices=[
@@ -408,6 +416,7 @@ class ServingTokens(OpenAIServing):
logprobs=logprobs,
finish_reason=finish_reason,
token_ids=as_list(delta_token_ids),
routed_experts=routed_experts_b64,
)
],
)