From 3e6e33526da729fbe30ebec86be9049e1899ce67 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Sun, 21 Jun 2026 22:37:10 +0800 Subject: [PATCH] [Disagg] return routed_experts on streaming generate responses (#44638) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Roger Wang --- vllm/entrypoints/serve/disagg/protocol.py | 1 + vllm/entrypoints/serve/disagg/serving.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/vllm/entrypoints/serve/disagg/protocol.py b/vllm/entrypoints/serve/disagg/protocol.py index 7e776ae7178..2e98f5e811c 100644 --- a/vllm/entrypoints/serve/disagg/protocol.py +++ b/vllm/entrypoints/serve/disagg/protocol.py @@ -181,6 +181,7 @@ class GenerateResponseStreamChoice(BaseModel): logprobs: ChatCompletionLogProbs | None = None finish_reason: str | None = None token_ids: list[int] | None = None + routed_experts: str | None = None class GenerateStreamResponse(BaseModel): diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/serve/disagg/serving.py index 0bb29c68d01..5031627ea01 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/serve/disagg/serving.py @@ -400,6 +400,14 @@ class ServingTokens(OpenAIServing): else: logprobs = None + routed_experts_b64 = None + if output.routed_experts is not None: + buf = io.BytesIO() + np.save(buf, output.routed_experts) + routed_experts_b64 = base64.b64encode(buf.getvalue()).decode( + "ascii" + ) + chunk = GenerateStreamResponse( request_id=request_id, choices=[ @@ -408,6 +416,7 @@ class ServingTokens(OpenAIServing): logprobs=logprobs, finish_reason=finish_reason, token_ids=as_list(delta_token_ids), + routed_experts=routed_experts_b64, ) ], )