[Disagg] return routed_experts on streaming generate responses (#44638)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Roger Wang <hey@rogerw.io>
aoshen02 committed
Jun 21, 2026 at 22:37 UTC
3e6e33526da729fbe30ebec86be9049e1899ce67
2 files changed
+10
vllm/entrypoints/serve/disagg/protocol.py
+1
@@ -181,6 +181,7 @@ class GenerateResponseStreamChoice(BaseModel):
181
logprobs: ChatCompletionLogProbs | None = None
182
finish_reason: str | None = None
183
token_ids: list[int] | None = None
184
+ routed_experts: str | None = None
185
186
187
class GenerateStreamResponse(BaseModel):
vllm/entrypoints/serve/disagg/serving.py
+9
@@ -400,6 +400,14 @@ class ServingTokens(OpenAIServing):
400
else:
401
logprobs = None
402
403
+ routed_experts_b64 = None
404
+ if output.routed_experts is not None:
405
+ buf = io.BytesIO()
406
+ np.save(buf, output.routed_experts)
407
+ routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
408
+ "ascii"
409
+ )
410
+
411
chunk = GenerateStreamResponse(
412
request_id=request_id,
413
choices=[
@@ -408,6 +416,7 @@ class ServingTokens(OpenAIServing):
416
logprobs=logprobs,
417
finish_reason=finish_reason,
418
token_ids=as_list(delta_token_ids),
419
+ routed_experts=routed_experts_b64,
420
)
421
],
422
)