[Bugfix] Thread kv_transfer_params into engine for /inference/v1/generate (disagg) (#42644)
Signed-off-by: hallerite <git@hallerite.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Konstantin Dunas committed
Aug 26, 2026 at 08:42 UTC
c71f6f8a81d3d3c49a045c8b88eed36366cc7d92
1 file changed
+10
vllm/entrypoints/scale_out/token_in_token_out/serving.py
+10
@@ -203,6 +203,16 @@ class ServingTokens(GenerateBaseServing):
203
# Schedule the request and get the result generator.
204
result_generator: AsyncGenerator[RequestOutput, None] | None = None
205
206
+ # Pass disaggregated-serving parameters through to the engine.
207
+ if request.kv_transfer_params is not None:
208
+ extra = sampling_params.extra_args or {}
209
+ extra["kv_transfer_params"] = request.kv_transfer_params
210
+ sampling_params.extra_args = extra
211
+ if request.ec_transfer_params is not None:
212
+ extra = sampling_params.extra_args or {}
213
+ extra["ec_transfer_params"] = request.ec_transfer_params
214
+ sampling_params.extra_args = extra
215
+
216
# Apply server-side ``max_tokens`` defaulting when the client did
217
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
218
# defaults ``max_tokens`` to 16, which would otherwise silently cap