A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[Bugfix][PD] set max_completion_tokens=1 if req has this value (#21841)
Signed-off-by: Abirdcfly <fp544037857@gmail.com>
Abirdcfly committed
Aug 1, 2025 at 21:08 UTC
5c54d9759d3e12d66919826bf1b7c196914d3a92
2 files changed
+4
examples/online_serving/disaggregated_serving/disagg_proxy_demo.py
+2
@@ -293,6 +293,8 @@ class Proxy:
293
# add params to request
294
kv_prepare_request = request.copy()
295
kv_prepare_request["max_tokens"] = 1
296
+ if "max_completion_tokens" in kv_prepare_request:
297
+ kv_prepare_request["max_completion_tokens"] = 1
298
299
# prefill stage
300
prefill_instance = self.schedule(self.prefill_cycler)
examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py
+2
@@ -128,6 +128,8 @@ async def handle_request():
128
prefill_request = original_request_data.copy()
129
# change max_tokens = 1 to let it only do prefill
130
prefill_request["max_tokens"] = 1
131
+ if "max_completion_tokens" in prefill_request:
132
+ prefill_request["max_completion_tokens"] = 1
133
134
global count
135
global prefill_instances