A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 02:15 UTC
[Misc] `toy_proxy_server` handle min_tokens (#39706)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Apr 16, 2026 at 17:08 UTC
3daca38e2279538b420641bd41853c19e5ad01e4
1 file changed
+7
tests/v1/kv_connector/nixl_integration/toy_proxy_server.py
+7
@@ -173,6 +173,9 @@ async def send_request_to_service(
173
req_data["max_completion_tokens"] = 1
174
if "stream_options" in req_data:
175
del req_data["stream_options"]
176
+ # These args are not supported for P
177
+ min_tokens = req_data.pop("min_tokens", None)
178
+ min_completion_tokens = req_data.pop("min_completion_tokens", None)
179
headers = {
180
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
181
"X-Request-Id": request_id,
@@ -187,6 +190,10 @@ async def send_request_to_service(
190
# otherwise, it would http.ReadError
191
await response.aread()
192
193
+ # Add back the min_tokens and min_completion_tokens so D can use them
194
+ req_data["min_tokens"] = min_tokens
195
+ req_data["min_completion_tokens"] = min_completion_tokens
196
+
197
return response
198
199