A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 14:45 UTC
[BugFix] Fix duplicate id tool-call race condition (#29355)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Nov 24, 2025 at 17:54 UTC
a178a0b40b50bf448ab50a853b7eb1744af18f31
2 files changed
+16
-7
vllm/entrypoints/openai/serving_chat.py
+9
-4
@@ -273,6 +273,11 @@ class OpenAIServingChat(OpenAIServing):
273
try:
274
for i, engine_prompt in enumerate(engine_prompts):
275
prompt_text, _, _ = self._get_prompt_components(request_prompts[i])
276
+ # If we are creating sub requests for multiple prompts, ensure that they
277
+ # have unique request ids.
278
+ sub_request_id = (
279
+ request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
280
+ )
281
282
if self.default_sampling_params is None:
283
self.default_sampling_params = {}
@@ -301,7 +306,7 @@ class OpenAIServingChat(OpenAIServing):
306
)
307
308
self._log_inputs(
304
- request_id,
309
+ sub_request_id,
310
request_prompts[i],
311
params=sampling_params,
312
lora_request=lora_request,
@@ -316,14 +321,14 @@ class OpenAIServingChat(OpenAIServing):
321
if isinstance(sampling_params, BeamSearchParams):
322
generator = self.beam_search(
323
prompt=engine_prompt,
319
- request_id=request_id,
324
+ request_id=sub_request_id,
325
params=sampling_params,
326
lora_request=lora_request,
327
trace_headers=trace_headers,
328
)
329
else:
330
engine_request, tokenization_kwargs = await self._process_inputs(
326
- request_id,
331
+ sub_request_id,
332
engine_prompt,
333
sampling_params,
334
lora_request=lora_request,
@@ -334,7 +339,7 @@ class OpenAIServingChat(OpenAIServing):
339
generator = self.engine_client.generate(
340
engine_request,
341
sampling_params,
337
- request_id,
342
+ sub_request_id,
343
lora_request=lora_request,
344
trace_headers=trace_headers,
345
priority=request.priority,
vllm/entrypoints/openai/serving_engine.py
+7
-3
@@ -1242,16 +1242,19 @@ class OpenAIServing:
1242
):
1243
prompt_text, _, _ = self._get_prompt_components(request_prompt)
1244
orig_priority = priority
1245
+ sub_request = 0
1246
while True:
1247
+ # Ensure that each sub-request has a unique request id.
1248
+ sub_request_id = f"{request_id}_{sub_request}"
1249
self._log_inputs(
1247
- request_id,
1250
+ sub_request_id,
1251
request_prompt,
1252
params=sampling_params,
1253
lora_request=lora_request,
1254
)
1255
trace_headers = kwargs.get("trace_headers")
1256
engine_request, tokenization_kwargs = await self._process_inputs(
1254
- request_id,
1257
+ sub_request_id,
1258
engine_prompt,
1259
sampling_params,
1260
lora_request=lora_request,
@@ -1262,7 +1265,7 @@ class OpenAIServing:
1265
generator = self.engine_client.generate(
1266
engine_request,
1267
sampling_params,
1265
- request_id,
1268
+ sub_request_id,
1269
lora_request=lora_request,
1270
priority=priority,
1271
prompt_text=prompt_text,
@@ -1295,6 +1298,7 @@ class OpenAIServing:
1298
sampling_params.max_tokens = self.max_model_len - len(prompt_token_ids)
1299
# OPTIMIZATION
1300
priority = orig_priority - 1
1301
+ sub_request += 1
1302
1303
def _get_prompt_components(
1304
self,