A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 23:15 UTC
[bugfix] Missing cached item in beam search (#27874)
Signed-off-by: fake0fan <645327136@qq.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Chenguang Zheng committed
Nov 1, 2025 at 01:34 UTC
103a468bbfd3d848cdfa5845909500d58a43119a
1 file changed
+10
-18
vllm/entrypoints/openai/serving_engine.py
+10
-18
@@ -345,22 +345,7 @@ class OpenAIServing:
345
346
if is_explicit_encoder_decoder_prompt(prompt):
347
raise NotImplementedError
348
- else:
349
- processed_inputs = processor.input_preprocessor._prompt_to_llm_inputs(
350
- prompt
351
- )
352
-
353
- if processed_inputs["type"] == "embeds":
354
- raise NotImplementedError
348
356
- # This is a workaround to fix multimodal beam search; this is a
357
- # bandaid fix for 2 small problems:
358
- # 1. Multi_modal_data on the processed_inputs currently resolves to
359
- # `None`.
360
- # 2. preprocessing above expands the multimodal placeholders. However,
361
- # this happens again in generation, so the double expansion causes
362
- # a mismatch.
363
- # TODO - would be ideal to handle this more gracefully.
349
prompt_text: str | None
350
prompt_token_ids: list[int]
351
multi_modal_data: MultiModalDataDict | None
@@ -373,9 +358,16 @@ class OpenAIServing:
358
prompt_token_ids = prompt.get("prompt_token_ids", []) # type: ignore
359
multi_modal_data = prompt.get("multi_modal_data") # type: ignore
360
376
- mm_processor_kwargs: dict[str, Any] | None = processed_inputs.get(
377
- "mm_processor_kwargs"
378
- ) # type: ignore
361
+ mm_processor_kwargs: dict[str, Any] | None = None
362
+
363
+ # This is a workaround to fix multimodal beam search; this is a
364
+ # bandaid fix for 2 small problems:
365
+ # 1. Multi_modal_data on the processed_inputs currently resolves to
366
+ # `None`.
367
+ # 2. preprocessing above expands the multimodal placeholders. However,
368
+ # this happens again in generation, so the double expansion causes
369
+ # a mismatch.
370
+ # TODO - would be ideal to handle this more gracefully.
371
372
tokenized_length = len(prompt_token_ids)
373