A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] Fix phi3v incorrect image_idx when using async engine (#7916)
Isotr0py committed
Aug 28, 2024 at 01:36 UTC
b09c755be89edaaca7c9e010f423545f0cd014b4
1 file changed
+3
-1
vllm/model_executor/models/phi3v.py
+3
-1
@@ -422,7 +422,9 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
422
423
prompt = llm_inputs.get("prompt")
424
if prompt is None:
425
- image_idx = []
425
+ # for async server request, we assume prompt and its token_ids is always
426
+ # in correct format. And num_image_tags == len(image_data) always True.
427
+ image_idx = range(1, len(image_data) + 1)
428
new_prompt = None
429
else:
430
image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt)))