@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b09c755be8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] Fix phi3v incorrect image_idx when using async engine (#7916)

Isotr0py committed Aug 28, 2024 at 01:36 UTC b09c755be89edaaca7c9e010f423545f0cd014b4
1 file changed +3 -1
vllm/model_executor/models/phi3v.py
+3 -1
@@ -422,7 +422,9 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
422
423 prompt = llm_inputs.get("prompt")
424 if prompt is None:
425 - image_idx = []
425 + # for async server request, we assume prompt and its token_ids is always
426 + # in correct format. And num_image_tags == len(image_data) always True.
427 + image_idx = range(1, len(image_data) + 1)
428 new_prompt = None
429 else:
430 image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt)))