@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c01b8022c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix broken internvl2 inference with v1 (#12360)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jan 24, 2025 at 01:20 UTC 8c01b8022c78d583394509f8cfa2af4e7ca79279
1 file changed +7 -2
vllm/multimodal/utils.py
+7 -2
@@ -503,8 +503,13 @@ def group_mm_inputs_by_modality(
503 if len(mm_input.modalities) > 1:
504 return id(mm_input)
505
506 - # Otherwise return the modality string
507 - return list(mm_input.modalities)[0]
506 + elif len(mm_input.modalities) == 1:
507 + return list(mm_input.modalities)[0]
508 +
509 + # FIXME(Isotr0py): Modality of mm_input from legacy pipeline is empty,
510 + # this is used to make InternVL with legacy pipeline still work with v1.
511 + else:
512 + return ""
513
514 return [
515 list(group) for _, group in groupby(mm_inputs, key=modality_group_func)