A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix broken internvl2 inference with v1 (#12360)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jan 24, 2025 at 01:20 UTC
8c01b8022c78d583394509f8cfa2af4e7ca79279
1 file changed
+7
-2
vllm/multimodal/utils.py
+7
-2
@@ -503,8 +503,13 @@ def group_mm_inputs_by_modality(
503
if len(mm_input.modalities) > 1:
504
return id(mm_input)
505
506
- # Otherwise return the modality string
507
- return list(mm_input.modalities)[0]
506
+ elif len(mm_input.modalities) == 1:
507
+ return list(mm_input.modalities)[0]
508
+
509
+ # FIXME(Isotr0py): Modality of mm_input from legacy pipeline is empty,
510
+ # this is used to make InternVL with legacy pipeline still work with v1.
511
+ else:
512
+ return ""
513
514
return [
515
list(group) for _, group in groupby(mm_inputs, key=modality_group_func)