A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Fix profiling for models with merged input processor (#11370)
Signed-off-by: ywang96 <ywang@roblox.com>
Roger Wang committed
Dec 20, 2024 at 04:04 UTC
04139ade599eedd493ce8effcda7ceabb57f2fb5
1 file changed
+32
-12
vllm/v1/worker/gpu_model_runner.py
+32
-12
@@ -635,17 +635,6 @@ class GPUModelRunner:
635
)
636
dummy_mm_data = dummy_request_data.multi_modal_data
637
638
- # Compute MM hashes (if enabled)
639
- mm_hashes = None
640
- if self.use_hash:
641
- mm_hashes = self.mm_hasher.hash_dummy_mm_data(dummy_mm_data)
642
-
643
- dummy_mm_kwargs = self.mm_input_mapper_client.process_inputs(
644
- mm_data=dummy_mm_data,
645
- mm_hashes=mm_hashes,
646
- mm_processor_kwargs=None,
647
- precomputed_mm_inputs=None)
648
-
638
# NOTE: Currently model is profiled with a single non-text
639
# modality even when it supports multiple.
640
max_tokens_per_mm_item = max(
@@ -660,8 +649,39 @@ class GPUModelRunner:
649
# (e.g, multiple images) for a single request, therefore here we
650
# always replicate first item by max_num_mm_items times since in V1
651
# they are scheduled to be processed separately.
652
+
653
+ # Case when models have a merged processor, their dummy data is
654
+ # already batched `MultiModalKwargs`, therefore we need to "unbatch"
655
+ # and take the first item in each batched tensor.
656
+ # TODO (ywang96): This is somewhat hacky. Refactor this to be
657
+ # consistent with the other case.
658
+ if isinstance(dummy_mm_data, MultiModalKwargs):
659
+ dummy_mm_kwargs = {
660
+ k: v[0].unsqueeze(0)
661
+ for k, v in dummy_mm_data.items()
662
+ }
663
+
664
+ # Case when models have dummy data explicitly defined as
665
+ # `MultiModalDataDict`, so they need to be processed through input
666
+ # mapper.
667
+ else:
668
+ # Compute MM hashes (if enabled)
669
+ mm_hashes = None
670
+ if self.use_hash:
671
+ mm_hashes = self.mm_hasher.hash_dummy_mm_data(
672
+ dummy_mm_data)
673
+
674
+ mm_kwargs_list = self.mm_input_mapper_client.process_inputs(
675
+ mm_data=dummy_mm_data,
676
+ mm_hashes=mm_hashes,
677
+ mm_processor_kwargs=None,
678
+ precomputed_mm_inputs=None)
679
+
680
+ # Take the first `MultiModalKwargs`
681
+ dummy_mm_kwargs = mm_kwargs_list[0]
682
+
683
batched_dummy_mm_inputs = MultiModalKwargs.batch(
664
- [dummy_mm_kwargs[0]] * max_num_mm_items)
684
+ [dummy_mm_kwargs] * max_num_mm_items)
685
batched_dummy_mm_inputs = MultiModalKwargs.as_kwargs(
686
batched_dummy_mm_inputs, device=self.device)
687