@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04139ade59
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Fix profiling for models with merged input processor (#11370)

Signed-off-by: ywang96 <ywang@roblox.com>

Roger Wang committed Dec 20, 2024 at 04:04 UTC 04139ade599eedd493ce8effcda7ceabb57f2fb5
1 file changed +32 -12
vllm/v1/worker/gpu_model_runner.py
+32 -12
@@ -635,17 +635,6 @@ class GPUModelRunner:
635 )
636 dummy_mm_data = dummy_request_data.multi_modal_data
637
638 - # Compute MM hashes (if enabled)
639 - mm_hashes = None
640 - if self.use_hash:
641 - mm_hashes = self.mm_hasher.hash_dummy_mm_data(dummy_mm_data)
642 -
643 - dummy_mm_kwargs = self.mm_input_mapper_client.process_inputs(
644 - mm_data=dummy_mm_data,
645 - mm_hashes=mm_hashes,
646 - mm_processor_kwargs=None,
647 - precomputed_mm_inputs=None)
648 -
638 # NOTE: Currently model is profiled with a single non-text
639 # modality even when it supports multiple.
640 max_tokens_per_mm_item = max(
@@ -660,8 +649,39 @@ class GPUModelRunner:
649 # (e.g, multiple images) for a single request, therefore here we
650 # always replicate first item by max_num_mm_items times since in V1
651 # they are scheduled to be processed separately.
652 +
653 + # Case when models have a merged processor, their dummy data is
654 + # already batched `MultiModalKwargs`, therefore we need to "unbatch"
655 + # and take the first item in each batched tensor.
656 + # TODO (ywang96): This is somewhat hacky. Refactor this to be
657 + # consistent with the other case.
658 + if isinstance(dummy_mm_data, MultiModalKwargs):
659 + dummy_mm_kwargs = {
660 + k: v[0].unsqueeze(0)
661 + for k, v in dummy_mm_data.items()
662 + }
663 +
664 + # Case when models have dummy data explicitly defined as
665 + # `MultiModalDataDict`, so they need to be processed through input
666 + # mapper.
667 + else:
668 + # Compute MM hashes (if enabled)
669 + mm_hashes = None
670 + if self.use_hash:
671 + mm_hashes = self.mm_hasher.hash_dummy_mm_data(
672 + dummy_mm_data)
673 +
674 + mm_kwargs_list = self.mm_input_mapper_client.process_inputs(
675 + mm_data=dummy_mm_data,
676 + mm_hashes=mm_hashes,
677 + mm_processor_kwargs=None,
678 + precomputed_mm_inputs=None)
679 +
680 + # Take the first `MultiModalKwargs`
681 + dummy_mm_kwargs = mm_kwargs_list[0]
682 +
683 batched_dummy_mm_inputs = MultiModalKwargs.batch(
664 - [dummy_mm_kwargs[0]] * max_num_mm_items)
684 + [dummy_mm_kwargs] * max_num_mm_items)
685 batched_dummy_mm_inputs = MultiModalKwargs.as_kwargs(
686 batched_dummy_mm_inputs, device=self.device)
687