@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7379b3d4b2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Fix multimodal profiling for `Molmo` (#11325)

Signed-off-by: ywang96 <ywang@example.com> Co-authored-by: ywang96 <ywang@example.com>

Roger Wang committed Dec 19, 2024 at 08:27 UTC 7379b3d4b2e0b85de43e7c5145ff26c8200aac8a
4 files changed +24 -4
vllm/model_executor/models/molmo.py
+5
@@ -928,7 +928,11 @@ def image_input_mapper_for_molmo(
928 data: object,
929 ):
930 if isinstance(data, list):
931 + assert len(data) == 1, "Molmo supports only one image per prompt."
932 data = data[0]
933 +
934 + # Remove unused dummy PIL image
935 + data.pop('raw_mm_data', None)
936 return MultiModalKwargs(data)
937
938
@@ -974,6 +978,7 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
978 dummy_imgdata = {
979 "images": out["images"],
980 "image_input_idx": out["image_input_idx"],
981 + "raw_mm_data": dummy_image,
982 }
983 if "image_masks" in out:
984 dummy_imgdata["image_masks"] = out["image_masks"]
vllm/v1/engine/mm_input_mapper.py
+17 -2
@@ -151,17 +151,31 @@ class MMHasher:
151 def __init__(self):
152 pass
153
154 - def hash_mm_data(
154 + def hash_dummy_mm_data(
155 self,
156 mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
157 + """Hash user-defined dummy multimodal data used for profiling."""
158 +
159 if mm_data is None:
160 return None
161
162 image_inputs = mm_data['image']
163
164 + # This is a temporary workaround for models (e.g, Molmo) that
165 + # process multimodal data in the input processor (therefore
166 + # image_inputs is MultiModalKwargs instead of raw input format).
167 + # `raw_mm_data` with the original input format is expected
168 + # in this case.
169 + if isinstance(image_inputs, dict):
170 + assert "raw_mm_data" in image_inputs and isinstance(
171 + image_inputs["raw_mm_data"], PIL.Image.Image)
172 + image_inputs = image_inputs.pop("raw_mm_data")
173 +
174 return self.hash_images(image_inputs)
175
164 - def hash_prompt(self, prompt: PromptType) -> Optional[List[str]]:
176 + def hash_prompt_mm_data(self, prompt: PromptType) -> Optional[List[str]]:
177 + """Hash multimodal data in the user input prompt if they exist."""
178 +
179 if "multi_modal_data" not in prompt:
180 return None
181
@@ -171,6 +185,7 @@ class MMHasher:
185 return self.hash_images(image_inputs)
186
187 def hash_images(self, image_inputs) -> Optional[List[str]]:
188 + """Hash PIL image objects to strings."""
189 if not isinstance(image_inputs, list):
190 image_inputs = [image_inputs]
191 assert len(image_inputs) > 0
vllm/v1/engine/processor.py
+1 -1
@@ -79,7 +79,7 @@ class Processor:
79 # Compute MM hashes (if enabled)
80 mm_hashes = None
81 if self.use_hash:
82 - mm_hashes = self.mm_hasher.hash_prompt(prompt)
82 + mm_hashes = self.mm_hasher.hash_prompt_mm_data(prompt)
83
84 # Process inputs.
85 preprocessed_inputs = self.input_preprocessor.preprocess(
vllm/v1/worker/gpu_model_runner.py
+1 -1
@@ -638,7 +638,7 @@ class GPUModelRunner:
638 # Compute MM hashes (if enabled)
639 mm_hashes = None
640 if self.use_hash:
641 - mm_hashes = self.mm_hasher.hash_mm_data(dummy_mm_data)
641 + mm_hashes = self.mm_hasher.hash_dummy_mm_data(dummy_mm_data)
642
643 dummy_mm_kwargs = self.mm_input_mapper_client.process_inputs(
644 mm_data=dummy_mm_data,