A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Fix multimodal profiling for `Molmo` (#11325)
Signed-off-by: ywang96 <ywang@example.com> Co-authored-by: ywang96 <ywang@example.com>
Roger Wang committed
Dec 19, 2024 at 08:27 UTC
7379b3d4b2e0b85de43e7c5145ff26c8200aac8a
4 files changed
+24
-4
vllm/model_executor/models/molmo.py
+5
@@ -928,7 +928,11 @@ def image_input_mapper_for_molmo(
928
data: object,
929
):
930
if isinstance(data, list):
931
+ assert len(data) == 1, "Molmo supports only one image per prompt."
932
data = data[0]
933
+
934
+ # Remove unused dummy PIL image
935
+ data.pop('raw_mm_data', None)
936
return MultiModalKwargs(data)
937
938
@@ -974,6 +978,7 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
978
dummy_imgdata = {
979
"images": out["images"],
980
"image_input_idx": out["image_input_idx"],
981
+ "raw_mm_data": dummy_image,
982
}
983
if "image_masks" in out:
984
dummy_imgdata["image_masks"] = out["image_masks"]
vllm/v1/engine/mm_input_mapper.py
+17
-2
@@ -151,17 +151,31 @@ class MMHasher:
151
def __init__(self):
152
pass
153
154
- def hash_mm_data(
154
+ def hash_dummy_mm_data(
155
self,
156
mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
157
+ """Hash user-defined dummy multimodal data used for profiling."""
158
+
159
if mm_data is None:
160
return None
161
162
image_inputs = mm_data['image']
163
164
+ # This is a temporary workaround for models (e.g, Molmo) that
165
+ # process multimodal data in the input processor (therefore
166
+ # image_inputs is MultiModalKwargs instead of raw input format).
167
+ # `raw_mm_data` with the original input format is expected
168
+ # in this case.
169
+ if isinstance(image_inputs, dict):
170
+ assert "raw_mm_data" in image_inputs and isinstance(
171
+ image_inputs["raw_mm_data"], PIL.Image.Image)
172
+ image_inputs = image_inputs.pop("raw_mm_data")
173
+
174
return self.hash_images(image_inputs)
175
164
- def hash_prompt(self, prompt: PromptType) -> Optional[List[str]]:
176
+ def hash_prompt_mm_data(self, prompt: PromptType) -> Optional[List[str]]:
177
+ """Hash multimodal data in the user input prompt if they exist."""
178
+
179
if "multi_modal_data" not in prompt:
180
return None
181
@@ -171,6 +185,7 @@ class MMHasher:
185
return self.hash_images(image_inputs)
186
187
def hash_images(self, image_inputs) -> Optional[List[str]]:
188
+ """Hash PIL image objects to strings."""
189
if not isinstance(image_inputs, list):
190
image_inputs = [image_inputs]
191
assert len(image_inputs) > 0
vllm/v1/engine/processor.py
+1
-1
@@ -79,7 +79,7 @@ class Processor:
79
# Compute MM hashes (if enabled)
80
mm_hashes = None
81
if self.use_hash:
82
- mm_hashes = self.mm_hasher.hash_prompt(prompt)
82
+ mm_hashes = self.mm_hasher.hash_prompt_mm_data(prompt)
83
84
# Process inputs.
85
preprocessed_inputs = self.input_preprocessor.preprocess(
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -638,7 +638,7 @@ class GPUModelRunner:
638
# Compute MM hashes (if enabled)
639
mm_hashes = None
640
if self.use_hash:
641
- mm_hashes = self.mm_hasher.hash_mm_data(dummy_mm_data)
641
+ mm_hashes = self.mm_hasher.hash_dummy_mm_data(dummy_mm_data)
642
643
dummy_mm_kwargs = self.mm_input_mapper_client.process_inputs(
644
mm_data=dummy_mm_data,