@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 024ad87cdc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Fix dtype mismatch in PaliGemma (#6367)

Cyrus Leung committed Jul 12, 2024 at 23:22 UTC 024ad87cdc00bb44beed409dbd90c4490284d73e
3 files changed +12 -5
tests/models/test_paligemma.py
+1 -1
@@ -129,7 +129,7 @@ def run_test(
129 [0.25, 0.5, 1.0],
130 ],
131 )
132 -@pytest.mark.parametrize("dtype", ["float"])
132 +@pytest.mark.parametrize("dtype", ["float", "half"])
133 @pytest.mark.parametrize("max_tokens", [128])
134 @pytest.mark.parametrize("num_logprobs", [5])
135 def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
vllm/model_executor/models/gemma.py
+1
@@ -277,6 +277,7 @@ class GemmaModel(nn.Module):
277 positions: torch.Tensor,
278 kv_caches: List[torch.Tensor],
279 attn_metadata: AttentionMetadata,
280 + intermediate_tensors: Optional[IntermediateTensors] = None,
281 inputs_embeds: Optional[torch.Tensor] = None,
282 ) -> torch.Tensor:
283 if inputs_embeds is not None:
vllm/model_executor/models/paligemma.py
+10 -4
@@ -19,7 +19,7 @@ from vllm.model_executor.models.gemma import GemmaModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
21 from vllm.multimodal.image import cached_get_tokenizer
22 -from vllm.sequence import SamplerOutput, SequenceData
22 +from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
23
24 from .interfaces import SupportsVision
25 from .utils import merge_vision_embeddings
@@ -111,7 +111,7 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
111 orig_prompt = llm_inputs.get("prompt")
112 orig_prompt_ids = llm_inputs.get("prompt_token_ids")
113
114 - if image_token_str in orig_prompt:
114 + if orig_prompt is not None and image_token_str in orig_prompt:
115 logger.warning(
116 "The image token '%s' was detected in the prompt and "
117 "will be removed. Please follow the proper prompt format"
@@ -214,7 +214,9 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
214 def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
215 pixel_values: torch.Tensor) -> torch.Tensor:
216
217 - image_outputs = vision_tower(pixel_values, output_hidden_states=True)
217 + target_dtype = vision_tower.get_input_embeddings().weight.dtype
218 + image_outputs = vision_tower(pixel_values.to(dtype=target_dtype),
219 + output_hidden_states=True)
220
221 selected_image_features = image_outputs.last_hidden_state
222
@@ -236,9 +238,12 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
238
239 return self.multi_modal_projector(image_features)
240
239 - def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
241 + def forward(self,
242 + input_ids: torch.Tensor,
243 + positions: torch.Tensor,
244 kv_caches: List[torch.Tensor],
245 attn_metadata: AttentionMetadata,
246 + intermediate_tensors: Optional[IntermediateTensors] = None,
247 **kwargs: object) -> SamplerOutput:
248
249 parsed_image_input = self._parse_and_validate_image_input(**kwargs)
@@ -263,6 +268,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
268 positions,
269 kv_caches,
270 attn_metadata,
271 + None,
272 inputs_embeds=inputs_embeds)
273
274 return hidden_states