A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Fix dtype mismatch in PaliGemma (#6367)
Cyrus Leung committed
Jul 12, 2024 at 23:22 UTC
024ad87cdc00bb44beed409dbd90c4490284d73e
3 files changed
+12
-5
tests/models/test_paligemma.py
+1
-1
@@ -129,7 +129,7 @@ def run_test(
129
[0.25, 0.5, 1.0],
130
],
131
)
132
-@pytest.mark.parametrize("dtype", ["float"])
132
+@pytest.mark.parametrize("dtype", ["float", "half"])
133
@pytest.mark.parametrize("max_tokens", [128])
134
@pytest.mark.parametrize("num_logprobs", [5])
135
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
vllm/model_executor/models/gemma.py
+1
@@ -277,6 +277,7 @@ class GemmaModel(nn.Module):
277
positions: torch.Tensor,
278
kv_caches: List[torch.Tensor],
279
attn_metadata: AttentionMetadata,
280
+ intermediate_tensors: Optional[IntermediateTensors] = None,
281
inputs_embeds: Optional[torch.Tensor] = None,
282
) -> torch.Tensor:
283
if inputs_embeds is not None:
vllm/model_executor/models/paligemma.py
+10
-4
@@ -19,7 +19,7 @@ from vllm.model_executor.models.gemma import GemmaModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
from vllm.multimodal.image import cached_get_tokenizer
22
-from vllm.sequence import SamplerOutput, SequenceData
22
+from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
23
24
from .interfaces import SupportsVision
25
from .utils import merge_vision_embeddings
@@ -111,7 +111,7 @@ def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
111
orig_prompt = llm_inputs.get("prompt")
112
orig_prompt_ids = llm_inputs.get("prompt_token_ids")
113
114
- if image_token_str in orig_prompt:
114
+ if orig_prompt is not None and image_token_str in orig_prompt:
115
logger.warning(
116
"The image token '%s' was detected in the prompt and "
117
"will be removed. Please follow the proper prompt format"
@@ -214,7 +214,9 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
214
def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
215
pixel_values: torch.Tensor) -> torch.Tensor:
216
217
- image_outputs = vision_tower(pixel_values, output_hidden_states=True)
217
+ target_dtype = vision_tower.get_input_embeddings().weight.dtype
218
+ image_outputs = vision_tower(pixel_values.to(dtype=target_dtype),
219
+ output_hidden_states=True)
220
221
selected_image_features = image_outputs.last_hidden_state
222
@@ -236,9 +238,12 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
238
239
return self.multi_modal_projector(image_features)
240
239
- def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
241
+ def forward(self,
242
+ input_ids: torch.Tensor,
243
+ positions: torch.Tensor,
244
kv_caches: List[torch.Tensor],
245
attn_metadata: AttentionMetadata,
246
+ intermediate_tensors: Optional[IntermediateTensors] = None,
247
**kwargs: object) -> SamplerOutput:
248
249
parsed_image_input = self._parse_and_validate_image_input(**kwargs)
@@ -263,6 +268,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
268
positions,
269
kv_caches,
270
attn_metadata,
271
+ None,
272
inputs_embeds=inputs_embeds)
273
274
return hidden_states