A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix issues for `Pixtral-Large-Instruct-2411` (#11393)
Signed-off-by: ywang96 <ywang@example.com> Co-authored-by: ywang96 <ywang@example.com>
Roger Wang committed
Dec 21, 2024 at 02:15 UTC
c2d1b075ba88271ccc23b981c223a0617afc6bfc
1 file changed
+15
-8
vllm/model_executor/models/pixtral.py
+15
-8
@@ -45,8 +45,12 @@ try:
45
except ImportError:
46
USE_XFORMERS_OPS = False
47
48
-PIXTRAL_IMAGE_BREAK_ID = 12
49
-PIXTRAL_IMAGE_END_ID = 13
48
+# These token ids cannot be retrieved from model config
49
+# so we hardcode them here.
50
+PIXTRAL_12B_IMAGE_BREAK_ID = 12
51
+PIXTRAL_12B_IMAGE_END_ID = 13
52
+PIXTRAL_LARGE_IMAGE_BREAK_ID = 14
53
+PIXTRAL_LARGE_IMAGE_END_ID = 15
54
55
56
def get_max_pixtral_image_tokens(ctx: InputContext):
@@ -118,8 +122,7 @@ def input_mapper_for_pixtral(ctx: InputContext,
122
for image_data in data_list:
123
image = ImageChunk(image=image_data)
124
encoding = tokenizer.instruct.mm_encoder(image)
121
- image = torch.from_numpy(encoding.image).to(device="cuda",
122
- dtype=torch.float16)
125
+ image = torch.from_numpy(encoding.image).to(dtype=torch.float16)
126
images.append(image)
127
image_tokens_list.append(encoding.tokens)
128
@@ -237,8 +240,9 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
240
241
# NOTE: Image embeddings are split into separate tensors for each image
242
# by the indices of `[IMG_END]` token.
240
- split_indices = torch.where(
241
- image_tokens == PIXTRAL_IMAGE_END_ID)[0] + 1
243
+ image_end_condition = (image_tokens == PIXTRAL_12B_IMAGE_END_ID) | (
244
+ image_tokens == PIXTRAL_LARGE_IMAGE_END_ID)
245
+ split_indices = torch.where(image_end_condition)[0] + 1
246
if len(split_indices) <= 1:
247
# Do not split, return as tensor of shape [1, fs, hs]
248
return image_embeds.unsqueeze(0)
@@ -260,8 +264,11 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
264
if multimodal_embeddings is not None:
265
inputs_embeds = merge_multimodal_embeddings(
266
input_ids, inputs_embeds, multimodal_embeddings, [
263
- self.vision_args.image_token_id, PIXTRAL_IMAGE_END_ID,
264
- PIXTRAL_IMAGE_BREAK_ID
267
+ self.vision_args.image_token_id,
268
+ PIXTRAL_12B_IMAGE_END_ID,
269
+ PIXTRAL_12B_IMAGE_BREAK_ID,
270
+ PIXTRAL_LARGE_IMAGE_BREAK_ID,
271
+ PIXTRAL_LARGE_IMAGE_END_ID,
272
])
273
return inputs_embeds
274