@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c2d1b075ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix issues for `Pixtral-Large-Instruct-2411` (#11393)

Signed-off-by: ywang96 <ywang@example.com> Co-authored-by: ywang96 <ywang@example.com>

Roger Wang committed Dec 21, 2024 at 02:15 UTC c2d1b075ba88271ccc23b981c223a0617afc6bfc
1 file changed +15 -8
vllm/model_executor/models/pixtral.py
+15 -8
@@ -45,8 +45,12 @@ try:
45 except ImportError:
46 USE_XFORMERS_OPS = False
47
48 -PIXTRAL_IMAGE_BREAK_ID = 12
49 -PIXTRAL_IMAGE_END_ID = 13
48 +# These token ids cannot be retrieved from model config
49 +# so we hardcode them here.
50 +PIXTRAL_12B_IMAGE_BREAK_ID = 12
51 +PIXTRAL_12B_IMAGE_END_ID = 13
52 +PIXTRAL_LARGE_IMAGE_BREAK_ID = 14
53 +PIXTRAL_LARGE_IMAGE_END_ID = 15
54
55
56 def get_max_pixtral_image_tokens(ctx: InputContext):
@@ -118,8 +122,7 @@ def input_mapper_for_pixtral(ctx: InputContext,
122 for image_data in data_list:
123 image = ImageChunk(image=image_data)
124 encoding = tokenizer.instruct.mm_encoder(image)
121 - image = torch.from_numpy(encoding.image).to(device="cuda",
122 - dtype=torch.float16)
125 + image = torch.from_numpy(encoding.image).to(dtype=torch.float16)
126 images.append(image)
127 image_tokens_list.append(encoding.tokens)
128
@@ -237,8 +240,9 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
240
241 # NOTE: Image embeddings are split into separate tensors for each image
242 # by the indices of `[IMG_END]` token.
240 - split_indices = torch.where(
241 - image_tokens == PIXTRAL_IMAGE_END_ID)[0] + 1
243 + image_end_condition = (image_tokens == PIXTRAL_12B_IMAGE_END_ID) | (
244 + image_tokens == PIXTRAL_LARGE_IMAGE_END_ID)
245 + split_indices = torch.where(image_end_condition)[0] + 1
246 if len(split_indices) <= 1:
247 # Do not split, return as tensor of shape [1, fs, hs]
248 return image_embeds.unsqueeze(0)
@@ -260,8 +264,11 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
264 if multimodal_embeddings is not None:
265 inputs_embeds = merge_multimodal_embeddings(
266 input_ids, inputs_embeds, multimodal_embeddings, [
263 - self.vision_args.image_token_id, PIXTRAL_IMAGE_END_ID,
264 - PIXTRAL_IMAGE_BREAK_ID
267 + self.vision_args.image_token_id,
268 + PIXTRAL_12B_IMAGE_END_ID,
269 + PIXTRAL_12B_IMAGE_BREAK_ID,
270 + PIXTRAL_LARGE_IMAGE_BREAK_ID,
271 + PIXTRAL_LARGE_IMAGE_END_ID,
272 ])
273 return inputs_embeds
274