@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b7dcc003dc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Model] Remove hardcoded image tokens ids from Pixtral (#11582)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Dec 28, 2024 at 02:54 UTC b7dcc003dc1ace7605946d52b7e077ba1d3bbe86
1 file changed +13 -14
vllm/model_executor/models/pixtral.py
+13 -14
@@ -45,13 +45,6 @@ try:
45 except ImportError:
46 USE_XFORMERS_OPS = False
47
48 -# These token ids cannot be retrieved from model config
49 -# so we hardcode them here.
50 -PIXTRAL_12B_IMAGE_BREAK_ID = 12
51 -PIXTRAL_12B_IMAGE_END_ID = 13
52 -PIXTRAL_LARGE_IMAGE_BREAK_ID = 14
53 -PIXTRAL_LARGE_IMAGE_END_ID = 15
54 -
48
49 def get_max_pixtral_image_tokens(ctx: InputContext):
50 tokenizer = cached_get_tokenizer(
@@ -201,6 +194,13 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
194 if key in dataclass_fields
195 }
196
197 + if not ("image_break_token_id" in vision_args
198 + and "image_end_token_id" in vision_args):
199 + raise ValueError(
200 + "'image_break_token_id' and 'image_end_token_id' not found "
201 + "in the vision_encoder arguments. Please download the latest "
202 + "version of 'params.json' from the model repository.")
203 +
204 self.vision_args = VisionEncoderArgs(**vision_args)
205
206 # init MistralForCausalLM
@@ -240,9 +240,8 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
240
241 # NOTE: Image embeddings are split into separate tensors for each image
242 # by the indices of `[IMG_END]` token.
243 - image_end_condition = (image_tokens == PIXTRAL_12B_IMAGE_END_ID) | (
244 - image_tokens == PIXTRAL_LARGE_IMAGE_END_ID)
245 - split_indices = torch.where(image_end_condition)[0] + 1
243 + image_end_mask = image_tokens == self.vision_args.image_end_token_id
244 + split_indices = torch.where(image_end_mask)[0] + 1
245 if len(split_indices) <= 1:
246 # Do not split, return as tensor of shape [1, fs, hs]
247 return image_embeds.unsqueeze(0)
@@ -265,10 +264,8 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
264 inputs_embeds = merge_multimodal_embeddings(
265 input_ids, inputs_embeds, multimodal_embeddings, [
266 self.vision_args.image_token_id,
268 - PIXTRAL_12B_IMAGE_END_ID,
269 - PIXTRAL_12B_IMAGE_BREAK_ID,
270 - PIXTRAL_LARGE_IMAGE_BREAK_ID,
271 - PIXTRAL_LARGE_IMAGE_END_ID,
267 + self.vision_args.image_break_token_id,
268 + self.vision_args.image_end_token_id,
269 ])
270 return inputs_embeds
271
@@ -409,6 +406,8 @@ class VisionEncoderArgs:
406 num_attention_heads: int
407 rope_theta: float # for rope-2D
408 image_token_id: int
409 + image_break_token_id: int
410 + image_end_token_id: int
411 adapter_bias: bool = True
412
413