A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Model] Remove hardcoded image tokens ids from Pixtral (#11582)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Dec 28, 2024 at 02:54 UTC
b7dcc003dc1ace7605946d52b7e077ba1d3bbe86
1 file changed
+13
-14
vllm/model_executor/models/pixtral.py
+13
-14
@@ -45,13 +45,6 @@ try:
45
except ImportError:
46
USE_XFORMERS_OPS = False
47
48
-# These token ids cannot be retrieved from model config
49
-# so we hardcode them here.
50
-PIXTRAL_12B_IMAGE_BREAK_ID = 12
51
-PIXTRAL_12B_IMAGE_END_ID = 13
52
-PIXTRAL_LARGE_IMAGE_BREAK_ID = 14
53
-PIXTRAL_LARGE_IMAGE_END_ID = 15
54
-
48
49
def get_max_pixtral_image_tokens(ctx: InputContext):
50
tokenizer = cached_get_tokenizer(
@@ -201,6 +194,13 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
194
if key in dataclass_fields
195
}
196
197
+ if not ("image_break_token_id" in vision_args
198
+ and "image_end_token_id" in vision_args):
199
+ raise ValueError(
200
+ "'image_break_token_id' and 'image_end_token_id' not found "
201
+ "in the vision_encoder arguments. Please download the latest "
202
+ "version of 'params.json' from the model repository.")
203
+
204
self.vision_args = VisionEncoderArgs(**vision_args)
205
206
# init MistralForCausalLM
@@ -240,9 +240,8 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
240
241
# NOTE: Image embeddings are split into separate tensors for each image
242
# by the indices of `[IMG_END]` token.
243
- image_end_condition = (image_tokens == PIXTRAL_12B_IMAGE_END_ID) | (
244
- image_tokens == PIXTRAL_LARGE_IMAGE_END_ID)
245
- split_indices = torch.where(image_end_condition)[0] + 1
243
+ image_end_mask = image_tokens == self.vision_args.image_end_token_id
244
+ split_indices = torch.where(image_end_mask)[0] + 1
245
if len(split_indices) <= 1:
246
# Do not split, return as tensor of shape [1, fs, hs]
247
return image_embeds.unsqueeze(0)
@@ -265,10 +264,8 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
264
inputs_embeds = merge_multimodal_embeddings(
265
input_ids, inputs_embeds, multimodal_embeddings, [
266
self.vision_args.image_token_id,
268
- PIXTRAL_12B_IMAGE_END_ID,
269
- PIXTRAL_12B_IMAGE_BREAK_ID,
270
- PIXTRAL_LARGE_IMAGE_BREAK_ID,
271
- PIXTRAL_LARGE_IMAGE_END_ID,
267
+ self.vision_args.image_break_token_id,
268
+ self.vision_args.image_end_token_id,
269
])
270
return inputs_embeds
271
@@ -409,6 +406,8 @@ class VisionEncoderArgs:
406
num_attention_heads: int
407
rope_theta: float # for rope-2D
408
image_token_id: int
409
+ image_break_token_id: int
410
+ image_end_token_id: int
411
adapter_bias: bool = True
412
413