[DSV4.1] Remove compressor-aware image sentinel token padding (#56554)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Isotr0py committed
Sep 12, 2026 at 15:07 UTC
30118ba27d1d923bdd91f97d945528dcb4a862c1
4 files changed
+10
-132
vllm/models/deepseek_v4_1/amd/vl_model.py
-8
@@ -47,9 +47,7 @@ from ..common.mm_preprocess import (
47
IMAGE,
48
IMAGE_END,
49
IMAGE_NEW_LINE,
50
- IMAGE_PAD_ID,
50
IMAGE_PLACEHOLDER,
52
- IMAGE_SENTINEL_BASE_ID,
51
IMAGE_START,
52
DeepseekV4VLDummyInputsBuilder,
53
DeepseekV4VLMultiModalProcessor,
@@ -277,12 +275,6 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
275
) -> torch.Tensor:
276
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
277
280
- # Compressor-alignment pads borrow a reserved id; embed them as the
281
- # plain image token (the checkpoint has no image_pad vector).
282
- # Branch-free: safe inside compiled/captured regions.
283
- input_ids = input_ids.masked_fill(
284
- input_ids == IMAGE_PAD_ID, IMAGE_SENTINEL_BASE_ID
285
- )
278
inputs_embeds = self.language_model.embed_input_ids(input_ids)
279
280
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
vllm/models/deepseek_v4_1/common/mm_preprocess.py
+10
-110
@@ -12,14 +12,9 @@ to ``[IMAGE_START] + ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h +
12
rows in reading order; the delimiters take the learned ``image_start`` /
13
``image_newline`` / ``image_end`` vectors.
14
15
-One vLLM-side deviation from the reference token stream: a leading
16
-compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO``
17
-positions, so the span always starts at the same compressor phase) is
18
-prepended when the block is spliced into the final prompt. Pad positions
19
-borrow the reserved in-vocab token ``<|place_holder_mm_span_0436|>`` so
20
-they stay distinguishable from real span positions; they embed as the
21
-plain image token (v4.1 has no ``image_pad`` vector) and are routed and
22
-engram-deadened like image tokens.
15
+The token stream matches the reference exactly: no compressor-alignment
16
+pad is inserted (the reference pools image tokens across compressor-group
17
+boundaries freely).
18
"""
19
20
import math
@@ -30,7 +25,6 @@ import numpy as np
25
import torch
26
from PIL import Image, ImageOps
27
from transformers import BatchFeature
33
-from typing_extensions import assert_never
28
29
from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions
30
from vllm.inputs import MultiModalDataDict
@@ -44,42 +38,26 @@ from vllm.multimodal.processing import (
38
PromptUpdate,
39
PromptUpdateDetails,
40
)
47
-from vllm.multimodal.processing.processor import (
48
- MultiModalPromptUpdates,
49
- MultiModalPromptUpdatesApplyResult,
50
- PlaceholderFeaturesInfo,
51
- UpdateMode,
52
- _plan_prompt_updates,
53
-)
41
from vllm.transformers_utils.configs.deepseek_v41 import DeepseekV41Config
42
43
IMAGE_START, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(4)
57
-# Image spans are padded so they always start at the same compressor group
58
-# phase; v4.1 uses ratio-2 compressors (v4.0 used 4).
59
-COMPRESS_PAD_TO = 2
44
45
IMAGE_PLACEHOLDER = "<｜deepseek_image｜>"
46
47
# The checkpoint tokenizer's id for IMAGE_PLACEHOLDER (the config's
48
# ``image_token_id``). Every image-span position carries this id, exactly
49
# like the reference; the MoE router detects image tokens as the range
66
-# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5), which also covers
67
-# IMAGE_PAD_ID below.
50
+# [IMAGE_SENTINEL_BASE_ID, IMAGE_SENTINEL_BASE_ID + 5).
51
IMAGE_SENTINEL_BASE_ID = 129264
69
-# Leading compressor-alignment pads borrow this reserved in-vocab token
70
-# (``<|place_holder_mm_span_0436|>``) so they can be told apart from real
71
-# span positions when the per-position role is decided from token ids.
72
-IMAGE_PAD_ID = 129265
73
-IMAGE_PAD_TOKEN_NAME = "<|place_holder_mm_span_0436|>"
52
53
54
def image_sentinel_mask(token_ids: torch.Tensor) -> torch.Tensor:
77
- """Boolean mask for image-span positions (span tokens and align pads)."""
78
- return (token_ids == IMAGE_SENTINEL_BASE_ID) | (token_ids == IMAGE_PAD_ID)
55
+ """Boolean mask for image-span positions."""
56
+ return token_ids == IMAGE_SENTINEL_BASE_ID
57
58
59
def validate_image_sentinel_ids(tokenizer) -> None:
82
- """Check the image/pad token ids against the tokenizer."""
60
+ """Check the image token id against the tokenizer."""
61
image_id = tokenizer.convert_tokens_to_ids(IMAGE_PLACEHOLDER)
62
if image_id != IMAGE_SENTINEL_BASE_ID:
63
raise ValueError(
@@ -88,13 +66,6 @@ def validate_image_sentinel_ids(tokenizer) -> None:
66
"image_token_id); the DeepSeek-V4.1 vision path keys image "
67
"routing and engram masking off this id."
68
)
91
- pad_id = tokenizer.convert_tokens_to_ids(IMAGE_PAD_TOKEN_NAME)
92
- if pad_id != IMAGE_PAD_ID:
93
- raise ValueError(
94
- f"Image pad token {IMAGE_PAD_TOKEN_NAME!r} has id {pad_id}, "
95
- f"expected {IMAGE_PAD_ID}; the DeepSeek-V4.1 vision path "
96
- "borrows this reserved id for compressor-alignment pads."
97
- )
69
70
71
def llm_grid(best_height, best_width, patch_size, downsample_ratio):
@@ -134,8 +105,7 @@ def safe_resize(
105
height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token
106
):
107
"""Shrink the pixel size until the image costs at most max_n_token LLM
137
- tokens (minus the reservation for the compressor-alignment pad)."""
138
- max_n_token -= COMPRESS_PAD_TO - 1
108
+ tokens."""
109
n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio)
110
if num_image_tokens(n_llm_h, n_llm_w) > max_n_token:
111
best_height, best_width = solve_resize_ratio(
@@ -293,13 +263,7 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
263
seq_len: int,
264
mm_counts: Mapping[str, int],
265
) -> Mapping[str, int]:
296
- # ``safe_resize`` reserves COMPRESS_PAD_TO - 1 tokens of
297
- # vision_max_n_token for the compressor-alignment pad, so the full
298
- # image span is bounded by vision_max_n_token; the margin is kept
299
- # in case that reservation changes.
300
- return {
301
- "image": self.get_hf_config().vision_max_n_token + COMPRESS_PAD_TO - 1,
302
- }
266
+ return {"image": self.get_hf_config().vision_max_n_token}
267
268
def get_image_placeholder_token_id(self) -> int:
269
token_id = self.get_tokenizer().convert_tokens_to_ids(IMAGE_PLACEHOLDER)
@@ -314,7 +278,7 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo):
278
# A square maximizes the ViT patch count (area) within the token
279
# budget; solve the budget-derived size directly to keep the dummy
280
# image small.
317
- budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1)
281
+ budget = hf_config.vision_max_n_token
282
side = budget * patch_size * downsample_ratio
283
best_h, best_w = solve_resize_ratio(
284
side, side, patch_size, downsample_ratio, budget
@@ -397,67 +361,3 @@ class DeepseekV4VLMultiModalProcessor(
361
replacement=get_image_replacement,
362
),
363
]
400
-
401
- def _apply_token_matches_with_placeholders(
402
- self,
403
- token_ids: list[int],
404
- mm_prompt_updates: MultiModalPromptUpdates,
405
- ) -> tuple[
406
- list[int],
407
- MultiModalPromptUpdatesApplyResult,
408
- Mapping[str, list[PlaceholderFeaturesInfo]],
409
- ]:
410
- # Same as the base implementation, except that each image block gets
411
- # its compressor-alignment pad (``COMPRESS_PAD_TO - 1 - start %
412
- # COMPRESS_PAD_TO`` IMAGE_PAD_ID tokens) prepended while splicing:
413
- # the pad depends on the block's final position in the prompt, which
414
- # is unknown when the (cacheable) prompt updates are built. Pad
415
- # positions are not embed positions (their id differs from
416
- # image_token_id), so ``is_embed`` stays aligned with the pad-free
417
- # replacement content.
418
- matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
419
- placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
420
- modality: [] for modality in mm_prompt_updates
421
- }
422
-
423
- new_token_ids = list[int]()
424
- prev_end_idx = 0
425
- for matched_update in matched_updates:
426
- update = matched_update.update
427
- match = matched_update.match
428
-
429
- if update.mode == UpdateMode.INSERT:
430
- end_idx_to_insert = match.end_idx
431
- elif update.mode == UpdateMode.REPLACE:
432
- end_idx_to_insert = match.start_idx
433
- else:
434
- assert_never(update.mode)
435
-
436
- new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
437
- start_idx = len(new_token_ids)
438
-
439
- tokens = list(update.content.full)
440
- if tokens and update.modality == "image":
441
- compress_pad = COMPRESS_PAD_TO - 1 - start_idx % COMPRESS_PAD_TO
442
- tokens = [IMAGE_PAD_ID] * compress_pad + tokens
443
-
444
- if tokens:
445
- content_is_embed = update.content.is_embed
446
- is_embed = (
447
- content_is_embed(tokens) if content_is_embed is not None else None
448
- )
449
- placeholders[update.modality].append(
450
- PlaceholderFeaturesInfo(
451
- modality=update.modality,
452
- item_idx=update.item_idx,
453
- start_idx=start_idx,
454
- tokens=tokens,
455
- is_embed=is_embed,
456
- )
457
- )
458
- new_token_ids.extend(tokens)
459
-
460
- prev_end_idx = match.end_idx
461
-
462
- new_token_ids.extend(token_ids[prev_end_idx:])
463
- return new_token_ids, result, placeholders
vllm/models/deepseek_v4_1/nvidia/vl_model.py
-8
@@ -47,9 +47,7 @@ from ..common.mm_preprocess import (
47
IMAGE,
48
IMAGE_END,
49
IMAGE_NEW_LINE,
50
- IMAGE_PAD_ID,
50
IMAGE_PLACEHOLDER,
52
- IMAGE_SENTINEL_BASE_ID,
51
IMAGE_START,
52
DeepseekV4VLDummyInputsBuilder,
53
DeepseekV4VLMultiModalProcessor,
@@ -277,12 +275,6 @@ class DeepseekV41ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Supports
275
) -> torch.Tensor:
276
from vllm.model_executor.models.utils import _merge_multimodal_embeddings
277
280
- # Compressor-alignment pads borrow a reserved id; embed them as the
281
- # plain image token (the checkpoint has no image_pad vector).
282
- # Branch-free: safe inside compiled/captured regions.
283
- input_ids = input_ids.masked_fill(
284
- input_ids == IMAGE_PAD_ID, IMAGE_SENTINEL_BASE_ID
285
- )
278
inputs_embeds = self.language_model.embed_input_ids(input_ids)
279
280
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
vllm/transformers_utils/configs/deepseek_v41.py
-6
@@ -68,9 +68,3 @@ class DeepseekV41Config(PretrainedConfig):
68
# in-kernel, so mm-prefix ranges longer than sliding_window must be
69
# kept (they are only consumed by those kernels).
70
self.mm_prefix_clamp_sliding_window = vision_n_layers > 0
71
- # The visibility span covers the image block [IMAGE_START,
72
- # IMAGE_END]; the mm placeholder additionally carries a leading
73
- # compressor-alignment pad of ``COMPRESS_PAD_TO - 1 - offset %
74
- # COMPRESS_PAD_TO`` tokens (see common/mm_preprocess.py), with
75
- # COMPRESS_PAD_TO = 2 for v4.1's ratio-2 compressors.
76
- self.mm_prefix_span_leading_pad_modulus = 2 if vision_n_layers > 0 else 0