[Bugfix] Fix Ovis2_5 special tokens for transformers v5 (#47298)
Signed-off-by: mgrunwal <milosz.grunwald@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Miłosz Grunwald committed
Jul 21, 2026 at 10:09 UTC
3e0c8875118e14be7822b7aafa8fd83c80c5e3e7
2 files changed
+12
-19
vllm/model_executor/models/ovis2_5.py
+3
-4
@@ -402,12 +402,11 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
402
hf_processor_mm_kwargs: Mapping[str, object],
403
out_mm_kwargs: MultiModalKwargsItems,
404
) -> list[PromptReplacement]:
405
- tokenizer = self.info.get_tokenizer()
406
- vocab = tokenizer.get_vocab()
405
+ hf_processor = self.info.get_hf_processor()
406
407
placeholder = {
409
- "image": vocab[IMAGE_TOKEN],
410
- "video": vocab[VIDEO_TOKEN],
408
+ "image": hf_processor.get_token_value("image_token"),
409
+ "video": hf_processor.get_token_value("video_token"),
410
}
411
412
def get_replacement_ovis(item_idx, modality: str):
vllm/transformers_utils/processors/ovis2_5.py
+9
-15
@@ -78,7 +78,6 @@ class Ovis2_5Processor(ProcessorMixin):
78
79
@cached_property
80
def extra_special_tokens(self):
81
- vocab = self.tokenizer.get_vocab()
81
required_tokens = {
82
"image_token": "<image>",
83
"video_token": "<video>",
@@ -90,21 +89,16 @@ class Ovis2_5Processor(ProcessorMixin):
89
"image_pad": "<|image_pad|>",
90
}
91
93
- extra_special_tokens = {}
94
- suggestion = (
95
- "please add '<image>', '<video>', '<ovis_visual_atom>', "
96
- "'<ovis_image_start>', '<ovis_image_end>', '<ovis_video_start>', "
97
- "'<ovis_video_end>' in 'additional_special_tokens' of "
98
- "tokenizer_config.json, You can refer to "
99
- "https://huggingface.co/AIDC-AI/Ovis2.6-30B-A3B/blob/main/tokenizer_config.json"
100
- )
101
-
102
- for key, token_name in required_tokens.items():
103
- if token_name not in vocab:
104
- raise ValueError(f"Can not find {token_name}, {suggestion}")
105
- extra_special_tokens[key] = vocab[token_name]
92
+ # The checkpoint defines both `additional_special_tokens` and
93
+ # `extra_special_tokens`, with the latter empty. Transformers ignores
94
+ # the former because the latter is explicitly empty, so the tokens are
95
+ # missing from the vocab. Re-add them to restore the expected ids.
96
+ self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
97
107
- return extra_special_tokens
98
+ return {
99
+ key: self.tokenizer.convert_tokens_to_ids(token_name)
100
+ for key, token_name in required_tokens.items()
101
+ }
102
103
def __call__(
104
self,