[BUGFIX] fix ovis2_5 multimodal tokens (#52623)
Signed-off-by: Milosz Grunwald <milosz.grunwald@intel.com>
Miłosz Grunwald committed
Sep 25, 2026 at 01:18 UTC
bd4cb3fe4efd1e19ceec1f31fcd4bfe599b0c3ac
2 files changed
+19
-3
vllm/model_executor/models/ovis2_5.py
+12
-1
@@ -175,6 +175,18 @@ class VisualTokenizer(torch.nn.Module):
175
176
177
class Ovis2_5ProcessingInfo(BaseProcessingInfo):
178
+ def __init__(self, ctx) -> None:
179
+ super().__init__(ctx)
180
+ # extra_special_tokens adds tokens to the tokenizer lazily, on first
181
+ # access, but the renderer deep-copies this tokenizer into a thread
182
+ # pool right after this processor is constructed; the copies never
183
+ # see mutations made afterwards. Force it now, before that happens,
184
+ # on the tokenizer we actually use (get_hf_processor()'s cache can
185
+ # otherwise hand back one bound to a self-loaded tokenizer instead).
186
+ hf_processor = self.get_hf_processor()
187
+ hf_processor.tokenizer = self.get_tokenizer()
188
+ hf_processor.extra_special_tokens # noqa: B018
189
+
190
def get_hf_config(self):
191
return self.ctx.get_hf_config()
192
@@ -390,7 +402,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])
402
out_mm_kwargs: MultiModalKwargsItems,
403
) -> list[PromptReplacement]:
404
hf_processor = self.info.get_hf_processor()
393
-
405
placeholder = {
406
"image": hf_processor.get_token_value("image_token"),
407
"video": hf_processor.get_token_value("video_token"),
vllm/transformers_utils/processors/ovis2_5.py
+7
-2
@@ -96,9 +96,14 @@ class Ovis2_5Processor(ProcessorMixin):
96
# missing from the vocab. Re-add them to restore the expected ids.
97
self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True)
98
99
+ # in TokenizerPool tokenizer.get_added_vocab() gets values from an instance
100
+ # that was updated with tokenizer.add_tokens(), so it is favourable over
101
+ # self.tokenizer.convert_tokens_to_ids(), which refers to a copy before
102
+ # mutation by tokenizer.add_tokens()
103
+ added_vocab = self.tokenizer.get_added_vocab()
104
+
105
return {
100
- key: self.tokenizer.convert_tokens_to_ids(token_name)
101
- for key, token_name in required_tokens.items()
106
+ key: added_vocab[token_name] for key, token_name in required_tokens.items()
107
}
108
109
def __call__(