A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 05:15 UTC
[Bugfix] Fix Qwen-VL tokenizer implementation (#36140)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 6, 2026 at 00:07 UTC
719634815791ad97cf1e35ad52d4e39e630aeafd
9 files changed
+118
-66
tests/tokenizers_/test_basic.py
+11
-1
@@ -29,7 +29,8 @@ def test_tokenizer_like_protocol():
29
_assert_tokenizer_like(tokenizer)
30
31
tokenizer = get_tokenizer(
32
- "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral"
32
+ "mistralai/Mistral-7B-Instruct-v0.3",
33
+ tokenizer_mode="mistral",
34
)
35
assert isinstance(tokenizer, MistralTokenizer)
36
_assert_tokenizer_like(tokenizer)
@@ -40,11 +41,20 @@ def test_tokenizer_like_protocol():
41
42
tokenizer = get_tokenizer("deepseek-ai/DeepSeek-V3", tokenizer_mode="deepseek_v32")
43
assert isinstance(tokenizer, HfTokenizer)
44
+
45
# Verify it's a fast tokenizer (required for FastIncrementalDetokenizer)
46
assert isinstance(tokenizer, PreTrainedTokenizerFast)
47
assert "DSV32" in tokenizer.__class__.__name__
48
_assert_tokenizer_like(tokenizer)
49
50
+ tokenizer = get_tokenizer(
51
+ "Qwen/Qwen-VL",
52
+ tokenizer_mode="qwen_vl",
53
+ trust_remote_code=True,
54
+ )
55
+ assert isinstance(tokenizer, HfTokenizer)
56
+ assert "WithoutImagePad" in tokenizer.__class__.__name__
57
+
58
59
@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"])
60
def test_tokenizer_revision(tokenizer_name: str):
vllm/benchmarks/serve.py
+1
@@ -1321,6 +1321,7 @@ def add_cli_args(parser: argparse.ArgumentParser):
1321
- "slow" will always use the slow tokenizer.\n
1322
- "mistral" will always use the tokenizer from `mistral_common`.\n
1323
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
1324
+ - "qwen_vl" will always use the tokenizer from `qwen_vl`.\n
1325
- Other custom values can be supported via plugins.""",
1326
)
1327
parser.add_argument("--use-beam-search", action="store_true")
vllm/config/model.py
+1
@@ -126,6 +126,7 @@ class ModelConfig:
126
- "slow" will always use the slow tokenizer.\n
127
- "mistral" will always use the tokenizer from `mistral_common`.\n
128
- "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
129
+ - "qwen_vl" will always use the tokenizer from `qwen_vl`.\n
130
- Other custom values can be supported via plugins."""
131
trust_remote_code: bool = False
132
"""Trust remote code (e.g., from HuggingFace) when downloading the model
vllm/model_executor/models/qwen_vl.py
+2
-64
@@ -6,11 +6,9 @@
6
# Copyright (c) Alibaba Cloud.
7
"""Inference-only Qwen-VL model compatible with HuggingFace weights."""
8
9
-import copy
9
import math
11
-import unicodedata
12
-from collections.abc import Callable, Collection, Mapping, Sequence, Set
13
-from functools import lru_cache, partial
10
+from collections.abc import Callable, Mapping, Sequence
11
+from functools import partial
12
from typing import Annotated, Literal, TypeAlias
13
14
import regex as re
@@ -436,60 +434,6 @@ class QwenVLModel(QWenModel):
434
)
435
436
439
-@lru_cache(maxsize=1)
440
-def _get_tokenizer_without_image_pad(
441
- tokenizer: PreTrainedTokenizer,
442
-) -> PreTrainedTokenizer:
443
- """
444
- The logic of adding image pad tokens should only be applied in
445
- [`QwenVLProcessor`][vllm.model_executor.models.qwen_vl.QwenVLProcessor],
446
- so they are patched out here.
447
-
448
- The definition of the wrapped tokenizer can be found here:
449
- https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
450
- """
451
- new_tokenizer = copy.deepcopy(tokenizer)
452
-
453
- class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
454
- def tokenize(
455
- self,
456
- text: str,
457
- allowed_special: Set[str] | str = "all",
458
- disallowed_special: Collection[str] | str = (),
459
- **kwargs,
460
- ) -> list[bytes | str]:
461
- text = unicodedata.normalize("NFC", text)
462
-
463
- return [
464
- self.decoder[t]
465
- for t in self.tokenizer.encode(
466
- text,
467
- allowed_special=allowed_special,
468
- disallowed_special=disallowed_special,
469
- )
470
- ]
471
-
472
- def _decode(
473
- self,
474
- token_ids: int | list[int],
475
- skip_special_tokens: bool = False,
476
- errors: str | None = None,
477
- **kwargs,
478
- ) -> str:
479
- if isinstance(token_ids, int):
480
- token_ids = [token_ids]
481
-
482
- return self.tokenizer.decode(
483
- token_ids,
484
- errors=errors or self.errors,
485
- )
486
-
487
- TokenizerWithoutImagePad.__name__ = f"{tokenizer.__class__.__name__}WithoutImagePad"
488
-
489
- new_tokenizer.__class__ = TokenizerWithoutImagePad
490
- return new_tokenizer
491
-
492
-
437
class QwenVLProcessor:
438
"""
439
This model doesn't define its own HF processor,
@@ -574,12 +518,6 @@ class QwenVLProcessor:
518
519
520
class QwenVLProcessingInfo(BaseProcessingInfo):
577
- def get_tokenizer(self) -> PreTrainedTokenizer:
578
- tokenizer = self.ctx.get_tokenizer()
579
- assert isinstance(tokenizer, PreTrainedTokenizer)
580
-
581
- return _get_tokenizer_without_image_pad(tokenizer)
582
-
521
def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
522
return self.ctx.init_processor(
523
QwenVLProcessor,
vllm/renderers/qwen_vl.py
new
+29
@@ -0,0 +1,29 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
+
5
+from vllm.config import VllmConfig
6
+from vllm.tokenizers import cached_get_tokenizer
7
+from vllm.tokenizers.qwen_vl import QwenVLTokenizer
8
+
9
+from .base import BaseRenderer
10
+from .hf import HfRenderer
11
+
12
+
13
+class QwenVLRenderer(BaseRenderer[QwenVLTokenizer]):
14
+ @classmethod
15
+ def from_config( # type: ignore[override]
16
+ cls,
17
+ config: VllmConfig,
18
+ tokenizer_kwargs: dict[str, Any],
19
+ ) -> "HfRenderer":
20
+ model_config = config.model_config
21
+ if model_config.skip_tokenizer_init:
22
+ tokenizer = None
23
+ else:
24
+ tokenizer = cached_get_tokenizer(
25
+ tokenizer_cls=QwenVLTokenizer,
26
+ **tokenizer_kwargs,
27
+ )
28
+
29
+ return HfRenderer(config, tokenizer)
vllm/renderers/registry.py
+1
@@ -20,6 +20,7 @@ _VLLM_RENDERERS = {
20
"hf": ("hf", "HfRenderer"),
21
"grok2": ("grok2", "Grok2Renderer"),
22
"mistral": ("mistral", "MistralRenderer"),
23
+ "qwen_vl": ("qwen_vl", "QwenVLRenderer"),
24
"terratorch": ("terratorch", "TerratorchRenderer"),
25
}
26
vllm/tokenizers/deepseek_v32.py
+1
-1
@@ -7,9 +7,9 @@ from transformers import AutoTokenizer
7
8
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
9
10
-from . import TokenizerLike
10
from .deepseek_v32_encoding import encode_messages
11
from .hf import HfTokenizer, get_cached_tokenizer
12
+from .protocol import TokenizerLike
13
14
15
def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
vllm/tokenizers/qwen_vl.py
new
+67
@@ -0,0 +1,67 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import copy
4
+import unicodedata
5
+from collections.abc import Collection, Set
6
+
7
+from transformers import AutoTokenizer
8
+
9
+from .hf import HfTokenizer, get_cached_tokenizer
10
+from .protocol import TokenizerLike
11
+
12
+
13
+def get_qwen_vl_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
14
+ """
15
+ The logic of adding image pad tokens should only be applied in
16
+ `QwenVLProcessor`, so they are patched out here.
17
+
18
+ The definition of the wrapped tokenizer can be found here:
19
+ https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
20
+ """
21
+ new_tokenizer = copy.copy(tokenizer)
22
+
23
+ class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
24
+ def tokenize(
25
+ self,
26
+ text: str,
27
+ allowed_special: Set[str] | str = "all",
28
+ disallowed_special: Collection[str] | str = (),
29
+ **kwargs,
30
+ ) -> list[bytes | str]:
31
+ text = unicodedata.normalize("NFC", text)
32
+
33
+ return [
34
+ self.decoder[t]
35
+ for t in self.tokenizer.encode(
36
+ text,
37
+ allowed_special=allowed_special,
38
+ disallowed_special=disallowed_special,
39
+ )
40
+ ]
41
+
42
+ def _decode(
43
+ self,
44
+ token_ids: int | list[int],
45
+ skip_special_tokens: bool = False,
46
+ errors: str | None = None,
47
+ **kwargs,
48
+ ) -> str:
49
+ if isinstance(token_ids, int):
50
+ token_ids = [token_ids]
51
+
52
+ return self.tokenizer.decode(
53
+ token_ids,
54
+ errors=errors or self.errors,
55
+ )
56
+
57
+ TokenizerWithoutImagePad.__name__ = f"{tokenizer.__class__.__name__}WithoutImagePad"
58
+
59
+ new_tokenizer.__class__ = TokenizerWithoutImagePad
60
+ return new_tokenizer
61
+
62
+
63
+class QwenVLTokenizer(TokenizerLike):
64
+ @classmethod
65
+ def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
66
+ tokenizer = AutoTokenizer.from_pretrained(*args, **kwargs)
67
+ return get_cached_tokenizer(get_qwen_vl_tokenizer(tokenizer))
vllm/tokenizers/registry.py
+5
@@ -36,6 +36,7 @@ _VLLM_TOKENIZERS = {
36
"grok2": ("grok2", "Grok2Tokenizer"),
37
"hf": ("hf", "CachedHfTokenizer"),
38
"mistral": ("mistral", "MistralTokenizer"),
39
+ "qwen_vl": ("qwen_vl", "QwenVLTokenizer"),
40
}
41
42
@@ -165,6 +166,10 @@ def resolve_tokenizer_args(
166
):
167
tokenizer_mode = "grok2"
168
169
+ # Model-specific tokenizers
170
+ if tokenizer_mode == "auto" and "/Qwen-VL" in str(tokenizer_name):
171
+ tokenizer_mode = "qwen_vl"
172
+
173
# Fallback to HF tokenizer
174
if tokenizer_mode == "auto":
175
tokenizer_mode = "hf"