A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 04:45 UTC
Fix some more Transformers nightly tests (#29872)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Dec 2, 2025 at 21:49 UTC
6fc5841db14efedae7e6a8d1abdde3516c6c35a1
8 files changed
+75
-27
examples/offline_inference/vision_language.py
+4
-1
@@ -1801,7 +1801,10 @@ def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
1801
engine_args = EngineArgs(
1802
model=model_name,
1803
max_model_len=4096,
1804
- hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]},
1804
+ hf_overrides={
1805
+ "architectures": ["Tarsier2ForConditionalGeneration"],
1806
+ "model_type": "tarsier2",
1807
+ },
1808
limit_mm_per_prompt={modality: 1},
1809
)
1810
examples/offline_inference/vision_language_multi_image.py
+4
-1
@@ -1222,7 +1222,10 @@ def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
1222
trust_remote_code=True,
1223
max_model_len=32768,
1224
limit_mm_per_prompt={"image": len(image_urls)},
1225
- hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]},
1225
+ hf_overrides={
1226
+ "architectures": ["Tarsier2ForConditionalGeneration"],
1227
+ "model_type": "tarsier2",
1228
+ },
1229
)
1230
1231
prompt = (
tests/models/registry.py
+4
-1
@@ -831,7 +831,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
831
"TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),
832
"Tarsier2ForConditionalGeneration": _HfExamplesInfo(
833
"omni-research/Tarsier2-Recap-7b",
834
- hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]},
834
+ hf_overrides={
835
+ "architectures": ["Tarsier2ForConditionalGeneration"],
836
+ "model_type": "tarsier2",
837
+ },
838
),
839
"VoxtralForConditionalGeneration": _HfExamplesInfo(
840
"mistralai/Voxtral-Mini-3B-2507",
vllm/model_executor/models/qwen2_vl.py
-9
@@ -1576,15 +1576,6 @@ class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration):
1576
}
1577
)
1578
1579
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1580
- # Tarsier2 uses llava as model_type, which will create a Qwen2VLConfig
1581
- # as text_config, we need to reconstruct Qwen2VLConfig from LlavaConfig.
1582
- config = vllm_config.model_config.hf_config
1583
- qwen2vl_config = config.text_config
1584
- qwen2vl_config.architectures = config.architectures
1585
- vllm_config.model_config.hf_config = qwen2vl_config
1586
- super().__init__(vllm_config=vllm_config, prefix=prefix)
1587
-
1579
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
1580
skip_prefixes = []
1581
if self.visual is None:
vllm/tokenizers/mistral.py
+20
-8
@@ -14,13 +14,19 @@ if TYPE_CHECKING:
14
)
15
from mistral_common.tokens.tokenizers.tekken import Tekkenizer
16
from transformers import BatchEncoding
17
- from transformers.tokenization_mistral_common import (
18
- MistralCommonTokenizer as TransformersMistralTokenizer,
19
- )
17
18
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
19
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
20
21
+ try:
22
+ # Transformers v5
23
+ from transformers.tokenization_mistral_common import MistralCommonBackend
24
+ except ImportError:
25
+ # Transformers v4
26
+ from transformers.tokenization_mistral_common import (
27
+ MistralCommonTokenizer as MistralCommonBackend,
28
+ )
29
+
30
logger = init_logger(__name__)
31
32
@@ -208,11 +214,17 @@ class MistralTokenizer(TokenizerLike):
214
**kwargs,
215
) -> "MistralTokenizer":
216
from mistral_common.protocol.instruct.validator import ValidationMode
211
- from transformers.tokenization_mistral_common import (
212
- MistralCommonTokenizer as TransformersMistralTokenizer,
213
- )
217
215
- tokenizer = TransformersMistralTokenizer.from_pretrained(
218
+ try:
219
+ # Transformers v5
220
+ from transformers.tokenization_mistral_common import MistralCommonBackend
221
+ except ImportError:
222
+ # Transformers v4
223
+ from transformers.tokenization_mistral_common import (
224
+ MistralCommonTokenizer as MistralCommonBackend,
225
+ )
226
+
227
+ tokenizer = MistralCommonBackend.from_pretrained(
228
path_or_repo_id,
229
*args,
230
mode=ValidationMode.test,
@@ -223,7 +235,7 @@ class MistralTokenizer(TokenizerLike):
235
236
return cls(tokenizer)
237
226
- def __init__(self, tokenizer: "TransformersMistralTokenizer") -> None:
238
+ def __init__(self, tokenizer: "MistralCommonBackend") -> None:
239
super().__init__()
240
241
from mistral_common.protocol.instruct.validator import ValidationMode
vllm/transformers_utils/config.py
+17
-7
@@ -89,6 +89,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
89
step3_text="Step3TextConfig",
90
qwen3_next="Qwen3NextConfig",
91
lfm2_moe="Lfm2MoeConfig",
92
+ tarsier2="Tarsier2Config",
93
)
94
95
_CONFIG_ATTRS_MAPPING: dict[str, str] = {
@@ -127,6 +128,9 @@ class HFConfigParser(ConfigParserBase):
128
if config_dict.get("speculators_config") is not None
129
else model_type
130
)
131
+ # Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY
132
+ if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None:
133
+ model_type = hf_overrides.get("model_type", model_type)
134
135
if model_type in _CONFIG_REGISTRY:
136
config_class = _CONFIG_REGISTRY[model_type]
@@ -310,7 +314,7 @@ def patch_rope_parameters(config: PretrainedConfig) -> None:
314
config.rope_parameters["rope_theta"] = rope_theta
315
316
# No RoPE parameters to patch
313
- if not hasattr(config, "rope_parameters"):
317
+ if getattr(config, "rope_parameters", None) is None:
318
return
319
320
# Add original_max_position_embeddings if present
@@ -351,7 +355,10 @@ def patch_rope_parameters_dict(rope_parameters: dict[str, Any]) -> None:
355
rope_parameters["rope_type"] = "longrope"
356
logger.warning("Replacing legacy rope_type 'su' with 'longrope'")
357
elif rope_parameters["rope_type"] == "mrope":
354
- assert "mrope_section" in rope_parameters
358
+ if "mrope_section" not in rope_parameters:
359
+ raise ValueError(
360
+ "Legacy rope_type 'mrope' requires 'mrope_section' in rope_parameters"
361
+ )
362
rope_parameters["rope_type"] = "default"
363
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
364
@@ -584,6 +591,7 @@ def get_config(
591
trust_remote_code=trust_remote_code,
592
revision=revision,
593
code_revision=code_revision,
594
+ hf_overrides=hf_overrides_kw,
595
**kwargs,
596
)
597
# Special architecture mapping check for GGUF models
@@ -915,11 +923,13 @@ def get_hf_text_config(config: PretrainedConfig):
923
"""
924
text_config = config.get_text_config()
925
918
- if text_config is not config:
919
- # The code operates under the assumption that text_config should have
920
- # `num_attention_heads` (among others). Assert here to fail early
921
- # if transformers config doesn't align with this assumption.
922
- assert hasattr(text_config, "num_attention_heads")
926
+ if text_config is not config and not hasattr(text_config, "num_attention_heads"):
927
+ raise ValueError(
928
+ "The text_config extracted from the model config does not have "
929
+ "`num_attention_heads` attribute. This indicates a mismatch "
930
+ "between the model config and vLLM's expectations. Please "
931
+ "ensure that the model config is compatible with vLLM."
932
+ )
933
934
return text_config
935
vllm/transformers_utils/configs/__init__.py
+2
@@ -48,6 +48,7 @@ from vllm.transformers_utils.configs.step3_vl import (
48
Step3VisionEncoderConfig,
49
Step3VLConfig,
50
)
51
+from vllm.transformers_utils.configs.tarsier2 import Tarsier2Config
52
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
53
54
__all__ = [
@@ -81,4 +82,5 @@ __all__ = [
82
"Step3VisionEncoderConfig",
83
"Step3TextConfig",
84
"Qwen3NextConfig",
85
+ "Tarsier2Config",
86
]
vllm/transformers_utils/configs/tarsier2.py
new
+24
@@ -0,0 +1,24 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from transformers import Qwen2VLConfig
4
+
5
+
6
+class Tarsier2Config(Qwen2VLConfig):
7
+ """
8
+ Tarsier2's config.json is written such that AutoConfig.from_pretrained will create
9
+ a deeply nested config consisting of:
10
+
11
+ - LlavaConfig
12
+ - Qwen2VLConfig
13
+ - Qwen2VLTextConfig
14
+ - Qwen2VLVisionConfig
15
+ - Qwen2VLConfig
16
+ - Qwen2VLTextConfig
17
+ - Qwen2VLVisionConfig
18
+
19
+ When it should really just be a single Qwen2VLConfig.
20
+
21
+ This class is a hack to stop AutoConfig from creating the nested config structure.
22
+ """
23
+
24
+ model_type = "tarsier2"