A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI/Build] Remove unnecessary flags from test registry (#27353)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 23, 2025 at 22:42 UTC
fe2016de2d2d68bb967e9c43ee168db01322b0f1
13 files changed
+89
-123
docs/models/supported_models.md
+2
-2
@@ -374,8 +374,8 @@ th {
374
| `GraniteMoeSharedForCausalLM` | Granite MoE Shared | `ibm-research/moe-7b-1b-active-shared-experts` (test model) | ✅︎ | ✅︎ |
375
| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
376
| `Grok1ModelForCausalLM` | Grok1 | `hpcai-tech/grok-1`. | ✅︎ | ✅︎ |
377
-| `HunYuanDenseV1ForCausalLM` | Hunyuan-7B-Instruct-0124 | `tencent/Hunyuan-7B-Instruct-0124` | ✅︎ | ✅︎ |
378
-| `HunYuanMoEV1ForCausalLM` | Hunyuan-80B-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
377
+| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
378
+| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
379
| `HCXVisionForCausalLM` | HyperCLOVAX-SEED-Vision-Instruct-3B | `naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B` | | |
380
| `InternLMForCausalLM` | InternLM | `internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc. | ✅︎ | ✅︎ |
381
| `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ |
tests/distributed/test_pipeline_parallel.py
+9
-3
@@ -244,7 +244,7 @@ def _compare_tp(
244
tokenizer_mode = model_info.tokenizer_mode
245
hf_overrides = model_info.hf_overrides
246
hf_config = get_config(model_id, trust_remote_code)
247
- skip_tokenizer_init = model_info.skip_tokenizer_init
247
+ require_embed_inputs = model_info.require_embed_inputs
248
max_num_seqs = model_info.max_num_seqs
249
250
dtype = "float16"
@@ -299,8 +299,14 @@ def _compare_tp(
299
common_args.extend(["--load-format", load_format])
300
if hf_overrides:
301
common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
302
- if skip_tokenizer_init:
303
- common_args.append("--skip-tokenizer-init")
302
+ if require_embed_inputs:
303
+ common_args.extend(
304
+ [
305
+ "--skip-tokenizer-init",
306
+ "--enable-prompt-embeds",
307
+ "--enable-mm-embeds",
308
+ ]
309
+ )
310
if max_num_seqs:
311
common_args.extend(["--max-num-seqs", f"{max_num_seqs}"])
312
tests/distributed/test_sequence_parallel.py
+9
-3
@@ -181,7 +181,7 @@ def _compare_sp(
181
trust_remote_code = model_info.trust_remote_code
182
tokenizer_mode = model_info.tokenizer_mode
183
hf_overrides = model_info.hf_overrides
184
- skip_tokenizer_init = model_info.skip_tokenizer_init
184
+ require_embed_inputs = model_info.require_embed_inputs
185
186
if load_format == "dummy":
187
# Avoid OOM
@@ -233,8 +233,14 @@ def _compare_sp(
233
common_args.extend(["--load-format", load_format])
234
if hf_overrides:
235
common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
236
- if skip_tokenizer_init:
237
- common_args.append("--skip-tokenizer-init")
236
+ if require_embed_inputs:
237
+ common_args.extend(
238
+ [
239
+ "--skip-tokenizer-init",
240
+ "--enable-prompt-embeds",
241
+ "--enable-mm-embeds",
242
+ ]
243
+ )
244
245
compilation_config = {
246
"mode": CompilationMode.VLLM_COMPILE,
tests/entrypoints/openai/test_chat_template.py
+3
-1
@@ -114,7 +114,9 @@ def test_get_gen_prompt(
114
trust_remote_code=model_info.trust_remote_code,
115
revision=model_info.revision,
116
hf_overrides=model_info.hf_overrides,
117
- skip_tokenizer_init=model_info.skip_tokenizer_init,
117
+ skip_tokenizer_init=model_info.require_embed_inputs,
118
+ enable_prompt_embeds=model_info.require_embed_inputs,
119
+ enable_mm_embeds=model_info.require_embed_inputs,
120
enforce_eager=model_info.enforce_eager,
121
dtype=model_info.dtype,
122
)
tests/entrypoints/test_chat_utils.py
+12
-4
@@ -1742,7 +1742,9 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
1742
revision=model_info.revision,
1743
trust_remote_code=model_info.trust_remote_code,
1744
hf_overrides=model_info.hf_overrides,
1745
- skip_tokenizer_init=model_info.skip_tokenizer_init,
1745
+ skip_tokenizer_init=model_info.require_embed_inputs,
1746
+ enable_prompt_embeds=model_info.require_embed_inputs,
1747
+ enable_mm_embeds=model_info.require_embed_inputs,
1748
enforce_eager=model_info.enforce_eager,
1749
dtype=model_info.dtype,
1750
)
@@ -1842,7 +1844,9 @@ def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwa
1844
revision=model_info.revision,
1845
trust_remote_code=model_info.trust_remote_code,
1846
hf_overrides=model_info.hf_overrides,
1845
- skip_tokenizer_init=model_info.skip_tokenizer_init,
1847
+ skip_tokenizer_init=model_info.require_embed_inputs,
1848
+ enable_prompt_embeds=model_info.require_embed_inputs,
1849
+ enable_mm_embeds=model_info.require_embed_inputs,
1850
enforce_eager=model_info.enforce_eager,
1851
dtype=model_info.dtype,
1852
)
@@ -1903,7 +1907,9 @@ def test_resolve_content_format_hf_defined(model, expected_format):
1907
revision=model_info.revision,
1908
trust_remote_code=model_info.trust_remote_code,
1909
hf_overrides=model_info.hf_overrides,
1906
- skip_tokenizer_init=model_info.skip_tokenizer_init,
1910
+ skip_tokenizer_init=model_info.require_embed_inputs,
1911
+ enable_prompt_embeds=model_info.require_embed_inputs,
1912
+ enable_mm_embeds=model_info.require_embed_inputs,
1913
enforce_eager=model_info.enforce_eager,
1914
dtype=model_info.dtype,
1915
)
@@ -1961,7 +1967,9 @@ def test_resolve_content_format_fallbacks(model, expected_format):
1967
revision=model_info.revision,
1968
trust_remote_code=model_info.trust_remote_code,
1969
hf_overrides=model_info.hf_overrides,
1964
- skip_tokenizer_init=model_info.skip_tokenizer_init,
1970
+ skip_tokenizer_init=model_info.require_embed_inputs,
1971
+ enable_prompt_embeds=model_info.require_embed_inputs,
1972
+ enable_mm_embeds=model_info.require_embed_inputs,
1973
enforce_eager=model_info.enforce_eager,
1974
dtype=model_info.dtype,
1975
)
tests/models/multimodal/generation/vlm_utils/core.py
+3
-2
@@ -71,8 +71,9 @@ def run_test(
71
vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
72
if model_info.hf_overrides:
73
vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
74
- if model_info.skip_tokenizer_init:
75
- vllm_runner_kwargs_["skip_tokenizer_init"] = model_info.skip_tokenizer_init
74
+ if model_info.require_embed_inputs:
75
+ for k in ("skip_tokenizer_init", "enable_prompt_embeds", "enable_mm_embeds"):
76
+ vllm_runner_kwargs_[k] = model_info.require_embed_inputs
77
78
if vllm_runner_kwargs:
79
vllm_runner_kwargs_.update(vllm_runner_kwargs)
tests/models/multimodal/processing/test_common.py
+3
-1
@@ -108,7 +108,9 @@ def _test_processing_correctness(
108
hf_overrides=model_info.hf_overrides,
109
# Ensure that the cache can fit all of the data
110
mm_processor_cache_gb=2048,
111
- skip_tokenizer_init=model_info.skip_tokenizer_init,
111
+ skip_tokenizer_init=model_info.require_embed_inputs,
112
+ enable_prompt_embeds=model_info.require_embed_inputs,
113
+ enable_mm_embeds=model_info.require_embed_inputs,
114
enforce_eager=model_info.enforce_eager,
115
dtype=model_info.dtype,
116
)
tests/models/multimodal/processing/test_tensor_schema.py
+3
-1
@@ -218,7 +218,9 @@ def test_model_tensor_schema(model_arch: str, model_id: str):
218
revision=model_info.revision,
219
trust_remote_code=model_info.trust_remote_code,
220
hf_overrides=hf_overrides_fn,
221
- skip_tokenizer_init=model_info.skip_tokenizer_init,
221
+ skip_tokenizer_init=model_info.require_embed_inputs,
222
+ enable_prompt_embeds=model_info.require_embed_inputs,
223
+ enable_mm_embeds=model_info.require_embed_inputs,
224
enforce_eager=model_info.enforce_eager,
225
dtype=model_info.dtype,
226
)
tests/models/multimodal/test_mapping.py
+3
-1
@@ -59,7 +59,9 @@ def test_hf_model_weights_mapper(model_arch: str):
59
revision=model_info.revision,
60
trust_remote_code=model_info.trust_remote_code,
61
hf_overrides=model_info.hf_overrides,
62
- skip_tokenizer_init=model_info.skip_tokenizer_init,
62
+ skip_tokenizer_init=model_info.require_embed_inputs,
63
+ enable_prompt_embeds=model_info.require_embed_inputs,
64
+ enable_mm_embeds=model_info.require_embed_inputs,
65
enforce_eager=model_info.enforce_eager,
66
dtype=model_info.dtype,
67
)
tests/models/registry.py
+30
-71
@@ -6,7 +6,6 @@ from dataclasses import dataclass, field
6
from typing import Any, Literal
7
8
import pytest
9
-import torch
9
from packaging.version import Version
10
from transformers import __version__ as TRANSFORMERS_VERSION
11
@@ -33,6 +32,11 @@ class _HfExamplesInfo:
32
for speculative decoding.
33
"""
34
35
+ speculative_method: str | None = None
36
+ """
37
+ The method to use for speculative decoding.
38
+ """
39
+
40
min_transformers_version: str | None = None
41
"""
42
The minimum version of HF Transformers that is required to run this model.
@@ -48,9 +52,10 @@ class _HfExamplesInfo:
52
The reason for the minimum/maximum version requirement.
53
"""
54
51
- skip_tokenizer_init: bool = False
55
+ require_embed_inputs: bool = False
56
"""
53
- If true, skip initialization of tokenizer and detokenizer.
57
+ If `True`, enables prompt and multi-modal embedding inputs while
58
+ disabling tokenization.
59
"""
60
61
dtype: ModelDType = "auto"
@@ -168,10 +173,7 @@ class _HfExamplesInfo:
173
174
_TEXT_GENERATION_EXAMPLE_MODELS = {
175
# [Decoder-only]
171
- "ApertusForCausalLM": _HfExamplesInfo(
172
- "swiss-ai/Apertus-8B-Instruct-2509",
173
- min_transformers_version="4.56.0",
174
- ),
176
+ "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),
177
"AquilaModel": _HfExamplesInfo("BAAI/AquilaChat-7B", trust_remote_code=True),
178
"AquilaForCausalLM": _HfExamplesInfo("BAAI/AquilaChat2-7B", trust_remote_code=True),
179
"ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),
@@ -192,7 +194,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
194
),
195
"BambaForCausalLM": _HfExamplesInfo(
196
"ibm-ai-platform/Bamba-9B-v1",
195
- min_transformers_version="4.55.3",
197
extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"},
198
),
199
"BloomForCausalLM": _HfExamplesInfo(
@@ -212,11 +213,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
213
"CohereForAI/c4ai-command-r7b-12-2024",
214
trust_remote_code=True,
215
),
215
- "CwmForCausalLM": _HfExamplesInfo(
216
- "facebook/cwm",
217
- trust_remote_code=True,
218
- is_available_online=False,
219
- ),
216
+ "CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
217
"DbrxForCausalLM": _HfExamplesInfo("databricks/dbrx-instruct"),
218
"DeciLMForCausalLM": _HfExamplesInfo(
219
"nvidia/Llama-3_3-Nemotron-Super-49B-v1",
@@ -232,18 +229,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
229
trust_remote_code=True,
230
),
231
"DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"),
235
- "Ernie4_5ForCausalLM": _HfExamplesInfo(
236
- "baidu/ERNIE-4.5-0.3B-PT", min_transformers_version="4.54"
237
- ),
238
- "Ernie4_5_MoeForCausalLM": _HfExamplesInfo(
239
- "baidu/ERNIE-4.5-21B-A3B-PT", min_transformers_version="4.54"
240
- ),
232
+ "Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT"),
233
+ "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT"),
234
"ExaoneForCausalLM": _HfExamplesInfo(
235
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
236
),
244
- "Exaone4ForCausalLM": _HfExamplesInfo(
245
- "LGAI-EXAONE/EXAONE-4.0-32B", min_transformers_version="4.54"
246
- ),
237
+ "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
238
"Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"),
239
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
240
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
@@ -251,14 +242,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
242
"GemmaForCausalLM": _HfExamplesInfo("google/gemma-1.1-2b-it"),
243
"Gemma2ForCausalLM": _HfExamplesInfo("google/gemma-2-9b"),
244
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
254
- "Gemma3nForCausalLM": _HfExamplesInfo(
255
- "google/gemma-3n-E2B-it", min_transformers_version="4.53"
256
- ),
245
+ "Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
246
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
247
"Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),
259
- "Glm4MoeForCausalLM": _HfExamplesInfo(
260
- "zai-org/GLM-4.5", min_transformers_version="4.54"
261
- ),
248
+ "Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
249
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}),
250
"GPTBigCodeForCausalLM": _HfExamplesInfo(
251
"bigcode/starcoder",
@@ -266,8 +253,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
253
"tiny": "bigcode/tiny_starcoder_py",
254
"santacoder": "bigcode/gpt_bigcode-santacoder",
255
},
269
- min_transformers_version="4.55.1",
270
- transformers_version_reason="HF model broken in 4.55.0",
256
),
257
"GPTJForCausalLM": _HfExamplesInfo(
258
"Milos/slovak-gpt-j-405M", {"6b": "EleutherAI/gpt-j-6b"}
@@ -279,8 +264,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
264
"GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
265
"GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
266
"GraniteMoeHybridForCausalLM": _HfExamplesInfo(
282
- "ibm-granite/granite-4.0-tiny-preview",
283
- min_transformers_version="4.55.3",
267
+ "ibm-granite/granite-4.0-tiny-preview"
268
),
269
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
270
"ibm-research/moe-7b-1b-active-shared-experts"
@@ -288,15 +272,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
272
"Grok1ModelForCausalLM": _HfExamplesInfo(
273
"hpcai-tech/grok-1", trust_remote_code=True
274
),
275
+ "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
276
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo(
277
"tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
278
),
294
- # TODO: Remove is_available_online once their config.json is fixed
295
- "HunYuanDenseV1ForCausalLM": _HfExamplesInfo(
296
- "tencent/Hunyuan-7B-Instruct-0124",
297
- trust_remote_code=True,
298
- is_available_online=False,
299
- ),
279
"InternLMForCausalLM": _HfExamplesInfo(
280
"internlm/internlm-chat-7b", trust_remote_code=True
281
),
@@ -312,15 +291,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
291
"JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
292
"JambaForCausalLM": _HfExamplesInfo(
293
"ai21labs/AI21-Jamba-1.5-Mini",
315
- min_transformers_version="4.55.3",
294
extras={
295
"tiny": "ai21labs/Jamba-tiny-dev",
296
"random": "ai21labs/Jamba-tiny-random",
297
},
298
),
321
- "Lfm2ForCausalLM": _HfExamplesInfo(
322
- "LiquidAI/LFM2-1.2B", min_transformers_version="4.54"
323
- ),
299
+ "Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
300
"Lfm2MoeForCausalLM": _HfExamplesInfo(
301
"LiquidAI/LFM2-8B-A1B", min_transformers_version="4.58"
302
),
@@ -338,7 +314,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
314
),
315
"Llama4ForCausalLM": _HfExamplesInfo(
316
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
341
- is_available_online=False,
317
),
318
"LongcatFlashForCausalLM": _HfExamplesInfo(
319
"meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True
@@ -346,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
321
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
322
"Mamba2ForCausalLM": _HfExamplesInfo(
323
"mistralai/Mamba-Codestral-7B-v0.1",
349
- min_transformers_version="4.55.3",
324
extras={
325
"random": "yujiepan/mamba2-codestral-v0.1-tiny-random",
326
},
@@ -421,7 +395,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
395
"SeedOssForCausalLM": _HfExamplesInfo(
396
"ByteDance-Seed/Seed-OSS-36B-Instruct",
397
trust_remote_code=True,
424
- is_available_online=False,
398
),
399
"SmolLM3ForCausalLM": _HfExamplesInfo("HuggingFaceTB/SmolLM3-3B"),
400
"StableLMEpochForCausalLM": _HfExamplesInfo("stabilityai/stablelm-zephyr-3b"),
@@ -488,7 +461,8 @@ _EMBEDDING_EXAMPLE_MODELS = {
461
"RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"),
462
"XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"),
463
"BertSpladeSparseEmbeddingModel": _HfExamplesInfo(
491
- "naver/splade-v3", is_available_online=False
464
+ "naver/splade-v3",
465
+ hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]},
466
),
467
# [Multimodal]
468
"CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"),
@@ -499,18 +473,17 @@ _EMBEDDING_EXAMPLE_MODELS = {
473
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
474
"PrithviGeoSpatialMAE": _HfExamplesInfo(
475
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
502
- dtype=torch.float16,
476
+ dtype="float16",
477
enforce_eager=True,
504
- skip_tokenizer_init=True,
505
- # This is to avoid the model
506
- # going OOM in CI
478
+ require_embed_inputs=True,
479
+ # This is to avoid the model going OOM in CI
480
max_num_seqs=32,
481
),
482
"Terratorch": _HfExamplesInfo(
483
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
511
- dtype=torch.float16,
484
+ dtype="float16",
485
enforce_eager=True,
513
- skip_tokenizer_init=True,
486
+ require_embed_inputs=True,
487
# This is to avoid the model going OOM in CI
488
max_num_seqs=32,
489
),
@@ -598,10 +571,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
571
),
572
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
573
"Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
601
- "Gemma3nForConditionalGeneration": _HfExamplesInfo(
602
- "google/gemma-3n-E2B-it",
603
- min_transformers_version="4.53",
604
- ),
574
+ "Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
575
"GraniteSpeechForConditionalGeneration": _HfExamplesInfo(
576
"ibm-granite/granite-speech-3.3-2b"
577
),
@@ -611,9 +581,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
581
hf_overrides={"architectures": ["GLM4VForCausalLM"]},
582
),
583
"Glm4vForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.1V-9B-Thinking"),
614
- "Glm4vMoeForConditionalGeneration": _HfExamplesInfo(
615
- "zai-org/GLM-4.5V", min_transformers_version="4.56"
616
- ),
584
+ "Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
585
"H2OVLChatModel": _HfExamplesInfo(
586
"h2oai/h2ovl-mississippi-800m",
587
trust_remote_code=True,
@@ -627,9 +595,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
595
),
596
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
597
"HuggingFaceM4/Idefics3-8B-Llama3",
630
- {"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
631
- min_transformers_version="4.56",
632
- transformers_version_reason="HF model broken in 4.55",
598
+ extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
599
),
600
"InternS1ForConditionalGeneration": _HfExamplesInfo(
601
"internlm/Intern-S1", trust_remote_code=True
@@ -781,13 +747,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
747
"Qwen/Qwen3-VL-4B-Instruct",
748
max_model_len=4096,
749
min_transformers_version="4.57",
784
- is_available_online=False,
750
),
751
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
752
"Qwen/Qwen3-VL-30B-A3B-Instruct",
753
max_model_len=4096,
754
min_transformers_version="4.57",
790
- is_available_online=False,
755
),
756
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
757
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
@@ -799,9 +763,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
763
"Skywork/Skywork-R1V-38B", trust_remote_code=True
764
),
765
"SmolVLMForConditionalGeneration": _HfExamplesInfo(
802
- "HuggingFaceTB/SmolVLM2-2.2B-Instruct",
803
- min_transformers_version="4.56",
804
- transformers_version_reason="HF model broken in 4.55",
766
+ "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
767
),
768
"Step3VLForConditionalGeneration": _HfExamplesInfo(
769
"stepfun-ai/step3", trust_remote_code=True
@@ -817,7 +779,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
779
),
780
"VoxtralForConditionalGeneration": _HfExamplesInfo(
781
"mistralai/Voxtral-Mini-3B-2507",
820
- min_transformers_version="4.54",
782
# disable this temporarily until we support HF format
783
is_available_online=False,
784
),
@@ -878,8 +839,8 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
839
"EagleMiniCPMForCausalLM": _HfExamplesInfo(
840
"openbmb/MiniCPM-1B-sft-bf16",
841
trust_remote_code=True,
881
- is_available_online=False,
842
speculative_model="openbmb/MiniCPM-2B-sft-bf16",
843
+ speculative_method="eagle",
844
tokenizer="openbmb/MiniCPM-2B-sft-bf16",
845
),
846
"ErnieMTPModel": _HfExamplesInfo(
@@ -890,8 +851,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
851
"Glm4MoeMTPModel": _HfExamplesInfo(
852
"zai-org/GLM-4.5",
853
speculative_model="zai-org/GLM-4.5",
893
- min_transformers_version="4.56",
894
- is_available_online=False,
854
),
855
"LongCatFlashMTPModel": _HfExamplesInfo(
856
"meituan-longcat/LongCat-Flash-Chat",
tests/models/test_initialization.py
+4
-6
@@ -105,20 +105,19 @@ def can_initialize(
105
if model_arch == "WhisperForConditionalGeneration":
106
m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
107
108
- extra_args = {}
109
- if model_arch in ("PrithviGeoSpatialMAE", "Terratorch"):
110
- extra_args["enable_mm_embeds"] = True
111
-
108
LLM(
109
model_info.default,
110
tokenizer=model_info.tokenizer,
111
tokenizer_mode=model_info.tokenizer_mode,
112
revision=model_info.revision,
113
enforce_eager=model_info.enforce_eager,
118
- skip_tokenizer_init=model_info.skip_tokenizer_init,
114
+ skip_tokenizer_init=model_info.require_embed_inputs,
115
+ enable_prompt_embeds=model_info.require_embed_inputs,
116
+ enable_mm_embeds=model_info.require_embed_inputs,
117
dtype=model_info.dtype,
118
speculative_config={
119
"model": model_info.speculative_model,
120
+ "method": model_info.speculative_method,
121
"num_speculative_tokens": 1,
122
}
123
if model_info.speculative_model
@@ -133,7 +132,6 @@ def can_initialize(
132
else "vllm",
133
hf_overrides=hf_overrides_fn,
134
max_num_seqs=model_info.max_num_seqs,
136
- **extra_args,
135
)
136
137
tests/models/utils.py
+3
-1
@@ -309,7 +309,9 @@ def build_model_context(
309
limit_mm_per_prompt=limit_mm_per_prompt,
310
mm_processor_cache_gb=mm_processor_cache_gb,
311
hf_overrides=model_info.hf_overrides,
312
- skip_tokenizer_init=model_info.skip_tokenizer_init,
312
+ skip_tokenizer_init=model_info.require_embed_inputs,
313
+ enable_prompt_embeds=model_info.require_embed_inputs,
314
+ enable_mm_embeds=model_info.require_embed_inputs,
315
enforce_eager=model_info.enforce_eager,
316
**model_config_kwargs,
317
)
vllm/model_executor/models/glm4_1v.py
+5
-27
@@ -36,9 +36,7 @@ import torch
36
import torch.nn as nn
37
import torch.nn.functional as F
38
from einops import rearrange
39
-from packaging.version import Version
39
from transformers import BatchFeature
41
-from transformers import __version__ as TRANSFORMERS_VERSION
40
from transformers.models.glm4v.configuration_glm4v import Glm4vVisionConfig
41
from transformers.models.glm4v.image_processing_glm4v import (
42
Glm4vImageProcessor,
@@ -1270,14 +1268,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1268
video_mm_data = dict()
1269
video_mm_data["videos"] = [[video_array]]
1270
1273
- # backward compatibility for Transformers 4.55
1271
unuse_metadata = ["do_sample_frames"]
1275
- if (
1276
- not hasattr(VideoMetadata, "frames_indices")
1277
- and "frames_indices" in metadata
1278
- ):
1279
- unuse_metadata.append("frames_indices")
1280
-
1272
video_mm_data["video_metadata"] = [
1273
[
1274
VideoMetadata(
@@ -1296,24 +1287,11 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1287
mm_kwargs=video_mm_kwargs,
1288
tok_kwargs=tok_kwargs,
1289
)
1299
- if not video_mm_kwargs["do_sample_frames"] and Version(
1300
- TRANSFORMERS_VERSION
1301
- ) < Version("4.56.0"):
1302
- # Transformers v4.55 has incorrect timestamps issue for
1303
- # skip sampling. We construct the placeholder manually to
1304
- # get placeholders with correct timestamps.
1305
- placeholder = self.info._construct_video_placeholder(
1306
- video_array,
1307
- metadata,
1308
- video_outputs["video_grid_thw"].squeeze(0),
1309
- )
1310
- video_placeholder = processor.tokenizer.decode(placeholder)
1311
- else:
1312
- input_ids = video_outputs.pop("input_ids")
1313
- input_ids[input_ids == processor.image_token_id] = (
1314
- processor.video_token_id
1315
- )
1316
- video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
1290
+ input_ids = video_outputs.pop("input_ids")
1291
+ input_ids[input_ids == processor.image_token_id] = (
1292
+ processor.video_token_id
1293
+ )
1294
+ video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
1295
prompt = prompt.replace(
1296
"<|begin_of_video|><|video|><|end_of_video|>",
1297
video_placeholder,