A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Enable all hf transformers baselines in test_hybrid (#23936)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Sep 2, 2025 at 22:15 UTC
d328f7894f140fdc643dc1aa5fe80f4596e6f418
2 files changed
+31
-56
tests/models/language/generation/test_hybrid.py
+22
-52
@@ -34,17 +34,6 @@ HYBRID_MODELS = [
34
"LiquidAI/LFM2-1.2B",
35
]
36
37
-HF_UNSUPPORTED_MODELS = [
38
- # The HF transformers implementation of
39
- # Mamba2 is buggy for Codestral as it doesn't handle n_groups, so the test
40
- # doesn't compare vLLM output with HF output.
41
- # See https://github.com/huggingface/transformers/pull/35943
42
- "yujiepan/mamba2-codestral-v0.1-tiny-random",
43
- # transformers 4.55 is still producing garbage for this model
44
- # TODO(tdoublep): follow-up on transformers side
45
- "ibm-granite/granite-4.0-tiny-preview"
46
-]
47
-
37
V1_SUPPORTED_MODELS = [
38
"state-spaces/mamba-130m-hf",
39
"ai21labs/Jamba-tiny-dev",
@@ -90,20 +79,13 @@ def test_models(
79
try:
80
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
81
model_info.check_available_online(on_fail="skip")
93
- hf_version_check = model_info.check_transformers_version(
94
- on_fail="return")
82
+ model_info.check_transformers_version(on_fail="skip")
83
except ValueError:
96
- hf_version_check = None
97
-
98
- if hf_version_check is not None:
99
- print(f"Skipping transformers comparison because: {hf_version_check}")
84
+ pass
85
86
with hf_runner(model) as hf_model:
102
- if model not in HF_UNSUPPORTED_MODELS and hf_version_check is None:
103
- hf_outputs = hf_model.generate_greedy_logprobs_limit(
104
- example_prompts, max_tokens, num_logprobs)
105
- else:
106
- hf_outputs = None
87
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
88
+ example_prompts, max_tokens, num_logprobs)
89
90
with monkeypatch.context() as m:
91
m.setenv("VLLM_USE_V1", "0")
@@ -121,7 +103,7 @@ def test_models(
103
else:
104
vllm_v1_outputs = None
105
124
- if hf_outputs is not None and vllm_v0_outputs is not None:
106
+ if vllm_v0_outputs is not None:
107
check_logprobs_close(
108
outputs_0_lst=hf_outputs,
109
outputs_1_lst=vllm_v0_outputs,
@@ -130,12 +112,10 @@ def test_models(
112
)
113
114
if model in V1_SUPPORTED_MODELS:
133
- ref_outputs = hf_outputs if hf_outputs is not None else vllm_v0_outputs
134
- assert ref_outputs is not None
115
check_logprobs_close(
136
- outputs_0_lst=ref_outputs,
116
+ outputs_0_lst=hf_outputs,
117
outputs_1_lst=vllm_v1_outputs,
138
- name_0="hf" if hf_outputs is not None else "vllm-v0",
118
+ name_0="hf",
119
name_1="vllm-v1",
120
)
121
@@ -402,11 +382,8 @@ def test_full_cuda_graph(
382
pass
383
384
with hf_runner(model) as hf_model:
405
- if model not in HF_UNSUPPORTED_MODELS:
406
- hf_outputs = hf_model.generate_greedy_logprobs_limit(
407
- example_prompts, max_tokens, num_logprobs)
408
- else:
409
- hf_outputs = None
385
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
386
+ example_prompts, max_tokens, num_logprobs)
387
388
with monkeypatch.context() as m:
389
m.setenv("VLLM_USE_V1", "0")
@@ -421,7 +398,7 @@ def test_full_cuda_graph(
398
vllm_v1_outputs = vllm_model.generate_greedy_logprobs(
399
example_prompts, max_tokens, num_logprobs)
400
424
- if hf_outputs is not None and vllm_v0_outputs is not None:
401
+ if vllm_v0_outputs is not None:
402
check_logprobs_close(
403
outputs_0_lst=hf_outputs,
404
outputs_1_lst=vllm_v0_outputs,
@@ -429,12 +406,10 @@ def test_full_cuda_graph(
406
name_1="vllm-v0",
407
)
408
432
- ref_outputs = hf_outputs if hf_outputs is not None else vllm_v0_outputs
433
- assert ref_outputs is not None
409
check_logprobs_close(
435
- outputs_0_lst=ref_outputs,
410
+ outputs_0_lst=hf_outputs,
411
outputs_1_lst=vllm_v1_outputs,
437
- name_0="hf" if hf_outputs is not None else "vllm-v0",
412
+ name_0="hf",
413
name_1="vllm-v1",
414
)
415
@@ -460,11 +435,8 @@ def test_fp32_state(
435
pass
436
437
with hf_runner(model) as hf_model:
463
- if model not in HF_UNSUPPORTED_MODELS:
464
- hf_outputs = hf_model.generate_greedy_logprobs_limit(
465
- example_prompts, max_tokens, num_logprobs)
466
- else:
467
- hf_outputs = None
438
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
439
+ example_prompts, max_tokens, num_logprobs)
440
441
with monkeypatch.context() as m:
442
m.setenv("VLLM_USE_V1", "0")
@@ -480,18 +452,16 @@ def test_fp32_state(
452
vllm_v1_outputs = vllm_model.generate_greedy_logprobs(
453
example_prompts, max_tokens, num_logprobs)
454
483
- if hf_outputs is not None:
484
- check_logprobs_close(
485
- outputs_0_lst=hf_outputs,
486
- outputs_1_lst=vllm_v0_outputs,
487
- name_0="hf",
488
- name_1="vllm-v0",
489
- )
455
+ check_logprobs_close(
456
+ outputs_0_lst=hf_outputs,
457
+ outputs_1_lst=vllm_v0_outputs,
458
+ name_0="hf",
459
+ name_1="vllm-v0",
460
+ )
461
491
- ref_outputs = hf_outputs if hf_outputs is not None else vllm_v0_outputs
462
check_logprobs_close(
493
- outputs_0_lst=ref_outputs,
463
+ outputs_0_lst=hf_outputs,
464
outputs_1_lst=vllm_v1_outputs,
495
- name_0="hf" if hf_outputs is not None else "vllm-v0",
465
+ name_0="hf",
466
name_1="vllm-v1",
467
)
tests/models/registry.py
+9
-4
@@ -154,7 +154,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
154
"BailingMoeForCausalLM": _HfExamplesInfo("inclusionAI/Ling-lite-1.5",
155
trust_remote_code=True),
156
"BambaForCausalLM": _HfExamplesInfo("ibm-ai-platform/Bamba-9B-v1",
157
- min_transformers_version="4.56.0",
157
+ min_transformers_version="4.55.3",
158
extras={"tiny": "hmellor/tiny-random-BambaForCausalLM"}), # noqa: E501
159
"BloomForCausalLM": _HfExamplesInfo("bigscience/bloom-560m",
160
{"1b": "bigscience/bloomz-1b1"}),
@@ -208,7 +208,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
208
"GptOssForCausalLM": _HfExamplesInfo("lmsys/gpt-oss-20b-bf16"),
209
"GraniteForCausalLM": _HfExamplesInfo("ibm/PowerLM-3b"),
210
"GraniteMoeForCausalLM": _HfExamplesInfo("ibm/PowerMoE-3b"),
211
- "GraniteMoeHybridForCausalLM": _HfExamplesInfo("ibm-granite/granite-4.0-tiny-preview"), # noqa: E501
211
+ "GraniteMoeHybridForCausalLM": _HfExamplesInfo("ibm-granite/granite-4.0-tiny-preview", # noqa: E501
212
+ min_transformers_version="4.55.3"),
213
"GraniteMoeSharedForCausalLM": _HfExamplesInfo("ibm-research/moe-7b-1b-active-shared-experts"), # noqa: E501
214
"Grok1ModelForCausalLM": _HfExamplesInfo("hpcai-tech/grok-1",
215
trust_remote_code=True),
@@ -228,7 +229,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
229
trust_remote_code=True),
230
"JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
231
"JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini",
231
- min_transformers_version="4.56.0",
232
+ min_transformers_version="4.55.3",
233
extras={
234
"tiny": "ai21labs/Jamba-tiny-dev",
235
"random": "ai21labs/Jamba-tiny-random", # noqa: E501
@@ -244,7 +245,11 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
245
"Llama4ForCausalLM": _HfExamplesInfo("meta-llama/Llama-4-Scout-17B-16E-Instruct", # noqa: E501
246
is_available_online=False),
247
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
247
- "Mamba2ForCausalLM": _HfExamplesInfo("mistralai/Mamba-Codestral-7B-v0.1"),
248
+ "Mamba2ForCausalLM": _HfExamplesInfo("mistralai/Mamba-Codestral-7B-v0.1",
249
+ min_transformers_version="4.55.3",
250
+ extras={
251
+ "random": "yujiepan/mamba2-codestral-v0.1-tiny-random", # noqa: E501
252
+ }),
253
"FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"), # noqa: E501
254
"MiniCPMForCausalLM": _HfExamplesInfo("openbmb/MiniCPM-2B-sft-bf16",
255
trust_remote_code=True),