A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Speed up model unit tests in CI (#24253)
Signed-off-by: Andrew Feldman <afeldman@redhat.com>
afeldman-nm committed
Sep 12, 2025 at 13:36 UTC
c8c42597abd5c6843fb8b6b02c0f62a5d3f284c8
6 files changed
+123
-21
.buildkite/test-pipeline.yaml
+66
-12
@@ -571,36 +571,85 @@ steps:
571
572
##### models test #####
573
574
-- label: Basic Models Test # 57min
575
- timeout_in_minutes: 75
574
+- label: Basic Models Tests (Initialization)
575
+ timeout_in_minutes: 45
576
mirror_hardwares: [amdexperimental]
577
torch_nightly: true
578
source_file_dependencies:
579
- vllm/
580
- - tests/models
580
+ - tests/models/test_initialization.py
581
commands:
582
- - pytest -v -s models/test_transformers.py
583
- - pytest -v -s models/test_registry.py
584
- - pytest -v -s models/test_utils.py
585
- - pytest -v -s models/test_vision.py
586
- - pytest -v -s models/test_initialization.py
582
+ # Run a subset of model initialization tests
583
+ - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
584
588
-- label: Language Models Test (Standard) # 35min
585
+- label: Basic Models Tests (Extra Initialization) %N
586
timeout_in_minutes: 45
587
mirror_hardwares: [amdexperimental]
588
torch_nightly: true
589
source_file_dependencies:
590
+ - vllm/model_executor/models/
591
+ - tests/models/test_initialization.py
592
+ commands:
593
+ # Only when vLLM model source is modified - test initialization of a large
594
+ # subset of supported models (the complement of the small subset in the above
595
+ # test.) Also run if model initialization test file is modified
596
+ - pytest -v -s models/test_initialization.py \
597
+ -k 'not test_can_initialize_small_subset' \
598
+ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
599
+ --shard-id=$$BUILDKITE_PARALLEL_JOB
600
+ parallelism: 2
601
+
602
+- label: Basic Models Tests (Other)
603
+ timeout_in_minutes: 45
604
+ mirror_hardwares: [amdexperimental]
605
+ torch_nightly: true
606
+ source_file_dependencies:
607
+ - vllm/
608
+ - tests/models/test_transformers.py
609
+ - tests/models/test_registry.py
610
+ - tests/models/test_utils.py
611
+ - tests/models/test_vision.py
612
+ commands:
613
+ - pytest -v -s models/test_transformers.py \
614
+ models/test_registry.py \
615
+ models/test_utils.py \
616
+ models/test_vision.py
617
+
618
+- label: Language Models Tests (Standard)
619
+ timeout_in_minutes: 25
620
+ mirror_hardwares: [amdexperimental]
621
+ torch_nightly: true
622
+ source_file_dependencies:
623
- vllm/
624
- tests/models/language
625
commands:
626
+ # Test standard language models, excluding a subset of slow tests
627
- pip freeze | grep -E 'torch'
597
- - pytest -v -s models/language -m core_model
628
+ - pytest -v -s models/language -m 'core_model and (not slow_test)'
629
599
-- label: Language Models Test (Hybrid) # 35 min
630
+- label: Language Models Tests (Extra Standard) %N
631
timeout_in_minutes: 45
632
mirror_hardwares: [amdexperimental]
633
torch_nightly: true
634
source_file_dependencies:
635
+ - vllm/model_executor/models/
636
+ - tests/models/language/pooling/test_embedding.py
637
+ - tests/models/language/generation/test_common.py
638
+ - tests/models/language/pooling/test_classification.py
639
+ commands:
640
+ # Shard slow subset of standard language models tests. Only run when model
641
+ # source is modified, or when specified test files are modified
642
+ - pip freeze | grep -E 'torch'
643
+ - pytest -v -s models/language -m 'core_model and slow_test' \
644
+ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
645
+ --shard-id=$$BUILDKITE_PARALLEL_JOB
646
+ parallelism: 2
647
+
648
+- label: Language Models Tests (Hybrid) %N
649
+ timeout_in_minutes: 75
650
+ mirror_hardwares: [amdexperimental]
651
+ torch_nightly: true
652
+ source_file_dependencies:
653
- vllm/
654
- tests/models/language/generation
655
commands:
@@ -608,7 +657,12 @@ steps:
657
# Note: also needed to run plamo2 model in vLLM
658
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.2.5'
659
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
611
- - pytest -v -s models/language/generation -m hybrid_model
660
+ # Shard hybrid language model tests
661
+ - pytest -v -s models/language/generation \
662
+ -m hybrid_model \
663
+ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \
664
+ --shard-id=$$BUILDKITE_PARALLEL_JOB
665
+ parallelism: 2
666
667
- label: Language Models Test (Extended Generation) # 80min
668
timeout_in_minutes: 110
pyproject.toml
+1
@@ -145,6 +145,7 @@ skip_gitignore = true
145
146
[tool.pytest.ini_options]
147
markers = [
148
+ "slow_test",
149
"skip_global_cleanup",
150
"core_model: enable this model test in each PR instead of only nightly",
151
"hybrid_model: models that contain mamba layers (including pure SSM and hybrid architectures)",
tests/models/language/generation/test_common.py
+10
-4
@@ -38,7 +38,7 @@ AITER_MODEL_LIST = [
38
[
39
pytest.param(
40
"bigscience/bloom-560m", # bloom - testing alibi slopes
41
- marks=[pytest.mark.core_model],
41
+ marks=[pytest.mark.core_model, pytest.mark.slow_test],
42
),
43
pytest.param(
44
"openai-community/gpt2", # gpt2
@@ -49,7 +49,10 @@ AITER_MODEL_LIST = [
49
pytest.param("EleutherAI/pythia-70m"), # gpt_neox
50
pytest.param(
51
"google/gemma-1.1-2b-it", # gemma
52
- marks=[pytest.mark.core_model, pytest.mark.cpu_model],
52
+ marks=[
53
+ pytest.mark.core_model, pytest.mark.cpu_model,
54
+ pytest.mark.slow_test
55
+ ],
56
),
57
pytest.param(
58
"zai-org/chatglm3-6b", # chatglm (text-only)
@@ -70,14 +73,17 @@ AITER_MODEL_LIST = [
73
),
74
pytest.param(
75
"microsoft/phi-2", # phi
73
- marks=[pytest.mark.core_model],
76
+ marks=[pytest.mark.core_model, pytest.mark.slow_test],
77
),
78
pytest.param(
79
"Qwen/Qwen-7B-Chat", # qwen (text-only)
80
),
81
pytest.param(
82
"Qwen/Qwen2.5-0.5B-Instruct", # qwen2
80
- marks=[pytest.mark.core_model, pytest.mark.cpu_model],
83
+ marks=[
84
+ pytest.mark.core_model, pytest.mark.cpu_model,
85
+ pytest.mark.slow_test
86
+ ],
87
),
88
pytest.param(
89
"Qwen/Qwen3-8B", # qwen (text-only)
tests/models/language/pooling/test_classification.py
+4
-1
@@ -11,7 +11,10 @@ from vllm.platforms import current_platform
11
"model",
12
[
13
pytest.param("jason9693/Qwen2.5-1.5B-apeach",
14
- marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
14
+ marks=[
15
+ pytest.mark.core_model, pytest.mark.cpu_model,
16
+ pytest.mark.slow_test
17
+ ]),
18
],
19
)
20
@pytest.mark.parametrize("dtype",
tests/models/language/pooling/test_embedding.py
+5
-2
@@ -19,7 +19,7 @@ from ...utils import check_embeddings_close
19
# model code with bidirectional attention.
20
# [Decoder-only]
21
pytest.param("BAAI/bge-multilingual-gemma2",
22
- marks=[pytest.mark.core_model]),
22
+ marks=[pytest.mark.core_model, pytest.mark.slow_test]),
23
pytest.param(
24
"intfloat/e5-mistral-7b-instruct",
25
# CPU v1 doesn't support sliding window
@@ -29,7 +29,10 @@ from ...utils import check_embeddings_close
29
# [Encoder-only]
30
pytest.param(
31
"BAAI/bge-base-en-v1.5",
32
- marks=[pytest.mark.core_model, pytest.mark.cpu_model],
32
+ marks=[
33
+ pytest.mark.core_model, pytest.mark.cpu_model,
34
+ pytest.mark.slow_test
35
+ ],
36
),
37
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
38
pytest.param("intfloat/multilingual-e5-small"),
tests/models/test_initialization.py
+37
-2
@@ -18,6 +18,26 @@ from .registry import (_TRANSFORMERS_BACKEND_MODELS, AUTO_EXAMPLE_MODELS,
18
HF_EXAMPLE_MODELS, HfExampleModels)
19
from .utils import dummy_hf_overrides
20
21
+# This minimal list of model architectures is smaller than the total list of
22
+# supported models. The intention is that in the "typical" regression testing
23
+# scenario, we only test initializing these models. This subset was chosen
24
+# to include representative examples of model varieties/workloads (conditional
25
+# generation, sequence classification, causal LM, ranking, chat, reward model,
26
+# multimodal, geospatial, voice, embedding, MTP)
27
+MINIMAL_MODEL_ARCH_LIST = [
28
+ "LlavaForConditionalGeneration", "Llama4ForConditionalGeneration",
29
+ "BertForSequenceClassification", "Gemma3nForCausalLM", "JinaVLForRanking",
30
+ "InternVLChatModel", "InternLM2ForRewardModel",
31
+ "TransformersForMultimodalLM", "PrithviGeoSpatialMAE", "UltravoxModel",
32
+ "DeepSeekMTPModel", "XLMRobertaModel"
33
+]
34
+
35
+# This list is the complement of the minimal list above. The intention is that
36
+# this list of models is only tested in a "special case" i.e. most PRs should
37
+# not test these models
38
+OTHER_MODEL_ARCH_LIST = (set(HF_EXAMPLE_MODELS.get_supported_archs()) -
39
+ set(MINIMAL_MODEL_ARCH_LIST))
40
+
41
42
@create_new_process_for_each_test()
43
def can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch,
@@ -101,8 +121,23 @@ def can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch,
121
max_num_seqs=model_info.max_num_seqs)
122
123
104
-@pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
105
-def test_can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch):
124
+@pytest.mark.parametrize("model_arch", MINIMAL_MODEL_ARCH_LIST)
125
+def test_can_initialize_small_subset(model_arch: str,
126
+ monkeypatch: pytest.MonkeyPatch):
127
+ """Test initializing small subset of supported models"""
128
+ if model_arch == "Lfm2ForCausalLM":
129
+ pytest.skip("Skipping until test supports V1-only models")
130
+ can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)
131
+
132
+
133
+@pytest.mark.parametrize("model_arch", OTHER_MODEL_ARCH_LIST)
134
+def test_can_initialize_large_subset(model_arch: str,
135
+ monkeypatch: pytest.MonkeyPatch):
136
+ """Test initializing large subset of supported models
137
+
138
+ This test covers the complement of the tests covered in the "small subset"
139
+ test.
140
+ """
141
if model_arch == "Lfm2ForCausalLM":
142
pytest.skip("Skipping until test supports V1-only models")
143
can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)