A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 18:15 UTC
[CI] Split V1 e2e + engine (1 GPU) into separate jobs (#36945)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Kevin H. Luu committed
Mar 13, 2026 at 14:16 UTC
f1816fb1920c1746c483bd1b67238d1cc85de46f
18 files changed
+81
-39
.buildkite/test-amd.yaml
+6
-6
@@ -369,7 +369,7 @@ steps:
369
- vllm/
370
- tests/v1
371
commands:
372
- - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism"
372
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
373
374
- label: V1 Test e2e (4 GPUs) # 65min
375
timeout_in_minutes: 90
@@ -380,7 +380,7 @@ steps:
380
- vllm/
381
- tests/v1
382
commands:
383
- - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy"
383
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
384
385
- label: V1 Test entrypoints # 35min
386
timeout_in_minutes: 50
@@ -1744,7 +1744,7 @@ steps:
1744
- tests/v1
1745
commands:
1746
# Only run tests that need exactly 2 GPUs
1747
- - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism"
1747
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
1748
1749
- label: V1 Test e2e (4 GPUs) # 65min
1750
timeout_in_minutes: 90
@@ -1759,7 +1759,7 @@ steps:
1759
- tests/v1
1760
commands:
1761
# Only run tests that need 4 GPUs
1762
- - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy"
1762
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
1763
1764
- label: V1 Test entrypoints # 35min
1765
timeout_in_minutes: 50
@@ -3494,7 +3494,7 @@ steps:
3494
- tests/v1
3495
commands:
3496
# Only run tests that need exactly 2 GPUs
3497
- - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism"
3497
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
3498
3499
- label: V1 Test e2e (4 GPUs) # 65min
3500
timeout_in_minutes: 90
@@ -3509,7 +3509,7 @@ steps:
3509
- tests/v1
3510
commands:
3511
# Only run tests that need 4 GPUs
3512
- - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy"
3512
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
3513
3514
- label: V1 Test entrypoints # 35min
3515
timeout_in_minutes: 50
.buildkite/test_areas/engine.yaml
+23
-21
@@ -1,5 +1,5 @@
1
group: Engine
2
-depends_on:
2
+depends_on:
3
- image-build
4
steps:
5
- label: Engine
@@ -14,28 +14,30 @@ steps:
14
commands:
15
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
16
17
-- label: V1 e2e + engine (1 GPU)
18
- timeout_in_minutes: 45
17
+- label: Engine (1 GPU)
18
+ timeout_in_minutes: 30
19
source_file_dependencies:
20
- - vllm/
21
- - tests/v1
20
+ - vllm/v1/engine/
21
+ - tests/v1/engine/
22
commands:
23
- # TODO: accuracy does not match, whether setting
24
- # VLLM_USE_FLASHINFER_SAMPLER or not on H100.
25
- - pytest -v -s v1/e2e
26
- # Run this test standalone for now;
27
- # need to untangle use (implicit) use of spawn/fork across the tests.
23
- pytest -v -s v1/engine/test_preprocess_error_handling.py
29
- # Run the rest of v1/engine tests
24
- pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
31
- mirror:
32
- amd:
33
- device: mi325_1
34
- depends_on:
35
- - image-build-amd
36
- commands:
37
- - pytest -v -s v1/e2e
38
- - pytest -v -s v1/engine
25
+
26
+- label: e2e Scheduling (1 GPU)
27
+ timeout_in_minutes: 30
28
+ source_file_dependencies:
29
+ - vllm/v1/
30
+ - tests/v1/e2e/general/
31
+ commands:
32
+ - pytest -v -s v1/e2e/general/test_async_scheduling.py
33
+
34
+- label: e2e Core (1 GPU)
35
+ timeout_in_minutes: 30
36
+ source_file_dependencies:
37
+ - vllm/v1/
38
+ - tests/v1/e2e/general/
39
+ commands:
40
+ - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
41
42
- label: V1 e2e (2 GPUs)
43
timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability
@@ -46,7 +48,7 @@ steps:
48
- tests/v1/e2e
49
commands:
50
# Only run tests that need exactly 2 GPUs
49
- - pytest -v -s v1/e2e/test_spec_decode.py -k "tensor_parallelism"
51
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
52
mirror:
53
amd:
54
device: mi325_2
@@ -62,7 +64,7 @@ steps:
64
- tests/v1/e2e
65
commands:
66
# Only run tests that need 4 GPUs
65
- - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle_correctness_heavy"
67
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
68
mirror:
69
amd:
70
device: mi325_4
.buildkite/test_areas/model_runner_v2.yaml
+5
-5
@@ -18,9 +18,9 @@ steps:
18
- pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
19
# This requires eager until we sort out CG correctness issues.
20
# TODO: remove ENFORCE_EAGER here after https://github.com/vllm-project/vllm/pull/32936 is merged.
21
- - ENFORCE_EAGER=1 pytest -v -s v1/e2e/test_async_scheduling.py -k "not ngram"
22
- - pytest -v -s v1/e2e/test_context_length.py
23
- - pytest -v -s v1/e2e/test_min_tokens.py
21
+ - ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
22
+ - pytest -v -s v1/e2e/general/test_context_length.py
23
+ - pytest -v -s v1/e2e/general/test_min_tokens.py
24
# Temporary hack filter to exclude ngram spec decoding based tests.
25
- pytest -v -s v1/entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
26
@@ -102,9 +102,9 @@ steps:
102
- vllm/v1/worker/gpu/
103
- vllm/v1/worker/gpu_worker.py
104
- tests/v1/spec_decode/test_max_len.py
105
- - tests/v1/e2e/test_spec_decode.py
105
+ - tests/v1/e2e/spec_decode/test_spec_decode.py
106
commands:
107
- set -x
108
- export VLLM_USE_V2_MODEL_RUNNER=1
109
- pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp"
110
- - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle or mtp"
110
+ - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle or mtp"
.buildkite/test_areas/spec_decode.yaml
new
+40
@@ -0,0 +1,40 @@
1
+group: Spec Decode
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Spec Decode Eagle
6
+ timeout_in_minutes: 30
7
+ source_file_dependencies:
8
+ - vllm/v1/spec_decode/
9
+ - vllm/v1/worker/gpu/spec_decode/
10
+ - tests/v1/e2e/spec_decode/
11
+ commands:
12
+ - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
13
+
14
+- label: Spec Decode Speculators + MTP
15
+ timeout_in_minutes: 30
16
+ source_file_dependencies:
17
+ - vllm/v1/spec_decode/
18
+ - vllm/v1/worker/gpu/spec_decode/
19
+ - vllm/transformers_utils/configs/speculators/
20
+ - tests/v1/e2e/spec_decode/
21
+ commands:
22
+ - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
23
+
24
+- label: Spec Decode Ngram + Suffix
25
+ timeout_in_minutes: 30
26
+ source_file_dependencies:
27
+ - vllm/v1/spec_decode/
28
+ - vllm/v1/worker/gpu/spec_decode/
29
+ - tests/v1/e2e/spec_decode/
30
+ commands:
31
+ - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
32
+
33
+- label: Spec Decode Draft Model
34
+ timeout_in_minutes: 30
35
+ source_file_dependencies:
36
+ - vllm/v1/spec_decode/
37
+ - vllm/v1/worker/gpu/spec_decode/
38
+ - tests/v1/e2e/spec_decode/
39
+ commands:
40
+ - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
tests/v1/e2e/general/__init__.py
tests/v1/e2e/general/test_async_scheduling.py
renamed
+2
-2
@@ -14,8 +14,8 @@ from vllm.platforms import current_platform
14
from vllm.sampling_params import StructuredOutputsParams
15
from vllm.v1.metrics.reader import Metric
16
17
-from ...conftest import VllmRunner
18
-from ...models.utils import check_outputs_equal
17
+from ....conftest import VllmRunner
18
+from ....models.utils import check_outputs_equal
19
20
MODEL = "Qwen/Qwen3-0.6B"
21
MTP_MODEL = "meta-llama/Llama-3.2-1B-Instruct"
tests/v1/e2e/general/test_cascade_attention.py
renamed
+1
-1
@@ -5,7 +5,7 @@ import pytest
5
6
from vllm import LLM, SamplingParams
7
8
-from ...utils import create_new_process_for_each_test
8
+from ....utils import create_new_process_for_each_test
9
10
11
@create_new_process_for_each_test()
tests/v1/e2e/general/test_context_length.py
renamed
tests/v1/e2e/general/test_correctness_sliding_window.py
renamed
+1
-1
@@ -7,7 +7,7 @@ import pytest
7
from vllm import LLM, SamplingParams
8
from vllm.platforms import current_platform
9
10
-from ...utils import check_answers, prep_prompts
10
+from ....utils import check_answers, prep_prompts
11
12
13
@dataclass
tests/v1/e2e/general/test_kv_sharing_fast_prefill.py
renamed
+2
-2
@@ -9,7 +9,7 @@ from vllm import LLM, SamplingParams
9
from vllm.config import CompilationConfig, CompilationMode
10
from vllm.platforms import current_platform
11
12
-from ...utils import check_answers, fork_new_process_for_each_test, prep_prompts
12
+from ....utils import check_answers, fork_new_process_for_each_test, prep_prompts
13
14
# global seed
15
SEED = 42
@@ -18,7 +18,7 @@ SEED = 42
18
@pytest.fixture
19
def test_prompts():
20
"""
21
- Adapted from tests/v1/e2e/test_spec_decode.py
21
+ Adapted from tests/v1/e2e/spec_decode/test_spec_decode.py
22
"""
23
prompt_types = ["repeat", "sentence"]
24
# Setting higher num prompts increases the chance of numerics mismatch
tests/v1/e2e/general/test_mamba_prefix_cache.py
renamed
tests/v1/e2e/general/test_min_tokens.py
renamed
+1
-1
@@ -497,6 +497,6 @@ if __name__ == "__main__":
497
498
Usage:
499
cd vllm/
500
- python -m pytest tests/v1/e2e/test_min_tokens.py -v
500
+ python -m pytest tests/v1/e2e/general/test_min_tokens.py -v
501
"""
502
pytest.main([__file__, "-v"])