A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 23:00 UTC
[Model Runner V2] Add initial CI tests (#36041)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Mar 10, 2026 at 14:55 UTC
8d983d7cd661aae1ac8781f67fbbff017db4d0af
1 file changed
+111
.buildkite/test_areas/model_runner_v2.yaml
new
+111
@@ -0,0 +1,111 @@
1
+group: Model Runner V2
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Model Runner V2 Core Tests
6
+ timeout_in_minutes: 45
7
+ source_file_dependencies:
8
+ - vllm/v1/worker/gpu/
9
+ - vllm/v1/worker/gpu_worker.py
10
+ - vllm/v1/core/sched/
11
+ - vllm/v1/attention/
12
+ - tests/v1/engine/test_llm_engine.py
13
+ - tests/v1/e2e/
14
+ - tests/v1/entrypoints/llm/test_struct_output_generate.py
15
+ commands:
16
+ - set -x
17
+ - export VLLM_USE_V2_MODEL_RUNNER=1
18
+ - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
19
+ # This requires eager until we sort out CG correctness issues.
20
+ # TODO: remove ENFORCE_EAGER here after https://github.com/vllm-project/vllm/pull/32936 is merged.
21
+ - ENFORCE_EAGER=1 pytest -v -s v1/e2e/test_async_scheduling.py -k "not ngram"
22
+ - pytest -v -s v1/e2e/test_context_length.py
23
+ - pytest -v -s v1/e2e/test_min_tokens.py
24
+ # Temporary hack filter to exclude ngram spec decoding based tests.
25
+ - pytest -v -s v1/entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
26
+
27
+- label: Model Runner V2 Examples
28
+ timeout_in_minutes: 45
29
+ working_dir: "/vllm-workspace/examples"
30
+ source_file_dependencies:
31
+ - vllm/v1/worker/gpu/
32
+ - vllm/v1/core/sched/
33
+ - vllm/v1/worker/gpu_worker.py
34
+ - examples/offline_inference/
35
+ - examples/basic/offline_inference/
36
+ - examples/pooling/embed/vision_embedding_offline.py
37
+ - examples/others/tensorize_vllm_model.py
38
+ commands:
39
+ - set -x
40
+ - export VLLM_USE_V2_MODEL_RUNNER=1
41
+ - pip install tensorizer # for tensorizer test
42
+ - python3 basic/offline_inference/chat.py # for basic
43
+ - python3 basic/offline_inference/generate.py --model facebook/opt-125m
44
+ #- python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 # TODO
45
+ #- python3 basic/offline_inference/embed.py # TODO
46
+ # for multi-modal models
47
+ - python3 offline_inference/audio_language.py --seed 0
48
+ - python3 offline_inference/vision_language.py --seed 0
49
+ - python3 offline_inference/vision_language_multi_image.py --seed 0
50
+ # TODO: uncomment once https://github.com/vllm-project/vllm/pull/35790 is merged.
51
+ #- python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0 # TODO
52
+ # for pooling models
53
+ - python3 pooling/embed/vision_embedding_offline.py --seed 0
54
+ # for features demo
55
+ - python3 offline_inference/prefix_caching.py
56
+ - python3 offline_inference/llm_engine_example.py
57
+ - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
58
+ - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
59
+ # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
60
+ - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
61
+
62
+- label: Model Runner V2 Distributed (2 GPUs)
63
+ timeout_in_minutes: 45
64
+ working_dir: "/vllm-workspace/tests"
65
+ num_devices: 2
66
+ source_file_dependencies:
67
+ - vllm/v1/worker/gpu/
68
+ - vllm/v1/worker/gpu_worker.py
69
+ - tests/basic_correctness/test_basic_correctness.py
70
+ - tests/v1/distributed/test_async_llm_dp.py
71
+ - tests/v1/distributed/test_eagle_dp.py
72
+ commands:
73
+ - set -x
74
+ - export VLLM_USE_V2_MODEL_RUNNER=1
75
+ # The "and not True" here is a hacky way to exclude the prompt_embeds cases which aren't yet supported.
76
+ - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True"
77
+ # https://github.com/NVIDIA/nccl/issues/1838
78
+ - export NCCL_CUMEM_HOST_ENABLE=0
79
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray"
80
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
81
+
82
+# These require fix https://github.com/vllm-project/vllm/pull/36280
83
+- label: Model Runner V2 Pipeline Parallelism (4 GPUs)
84
+ timeout_in_minutes: 60
85
+ working_dir: "/vllm-workspace/tests"
86
+ num_devices: 4
87
+ source_file_dependencies:
88
+ - vllm/v1/worker/gpu/
89
+ - vllm/v1/worker/gpu_worker.py
90
+ - tests/distributed/test_pipeline_parallel.py
91
+ #- tests/distributed/test_pp_cudagraph.py
92
+ commands:
93
+ - set -x
94
+ - export VLLM_USE_V2_MODEL_RUNNER=1
95
+ - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray and not Jamba"
96
+ # TODO: Uncomment once https://github.com/vllm-project/vllm/pull/35162 is merged.
97
+ #- pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
98
+
99
+- label: Model Runner V2 Spec Decode
100
+ timeout_in_minutes: 30
101
+ working_dir: "/vllm-workspace/tests"
102
+ source_file_dependencies:
103
+ - vllm/v1/worker/gpu/
104
+ - vllm/v1/worker/gpu_worker.py
105
+ - tests/v1/spec_decode/test_max_len.py
106
+ - tests/v1/e2e/test_spec_decode.py
107
+ commands:
108
+ - set -x
109
+ - export VLLM_USE_V2_MODEL_RUNNER=1
110
+ - pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp"
111
+ - pytest -v -s v1/e2e/test_spec_decode.py -k "eagle or mtp"