A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 20:00 UTC
[ci] Refactor CI file structure (#29343)
Kevin H. Luu committed
Dec 8, 2025 at 18:25 UTC
db14f61f2d6d33e38d382f2e3e7de514dac8e218
31 files changed
+1570
.buildkite/ci_config.yaml
new
+24
@@ -0,0 +1,24 @@
1
+name: vllm_ci
2
+job_dirs:
3
+ - ".buildkite/test_areas"
4
+ - ".buildkite/image_build"
5
+run_all_patterns:
6
+ - "docker/Dockerfile"
7
+ - "CMakeLists.txt"
8
+ - "requirements/common.txt"
9
+ - "requirements/cuda.txt"
10
+ - "requirements/build.txt"
11
+ - "requirements/test.txt"
12
+ - "setup.py"
13
+ - "csrc/"
14
+ - "cmake/"
15
+run_all_exclude_patterns:
16
+ - "docker/Dockerfile."
17
+ - "csrc/cpu/"
18
+ - "csrc/rocm/"
19
+ - "cmake/hipify.py"
20
+ - "cmake/cpu_extension.cmake"
21
+registries: public.ecr.aws/q9t5s3a7
22
+repositories:
23
+ main: "vllm-ci-postmerge-repo"
24
+ premerge: "vllm-ci-test-repo"
.buildkite/image_build/image_build.sh
new
+56
@@ -0,0 +1,56 @@
1
+#!/bin/bash
2
+set -e
3
+
4
+if [[ $# -lt 8 ]]; then
5
+ echo "Usage: $0 <registry> <repo> <commit> <branch> <vllm_use_precompiled> <vllm_merge_base_commit> <cache_from> <cache_to>"
6
+ exit 1
7
+fi
8
+
9
+REGISTRY=$1
10
+REPO=$2
11
+BUILDKITE_COMMIT=$3
12
+BRANCH=$4
13
+VLLM_USE_PRECOMPILED=$5
14
+VLLM_MERGE_BASE_COMMIT=$6
15
+CACHE_FROM=$7
16
+CACHE_TO=$8
17
+
18
+# authenticate with AWS ECR
19
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REGISTRY
20
+aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
21
+
22
+# docker buildx
23
+docker buildx create --name vllm-builder --driver docker-container --use
24
+docker buildx inspect --bootstrap
25
+docker buildx ls
26
+
27
+# skip build if image already exists
28
+if [[ -z $(docker manifest inspect $REGISTRY/$REPO:$BUILDKITE_COMMIT) ]]; then
29
+ echo "Image not found, proceeding with build..."
30
+else
31
+ echo "Image found"
32
+ exit 0
33
+fi
34
+
35
+if [[ "${VLLM_USE_PRECOMPILED:-0}" == "1" ]]; then
36
+ merge_base_commit_build_args="--build-arg VLLM_MERGE_BASE_COMMIT=${VLLM_MERGE_BASE_COMMIT}"
37
+else
38
+ merge_base_commit_build_args=""
39
+fi
40
+
41
+# build
42
+docker buildx build --file docker/Dockerfile \
43
+ --build-arg max_jobs=16 \
44
+ --build-arg buildkite_commit=$BUILDKITE_COMMIT \
45
+ --build-arg USE_SCCACHE=1 \
46
+ --build-arg TORCH_CUDA_ARCH_LIST="8.0 8.9 9.0 10.0" \
47
+ --build-arg FI_TORCH_CUDA_ARCH_LIST="8.0 8.9 9.0a 10.0a" \
48
+ --build-arg VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED:-0}" \
49
+ ${merge_base_commit_build_args} \
50
+ --cache-from type=registry,ref=${CACHE_FROM},mode=max \
51
+ --cache-to type=registry,ref=${CACHE_TO},mode=max \
52
+ --tag ${REGISTRY}/${REPO}:${BUILDKITE_COMMIT} \
53
+ $( [[ "${BRANCH}" == "main" ]] && echo "--tag ${REGISTRY}/${REPO}:latest" ) \
54
+ --push \
55
+ --target test \
56
+ --progress plain .
.buildkite/image_build/image_build.yaml
new
+57
@@ -0,0 +1,57 @@
1
+group: Abuild
2
+steps:
3
+ - label: ":docker: Build image"
4
+ key: image-build
5
+ depends_on: []
6
+ commands:
7
+ - .buildkite/image_build/image_build.sh $REGISTRY $REPO $BUILDKITE_COMMIT $BRANCH $VLLM_USE_PRECOMPILED $VLLM_MERGE_BASE_COMMIT $CACHE_FROM $CACHE_TO
8
+ retry:
9
+ automatic:
10
+ - exit_status: -1 # Agent was lost
11
+ limit: 2
12
+ - exit_status: -10 # Agent was lost
13
+ limit: 2
14
+
15
+ - label: ":docker: Build CPU image"
16
+ key: image-build-cpu
17
+ depends_on: []
18
+ commands:
19
+ - .buildkite/image_build/image_build_cpu.sh $REGISTRY $REPO $BUILDKITE_COMMIT
20
+ env:
21
+ DOCKER_BUILDKIT: "1"
22
+ retry:
23
+ automatic:
24
+ - exit_status: -1 # Agent was lost
25
+ limit: 2
26
+ - exit_status: -10 # Agent was lost
27
+ limit: 2
28
+
29
+ - label: ":docker: Build HPU image"
30
+ soft_fail: true
31
+ depends_on: []
32
+ key: image-build-hpu
33
+ commands:
34
+ - .buildkite/image_build/image_build_hpu.sh $REGISTRY $REPO $BUILDKITE_COMMIT
35
+ env:
36
+ DOCKER_BUILDKIT: "1"
37
+ retry:
38
+ automatic:
39
+ - exit_status: -1 # Agent was lost
40
+ limit: 2
41
+ - exit_status: -10 # Agent was lost
42
+ limit: 2
43
+
44
+ - label: ":docker: Build CPU arm64 image"
45
+ key: cpu-arm64-image-build
46
+ depends_on: []
47
+ optional: true
48
+ commands:
49
+ - .buildkite/image_build/image_build_cpu_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT
50
+ env:
51
+ DOCKER_BUILDKIT: "1"
52
+ retry:
53
+ automatic:
54
+ - exit_status: -1 # Agent was lost
55
+ limit: 2
56
+ - exit_status: -10 # Agent was lost
57
+ limit: 2
.buildkite/image_build/image_build_cpu.sh
new
+36
@@ -0,0 +1,36 @@
1
+#!/bin/bash
2
+set -e
3
+
4
+if [[ $# -lt 3 ]]; then
5
+ echo "Usage: $0 <registry> <repo> <commit>"
6
+ exit 1
7
+fi
8
+
9
+REGISTRY=$1
10
+REPO=$2
11
+BUILDKITE_COMMIT=$3
12
+
13
+# authenticate with AWS ECR
14
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REGISTRY
15
+
16
+# skip build if image already exists
17
+if [[ -z $(docker manifest inspect $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu) ]]; then
18
+ echo "Image not found, proceeding with build..."
19
+else
20
+ echo "Image found"
21
+ exit 0
22
+fi
23
+
24
+# build
25
+docker build --file docker/Dockerfile.cpu \
26
+ --build-arg max_jobs=16 \
27
+ --build-arg buildkite_commit=$BUILDKITE_COMMIT \
28
+ --build-arg VLLM_CPU_AVX512BF16=true \
29
+ --build-arg VLLM_CPU_AVX512VNNI=true \
30
+ --build-arg VLLM_CPU_AMXBF16=true \
31
+ --tag $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu \
32
+ --target vllm-test \
33
+ --progress plain .
34
+
35
+# push
36
+docker push $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu
.buildkite/image_build/image_build_cpu_arm64.sh
new
+33
@@ -0,0 +1,33 @@
1
+#!/bin/bash
2
+set -e
3
+
4
+if [[ $# -lt 3 ]]; then
5
+ echo "Usage: $0 <registry> <repo> <commit>"
6
+ exit 1
7
+fi
8
+
9
+REGISTRY=$1
10
+REPO=$2
11
+BUILDKITE_COMMIT=$3
12
+
13
+# authenticate with AWS ECR
14
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REGISTRY
15
+
16
+# skip build if image already exists
17
+if [[ -z $(docker manifest inspect $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu) ]]; then
18
+ echo "Image not found, proceeding with build..."
19
+else
20
+ echo "Image found"
21
+ exit 0
22
+fi
23
+
24
+# build
25
+docker build --file docker/Dockerfile.cpu \
26
+ --build-arg max_jobs=16 \
27
+ --build-arg buildkite_commit=$BUILDKITE_COMMIT \
28
+ --tag $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu \
29
+ --target vllm-test \
30
+ --progress plain .
31
+
32
+# push
33
+docker push $REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu
.buildkite/image_build/image_build_hpu.sh
new
+34
@@ -0,0 +1,34 @@
1
+#!/bin/bash
2
+set -e
3
+
4
+if [[ $# -lt 3 ]]; then
5
+ echo "Usage: $0 <registry> <repo> <commit>"
6
+ exit 1
7
+fi
8
+
9
+REGISTRY=$1
10
+REPO=$2
11
+BUILDKITE_COMMIT=$3
12
+
13
+# authenticate with AWS ECR
14
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REGISTRY
15
+
16
+# skip build if image already exists
17
+if [[ -z $(docker manifest inspect $REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu) ]]; then
18
+ echo "Image not found, proceeding with build..."
19
+else
20
+ echo "Image found"
21
+ exit 0
22
+fi
23
+
24
+# build
25
+docker build \
26
+ --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
27
+ --build-arg max_jobs=16 \
28
+ --build-arg buildkite_commit=$BUILDKITE_COMMIT \
29
+ --tag $REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu \
30
+ --progress plain \
31
+ https://github.com/vllm-project/vllm-gaudi.git
32
+
33
+# push
34
+docker push $REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu
.buildkite/test_areas/attention.yaml
new
+21
@@ -0,0 +1,21 @@
1
+group: Attention
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: V1 attention (H100)
6
+ timeout_in_minutes: 30
7
+ gpu: h100
8
+ source_file_dependencies:
9
+ - vllm/v1/attention
10
+ - tests/v1/attention
11
+ commands:
12
+ - pytest -v -s v1/attention
13
+
14
+- label: V1 attention (B200)
15
+ timeout_in_minutes: 30
16
+ gpu: b200
17
+ source_file_dependencies:
18
+ - vllm/v1/attention
19
+ - tests/v1/attention
20
+ commands:
21
+ - VLLM_DISABLE_FLASHINFER_PREFILL=1 pytest -v -s v1/attention # TODO: FI prefill is bugged and causes incorrectness, fix this
.buildkite/test_areas/basic_correctness.yaml
new
+16
@@ -0,0 +1,16 @@
1
+group: Basic Correctness
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Basic Correctness
6
+ timeout_in_minutes: 30
7
+ source_file_dependencies:
8
+ - vllm/
9
+ - tests/basic_correctness/test_basic_correctness
10
+ - tests/basic_correctness/test_cpu_offload
11
+ - tests/basic_correctness/test_cumem.py
12
+ commands:
13
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
14
+ - pytest -v -s basic_correctness/test_cumem.py
15
+ - pytest -v -s basic_correctness/test_basic_correctness.py
16
+ - pytest -v -s basic_correctness/test_cpu_offload.py
.buildkite/test_areas/benchmarks.yaml
new
+19
@@ -0,0 +1,19 @@
1
+group: Benchmarks
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Benchmarks
6
+ timeout_in_minutes: 20
7
+ working_dir: "/vllm-workspace/.buildkite"
8
+ source_file_dependencies:
9
+ - benchmarks/
10
+ commands:
11
+ - bash scripts/run-benchmarks.sh
12
+
13
+- label: Benchmarks CLI Test
14
+ timeout_in_minutes: 20
15
+ source_file_dependencies:
16
+ - vllm/
17
+ - tests/benchmarks/
18
+ commands:
19
+ - pytest -v -s benchmarks/
.buildkite/test_areas/compile.yaml
new
+57
@@ -0,0 +1,57 @@
1
+group: Compile
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Fusion and Compile Tests (B200)
6
+ timeout_in_minutes: 40
7
+ working_dir: "/vllm-workspace/"
8
+ gpu: b200
9
+ source_file_dependencies:
10
+ - csrc/quantization/fp4/
11
+ - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
12
+ - vllm/v1/attention/backends/flashinfer.py
13
+ - vllm/v1/worker/
14
+ - vllm/v1/cudagraph_dispatcher.py
15
+ - vllm/compilation/
16
+ # can affect pattern matching
17
+ - vllm/model_executor/layers/layernorm.py
18
+ - vllm/model_executor/layers/activation.py
19
+ - vllm/model_executor/layers/quantization/input_quant_fp8.py
20
+ - tests/compile/test_fusion_attn.py
21
+ - tests/compile/test_silu_mul_quant_fusion.py
22
+ - tests/compile/distributed/test_fusion_all_reduce.py
23
+ - tests/compile/distributed/test_fusions_e2e.py
24
+ - tests/compile/fullgraph/test_full_graph.py
25
+ commands:
26
+ - nvidia-smi
27
+ - pytest -v -s tests/compile/test_fusion_attn.py
28
+ - pytest -v -s tests/compile/test_silu_mul_quant_fusion.py
29
+ # this runner has 2 GPUs available even though num_gpus=2 is not set
30
+ - pytest -v -s tests/compile/distributed/test_fusion_all_reduce.py
31
+ # Limit to Inductor partition, no custom ops, and allreduce & attn fusion to reduce running time
32
+ # Wrap with quotes to escape yaml
33
+ - "pytest -v -s tests/compile/distributed/test_fusions_e2e.py::test_tp2_attn_quant_allreduce_rmsnorm -k 'True and not +quant_fp8 and not +rms_norm'"
34
+ # test_fp8_kv_scale_compile requires FlashAttention (not supported on default L4/L40)
35
+ - pytest -v -s tests/compile/fullgraph/test_full_graph.py::test_fp8_kv_scale_compile
36
+
37
+- label: Fusion E2E (2 GPUs)(B200)
38
+ timeout_in_minutes: 40
39
+ working_dir: "/vllm-workspace/"
40
+ gpu: b200
41
+ optional: true
42
+ num_gpus: 2
43
+ source_file_dependencies:
44
+ - csrc/quantization/fp4/
45
+ - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
46
+ - vllm/v1/attention/backends/flashinfer.py
47
+ - vllm/compilation/
48
+ # can affect pattern matching
49
+ - vllm/model_executor/layers/layernorm.py
50
+ - vllm/model_executor/layers/activation.py
51
+ - vllm/model_executor/layers/quantization/input_quant_fp8.py
52
+ - tests/compile/distributed/test_fusions_e2e.py
53
+ commands:
54
+ - nvidia-smi
55
+ # Run all e2e fusion tests
56
+ - pytest -v -s tests/compile/distributed/test_fusions_e2e.py
57
+
.buildkite/test_areas/cuda.yaml
new
+22
@@ -0,0 +1,22 @@
1
+group: CUDA
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Platform Tests (CUDA)
6
+ timeout_in_minutes: 15
7
+ source_file_dependencies:
8
+ - vllm/
9
+ - tests/cuda
10
+ commands:
11
+ - pytest -v -s cuda/test_cuda_context.py
12
+
13
+- label: Cudagraph
14
+ timeout_in_minutes: 20
15
+ source_file_dependencies:
16
+ - tests/v1/cudagraph
17
+ - vllm/v1/cudagraph_dispatcher.py
18
+ - vllm/config/compilation.py
19
+ - vllm/compilation
20
+ commands:
21
+ - pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
22
+ - pytest -v -s v1/cudagraph/test_cudagraph_mode.py
\ No newline at end of file
.buildkite/test_areas/distributed.yaml
new
+199
@@ -0,0 +1,199 @@
1
+group: Distributed
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Distributed Comm Ops
6
+ timeout_in_minutes: 20
7
+ working_dir: "/vllm-workspace/tests"
8
+ num_gpus: 2
9
+ source_file_dependencies:
10
+ - vllm/distributed
11
+ - tests/distributed
12
+ commands:
13
+ - pytest -v -s distributed/test_comm_ops.py
14
+ - pytest -v -s distributed/test_shm_broadcast.py
15
+ - pytest -v -s distributed/test_shm_buffer.py
16
+ - pytest -v -s distributed/test_shm_storage.py
17
+
18
+- label: Distributed (2 GPUs)
19
+ timeout_in_minutes: 90
20
+ working_dir: "/vllm-workspace/tests"
21
+ num_gpus: 2
22
+ source_file_dependencies:
23
+ - vllm/compilation/
24
+ - vllm/distributed/
25
+ - vllm/engine/
26
+ - vllm/executor/
27
+ - vllm/worker/worker_base.py
28
+ - vllm/v1/engine/
29
+ - vllm/v1/worker/
30
+ - tests/compile/fullgraph/test_basic_correctness.py
31
+ - tests/compile/test_wrapper.py
32
+ - tests/distributed/
33
+ - tests/entrypoints/llm/test_collective_rpc.py
34
+ - tests/v1/distributed
35
+ - tests/v1/entrypoints/openai/test_multi_api_servers.py
36
+ - tests/v1/shutdown
37
+ - tests/v1/worker/test_worker_memory_snapshot.py
38
+ commands:
39
+ # https://github.com/NVIDIA/nccl/issues/1838
40
+ - export NCCL_CUMEM_HOST_ENABLE=0
41
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
42
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
43
+ - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
44
+ - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
45
+ - pytest -v -s entrypoints/llm/test_collective_rpc.py
46
+ - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
47
+ - pytest -v -s ./compile/test_wrapper.py
48
+ - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
49
+ - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
50
+ - pytest -v -s distributed/test_sequence_parallel.py
51
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
52
+ - pytest -v -s v1/worker/test_worker_memory_snapshot.py
53
+
54
+- label: Distributed Tests (4 GPUs)
55
+ timeout_in_minutes: 50
56
+ working_dir: "/vllm-workspace/tests"
57
+ num_gpus: 4
58
+ source_file_dependencies:
59
+ - vllm/distributed/
60
+ - tests/distributed/test_utils
61
+ - tests/distributed/test_pynccl
62
+ - tests/distributed/test_events
63
+ - tests/compile/fullgraph/test_basic_correctness.py
64
+ - examples/offline_inference/rlhf.py
65
+ - examples/offline_inference/rlhf_colocate.py
66
+ - tests/examples/offline_inference/data_parallel.py
67
+ - tests/v1/distributed
68
+ - tests/v1/engine/test_engine_core_client.py
69
+ - tests/distributed/test_symm_mem_allreduce.py
70
+ commands:
71
+ # https://github.com/NVIDIA/nccl/issues/1838
72
+ - export NCCL_CUMEM_HOST_ENABLE=0
73
+ # test with torchrun tp=2 and external_dp=2
74
+ - torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
75
+ # test with torchrun tp=2 and pp=2
76
+ - PP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example.py
77
+ # test with torchrun tp=4 and dp=1
78
+ - TP_SIZE=4 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
79
+ # test with torchrun tp=2, pp=2 and dp=1
80
+ - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
81
+ # test with torchrun tp=1 and dp=4 with ep
82
+ - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
83
+ # test with torchrun tp=2 and dp=2 with ep
84
+ - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 distributed/test_torchrun_example_moe.py
85
+ # test with internal dp
86
+ - python3 ../examples/offline_inference/data_parallel.py --enforce-eager
87
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
88
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
89
+ - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
90
+ - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
91
+ - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
92
+ - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
93
+ - pytest -v -s distributed/test_utils.py
94
+ - pytest -v -s compile/fullgraph/test_basic_correctness.py
95
+ - pytest -v -s distributed/test_pynccl.py
96
+ - pytest -v -s distributed/test_events.py
97
+ - pytest -v -s distributed/test_symm_mem_allreduce.py
98
+ # TODO: create a dedicated test section for multi-GPU example tests
99
+ # when we have multiple distributed example tests
100
+ - cd ../examples/offline_inference
101
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf.py
102
+ - VLLM_ALLOW_INSECURE_SERIALIZATION=1 RAY_DEDUP_LOGS=0 python3 rlhf_colocate.py
103
+
104
+- label: Distributed Tests (8 GPUs)(H100)
105
+ timeout_in_minutes: 10
106
+ gpu: h100
107
+ num_gpus: 8
108
+ working_dir: "/vllm-workspace/tests"
109
+ source_file_dependencies:
110
+ - examples/offline_inference/torchrun_dp_example.py
111
+ - vllm/config/parallel.py
112
+ - vllm/distributed/
113
+ - vllm/v1/engine/llm_engine.py
114
+ - vllm/v1/executor/uniproc_executor.py
115
+ - vllm/v1/worker/gpu_worker.py
116
+ commands:
117
+ # https://github.com/NVIDIA/nccl/issues/1838
118
+ - export NCCL_CUMEM_HOST_ENABLE=0
119
+ # test with torchrun tp=2 and dp=4 with ep
120
+ - torchrun --nproc-per-node=8 ../examples/offline_inference/torchrun_dp_example.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
121
+
122
+- label: Distributed Tests (4 GPUs)(A100)
123
+ gpu: a100
124
+ optional: true
125
+ num_gpus: 4
126
+ source_file_dependencies:
127
+ - vllm/
128
+ commands:
129
+ # NOTE: don't test llama model here, it seems hf implementation is buggy
130
+ # see https://github.com/vllm-project/vllm/pull/5689 for details
131
+ - pytest -v -s distributed/test_custom_all_reduce.py
132
+ - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
133
+ - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
134
+ - pytest -v -s -x lora/test_mixtral.py
135
+
136
+- label: Distributed Tests (2 GPUs)(H200)
137
+ gpu: h200
138
+ optional: true
139
+ working_dir: "/vllm-workspace/"
140
+ num_gpus: 2
141
+ commands:
142
+ - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/distributed/test_async_tp.py
143
+ - pytest -v -s tests/compile/distributed/test_sequence_parallelism.py
144
+ - pytest -v -s tests/compile/distributed/test_fusion_all_reduce.py
145
+ - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/distributed/test_fusions_e2e.py -k 'not Llama-4'
146
+ - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/distributed/test_sequence_parallel.py
147
+ - pytest -v -s tests/distributed/test_context_parallel.py
148
+ - CUDA_VISIBLE_DEVICES=1,2 VLLM_ALL2ALL_BACKEND=deepep_high_throughput VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model Qwen/Qwen1.5-MoE-A2.7B --tp-size=1 --dp-size=2 --max-model-len 2048
149
+ - pytest -v -s tests/v1/distributed/test_dbo.py
150
+
151
+- label: Distributed Tests (2 GPUs)(B200)
152
+ gpu: b200
153
+ optional: true
154
+ working_dir: "/vllm-workspace/"
155
+ num_gpus: 2
156
+ commands:
157
+ - pytest -v -s tests/distributed/test_context_parallel.py
158
+ - pytest -v -s tests/distributed/test_nccl_symm_mem_allreduce.py
159
+ - pytest -v -s tests/v1/distributed/test_dbo.py
160
+
161
+- label: 2 Node Test (4 GPUs)
162
+ timeout_in_minutes: 30
163
+ working_dir: "/vllm-workspace/tests"
164
+ num_gpus: 2
165
+ num_nodes: 2
166
+ source_file_dependencies:
167
+ - vllm/distributed/
168
+ - vllm/engine/
169
+ - vllm/executor/
170
+ - vllm/model_executor/models/
171
+ - tests/distributed/
172
+ - tests/examples/offline_inference/data_parallel.py
173
+ commands:
174
+ - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 public.ecr.aws/q9t5s3a7/vllm-ci-postmerge-repo:0bec63fa317e1fbd62e19b0fc31c43c81bf89077 "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py --dp-size=2 --tp-size=1 --node-size=2 --node-rank=0 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py --dp-size=2 --tp-size=1 --node-size=2 --node-rank=1 --master-addr=192.168.10.10 --master-port=12345 --enforce-eager --trust-remote-code"
175
+
176
+- label: Distributed NixlConnector PD accuracy (4 GPUs)
177
+ timeout_in_minutes: 30
178
+ working_dir: "/vllm-workspace/tests"
179
+ num_gpus: 4
180
+ source_file_dependencies:
181
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
182
+ - tests/v1/kv_connector/nixl_integration/
183
+ commands:
184
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
185
+ - bash v1/kv_connector/nixl_integration/tp_config_sweep_accuracy_test.sh
186
+
187
+- label: Pipeline + Context Parallelism (4 GPUs))
188
+ timeout_in_minutes: 60
189
+ working_dir: "/vllm-workspace/tests"
190
+ num_gpus: 4
191
+ source_file_dependencies:
192
+ - vllm/distributed/
193
+ - vllm/engine/
194
+ - vllm/executor/
195
+ - vllm/model_executor/models/
196
+ - tests/distributed/
197
+ commands:
198
+ - pytest -v -s distributed/test_pp_cudagraph.py
199
+ - pytest -v -s distributed/test_pipeline_parallel.py
\ No newline at end of file
.buildkite/test_areas/e2e_integration.yaml
new
+59
@@ -0,0 +1,59 @@
1
+group: E2E Integration
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: DeepSeek V2-Lite Accuracy
6
+ timeout_in_minutes: 60
7
+ gpu: h100
8
+ optional: true
9
+ num_gpus: 4
10
+ working_dir: "/vllm-workspace"
11
+ commands:
12
+ - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
13
+
14
+- label: Qwen3-30B-A3B-FP8-block Accuracy
15
+ timeout_in_minutes: 60
16
+ gpu: h100
17
+ optional: true
18
+ num_gpus: 4
19
+ working_dir: "/vllm-workspace"
20
+ commands:
21
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
22
+
23
+- label: Qwen3-30B-A3B-FP8-block Accuracy (B200)
24
+ timeout_in_minutes: 60
25
+ gpu: b200
26
+ optional: true
27
+ num_gpus: 2
28
+ working_dir: "/vllm-workspace"
29
+ commands:
30
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
31
+
32
+- label: Prime-RL Integration (2 GPUs)
33
+ timeout_in_minutes: 30
34
+ optional: true
35
+ num_gpus: 2
36
+ working_dir: "/vllm-workspace"
37
+ source_file_dependencies:
38
+ - vllm/
39
+ - .buildkite/scripts/run-prime-rl-test.sh
40
+ commands:
41
+ - bash .buildkite/scripts/run-prime-rl-test.sh
42
+
43
+- label: DeepSeek V2-Lite Async EPLB Accuracy
44
+ timeout_in_minutes: 60
45
+ gpu: h100
46
+ optional: true
47
+ num_gpus: 4
48
+ working_dir: "/vllm-workspace"
49
+ commands:
50
+ - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_async_eplb.sh 0.25 1319 8030
51
+
52
+- label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy
53
+ timeout_in_minutes: 60
54
+ gpu: h100
55
+ optional: true
56
+ num_gpus: 4
57
+ working_dir: "/vllm-workspace"
58
+ commands:
59
+ - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
.buildkite/test_areas/engine.yaml
new
+26
@@ -0,0 +1,26 @@
1
+group: Engine
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Engine
6
+ timeout_in_minutes: 15
7
+ source_file_dependencies:
8
+ - vllm/
9
+ - tests/engine
10
+ - tests/test_sequence
11
+ - tests/test_config
12
+ - tests/test_logger
13
+ - tests/test_vllm_port
14
+ commands:
15
+ - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
16
+
17
+- label: V1 e2e + engine
18
+ timeout_in_minutes: 45
19
+ source_file_dependencies:
20
+ - vllm/
21
+ - tests/v1
22
+ commands:
23
+ # TODO: accuracy does not match, whether setting
24
+ # VLLM_USE_FLASHINFER_SAMPLER or not on H100.
25
+ - pytest -v -s v1/e2e
26
+ - pytest -v -s v1/engine
.buildkite/test_areas/entrypoints.yaml
new
+68
@@ -0,0 +1,68 @@
1
+group: Entrypoints
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Entrypoints Unit Tests
6
+ timeout_in_minutes: 10
7
+ working_dir: "/vllm-workspace/tests"
8
+ source_file_dependencies:
9
+ - vllm/entrypoints
10
+ - tests/entrypoints/
11
+ commands:
12
+ - pytest -v -s entrypoints/openai/tool_parsers
13
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
14
+
15
+- label: Entrypoints Integration (LLM)
16
+ timeout_in_minutes: 40
17
+ working_dir: "/vllm-workspace/tests"
18
+ source_file_dependencies:
19
+ - vllm/
20
+ - tests/entrypoints/llm
21
+ - tests/entrypoints/offline_mode
22
+ commands:
23
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
24
+ - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
25
+ - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
26
+ - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
27
+
28
+- label: Entrypoints Integration (API Server)
29
+ timeout_in_minutes: 130
30
+ working_dir: "/vllm-workspace/tests"
31
+ source_file_dependencies:
32
+ - vllm/
33
+ - tests/entrypoints/openai
34
+ - tests/entrypoints/test_chat_utils
35
+ commands:
36
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
37
+ - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/openai/test_collective_rpc.py # PYTHONPATH is needed to import custom Worker extension
38
+ - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_collective_rpc.py --ignore=entrypoints/openai/tool_parsers/
39
+ - pytest -v -s entrypoints/test_chat_utils.py
40
+
41
+
42
+- label: Entrypoints Integration (Pooling)
43
+ timeout_in_minutes: 50
44
+ working_dir: "/vllm-workspace/tests"
45
+ source_file_dependencies:
46
+ - vllm/
47
+ - tests/entrypoints/pooling
48
+ commands:
49
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
50
+ - pytest -v -s entrypoints/pooling
51
+
52
+
53
+- label: Entrypoints V1
54
+ timeout_in_minutes: 50
55
+ source_file_dependencies:
56
+ - vllm/
57
+ - tests/v1
58
+ commands:
59
+ - pytest -v -s v1/entrypoints
60
+
61
+- label: OpenAI API Correctness
62
+ timeout_in_minutes: 30
63
+ source_file_dependencies:
64
+ - csrc/
65
+ - vllm/entrypoints/openai/
66
+ - vllm/model_executor/models/whisper.py
67
+ commands: # LMEval+Transcription WER check
68
+ - pytest -s entrypoints/openai/correctness/
.buildkite/test_areas/expert_parallelism.yaml
new
+23
@@ -0,0 +1,23 @@
1
+group: Expert Parallelism
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: EPLB Algorithm
6
+ timeout_in_minutes: 15
7
+ working_dir: "/vllm-workspace/tests"
8
+ source_file_dependencies:
9
+ - vllm/distributed/eplb
10
+ - tests/distributed/test_eplb_algo.py
11
+ commands:
12
+ - pytest -v -s distributed/test_eplb_algo.py
13
+
14
+- label: EPLB Execution
15
+ timeout_in_minutes: 20
16
+ working_dir: "/vllm-workspace/tests"
17
+ num_gpus: 4
18
+ source_file_dependencies:
19
+ - vllm/distributed/eplb
20
+ - tests/distributed/test_eplb_execute.py
21
+ commands:
22
+ - pytest -v -s distributed/test_eplb_execute.py
23
+ - pytest -v -s distributed/test_eplb_spec_decode.py
\ No newline at end of file
.buildkite/test_areas/kernels.yaml
new
+117
@@ -0,0 +1,117 @@
1
+group: Kernels
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Kernels Core Operation Test
6
+ timeout_in_minutes: 75
7
+ source_file_dependencies:
8
+ - csrc/
9
+ - tests/kernels/core
10
+ - tests/kernels/test_top_k_per_row.py
11
+ commands:
12
+ - pytest -v -s kernels/core kernels/test_top_k_per_row.py
13
+
14
+- label: Kernels Attention Test %N
15
+ timeout_in_minutes: 35
16
+ source_file_dependencies:
17
+ - csrc/attention/
18
+ - vllm/attention
19
+ - vllm/v1/attention
20
+ - tests/kernels/attention
21
+ commands:
22
+ - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
23
+ parallelism: 2
24
+
25
+- label: Kernels Quantization Test %N
26
+ timeout_in_minutes: 90
27
+ source_file_dependencies:
28
+ - csrc/quantization/
29
+ - vllm/model_executor/layers/quantization
30
+ - tests/kernels/quantization
31
+ commands:
32
+ - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
33
+ parallelism: 2
34
+
35
+- label: Kernels MoE Test %N
36
+ timeout_in_minutes: 60
37
+ source_file_dependencies:
38
+ - csrc/quantization/cutlass_w8a8/moe/
39
+ - csrc/moe/
40
+ - tests/kernels/moe
41
+ - vllm/model_executor/layers/fused_moe/
42
+ - vllm/distributed/device_communicators/
43
+ - vllm/envs.py
44
+ - vllm/config
45
+ commands:
46
+ - pytest -v -s kernels/moe --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
47
+ parallelism: 2
48
+
49
+- label: Kernels Mamba Test
50
+ timeout_in_minutes: 45
51
+ source_file_dependencies:
52
+ - csrc/mamba/
53
+ - tests/kernels/mamba
54
+ - vllm/model_executor/layers/mamba/ops
55
+ commands:
56
+ - pytest -v -s kernels/mamba
57
+
58
+- label: Kernels DeepGEMM Test (H100)
59
+ timeout_in_minutes: 45
60
+ gpu: h100
61
+ num_gpus: 1
62
+ source_file_dependencies:
63
+ - tools/install_deepgemm.sh
64
+ - vllm/utils/deep_gemm.py
65
+ - vllm/model_executor/layers/fused_moe
66
+ - vllm/model_executor/layers/quantization
67
+ - tests/kernels/quantization/test_block_fp8.py
68
+ - tests/kernels/moe/test_deepgemm.py
69
+ - tests/kernels/moe/test_batched_deepgemm.py
70
+ - tests/kernels/attention/test_deepgemm_attention.py
71
+ commands:
72
+ - pytest -v -s kernels/quantization/test_block_fp8.py -k deep_gemm
73
+ - pytest -v -s kernels/moe/test_deepgemm.py
74
+ - pytest -v -s kernels/moe/test_batched_deepgemm.py
75
+ - pytest -v -s kernels/attention/test_deepgemm_attention.py
76
+
77
+- label: Kernels (B200)
78
+ timeout_in_minutes: 30
79
+ working_dir: "/vllm-workspace/"
80
+ gpu: b200
81
+ # optional: true
82
+ source_file_dependencies:
83
+ - csrc/quantization/fp4/
84
+ - csrc/attention/mla/
85
+ - csrc/quantization/cutlass_w8a8/moe/
86
+ - vllm/model_executor/layers/fused_moe/cutlass_moe.py
87
+ - vllm/model_executor/layers/fused_moe/flashinfer_cutlass_moe.py
88
+ - vllm/model_executor/layers/fused_moe/flashinfer_cutlass_prepare_finalize.py
89
+ - vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
90
+ - vllm/v1/attention/backends/flashinfer.py
91
+ - vllm/v1/attention/backends/mla/cutlass_mla.py
92
+ - vllm/v1/attention/backends/mla/flashinfer_mla.py
93
+ - vllm/platforms/cuda.py
94
+ - vllm/attention/selector.py
95
+ commands:
96
+ - nvidia-smi
97
+ - python3 examples/offline_inference/basic/chat.py
98
+ # Attention
99
+ # num_heads2 broken by https://github.com/flashinfer-ai/flashinfer/issues/1353
100
+ - pytest -v -s tests/kernels/attention/test_attention_selector.py
101
+ - pytest -v -s tests/kernels/attention/test_flashinfer.py -k 'not num_heads2'
102
+ - pytest -v -s tests/kernels/attention/test_flashinfer_trtllm_attention.py
103
+ - pytest -v -s tests/kernels/attention/test_cutlass_mla_decode.py
104
+ - pytest -v -s tests/kernels/attention/test_flashinfer_mla_decode.py
105
+ # Quantization
106
+ - pytest -v -s tests/kernels/quantization/test_cutlass_scaled_mm.py -k 'fp8'
107
+ - pytest -v -s tests/kernels/quantization/test_nvfp4_quant.py
108
+ - pytest -v -s tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
109
+ - pytest -v -s tests/kernels/quantization/test_nvfp4_scaled_mm.py
110
+ - pytest -v -s tests/kernels/quantization/test_flashinfer_scaled_mm.py
111
+ - pytest -v -s tests/kernels/quantization/test_flashinfer_nvfp4_scaled_mm.py
112
+ - pytest -v -s tests/kernels/quantization/test_nvfp4_qutlass.py
113
+ - pytest -v -s tests/kernels/quantization/test_mxfp4_qutlass.py
114
+ - pytest -v -s tests/kernels/moe/test_nvfp4_moe.py
115
+ - pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py
116
+ - pytest -v -s tests/kernels/moe/test_flashinfer.py
117
+ - pytest -v -s tests/kernels/moe/test_cutedsl_moe.py
\ No newline at end of file
.buildkite/test_areas/lm_eval.yaml
new
+46
@@ -0,0 +1,46 @@
1
+group: LM Eval
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: LM Eval Small Models
6
+ timeout_in_minutes: 75
7
+ source_file_dependencies:
8
+ - csrc/
9
+ - vllm/model_executor/layers/quantization
10
+ autorun_on_main: true
11
+ commands:
12
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
13
+
14
+- label: LM Eval Large Models (4 GPUs)(A100)
15
+ gpu: a100
16
+ optional: true
17
+ num_gpus: 4
18
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
19
+ source_file_dependencies:
20
+ - csrc/
21
+ - vllm/model_executor/layers/quantization
22
+ commands:
23
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
24
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
25
+
26
+- label: LM Eval Large Models (4 GPUs)(H100)
27
+ gpu: h100
28
+ optional: true
29
+ num_gpus: 4
30
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
31
+ source_file_dependencies:
32
+ - csrc/
33
+ - vllm/model_executor/layers/quantization
34
+ commands:
35
+ - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
36
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
37
+
38
+- label: LM Eval Small Models (B200)
39
+ timeout_in_minutes: 120
40
+ gpu: b200
41
+ optional: true
42
+ source_file_dependencies:
43
+ - csrc/
44
+ - vllm/model_executor/layers/quantization
45
+ commands:
46
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt --tp-size=1
.buildkite/test_areas/lora.yaml
new
+31
@@ -0,0 +1,31 @@
1
+group: LoRA
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: LoRA %N
6
+ timeout_in_minutes: 30
7
+ source_file_dependencies:
8
+ - vllm/lora
9
+ - tests/lora
10
+ commands:
11
+ - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py
12
+ parallelism: 4
13
+
14
+
15
+- label: LoRA TP (Distributed)
16
+ timeout_in_minutes: 30
17
+ num_gpus: 4
18
+ source_file_dependencies:
19
+ - vllm/lora
20
+ - tests/lora
21
+ commands:
22
+ # FIXIT: find out which code initialize cuda before running the test
23
+ # before the fix, we need to use spawn to test it
24
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
25
+ # There is some Tensor Parallelism related processing logic in LoRA that
26
+ # requires multi-GPU testing for validation.
27
+ - pytest -v -s -x lora/test_chatglm3_tp.py
28
+ - pytest -v -s -x lora/test_llama_tp.py
29
+ - pytest -v -s -x lora/test_llm_with_multi_loras.py
30
+ - pytest -v -s -x lora/test_olmoe_tp.py
31
+ - pytest -v -s -x lora/test_gptoss_tp.py
\ No newline at end of file
.buildkite/test_areas/misc.yaml
new
+163
@@ -0,0 +1,163 @@
1
+group: Miscellaneous
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: V1 Others
6
+ timeout_in_minutes: 60
7
+ source_file_dependencies:
8
+ - vllm/
9
+ - tests/v1
10
+ commands:
11
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
12
+ # split the test to avoid interference
13
+ - pytest -v -s -m 'not cpu_test' v1/core
14
+ - pytest -v -s v1/executor
15
+ - pytest -v -s v1/kv_offload
16
+ - pytest -v -s v1/sample
17
+ - pytest -v -s v1/logits_processors
18
+ - pytest -v -s v1/worker
19
+ - pytest -v -s v1/spec_decode
20
+ - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
21
+ - pytest -v -s -m 'not cpu_test' v1/metrics
22
+ - pytest -v -s v1/test_oracle.py
23
+ - pytest -v -s v1/test_request.py
24
+ - pytest -v -s v1/test_outputs.py
25
+ # Integration test for streaming correctness (requires special branch).
26
+ - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
27
+ - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
28
+
29
+- label: V1 Others (CPU)
30
+ depends_on: ~
31
+ source_file_dependencies:
32
+ - vllm/
33
+ - tests/v1
34
+ no_gpu: true
35
+ commands:
36
+ # split the test to avoid interference
37
+ - pytest -v -s -m 'cpu_test' v1/core
38
+ - pytest -v -s v1/structured_output
39
+ - pytest -v -s v1/test_serial_utils.py
40
+ - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
41
+ - pytest -v -s -m 'cpu_test' v1/metrics
42
+
43
+- label: Regression
44
+ timeout_in_minutes: 20
45
+ source_file_dependencies:
46
+ - vllm/
47
+ - tests/test_regression
48
+ commands:
49
+ - pip install modelscope
50
+ - pytest -v -s test_regression.py
51
+ working_dir: "/vllm-workspace/tests" # optional
52
+
53
+- label: Examples
54
+ timeout_in_minutes: 45
55
+ working_dir: "/vllm-workspace/examples"
56
+ source_file_dependencies:
57
+ - vllm/entrypoints
58
+ - vllm/multimodal
59
+ - examples/
60
+ commands:
61
+ - pip install tensorizer # for tensorizer test
62
+ - python3 offline_inference/basic/chat.py # for basic
63
+ - python3 offline_inference/basic/generate.py --model facebook/opt-125m
64
+ - python3 offline_inference/basic/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
65
+ - python3 offline_inference/basic/classify.py
66
+ - python3 offline_inference/basic/embed.py
67
+ - python3 offline_inference/basic/score.py
68
+ # for multi-modal models
69
+ - python3 offline_inference/audio_language.py --seed 0
70
+ - python3 offline_inference/vision_language.py --seed 0
71
+ - python3 offline_inference/vision_language_multi_image.py --seed 0
72
+ - python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
73
+ # for pooling models
74
+ - python3 pooling/pooling/vision_language_pooling.py --seed 0
75
+ # for features demo
76
+ - python3 offline_inference/prefix_caching.py
77
+ - python3 offline_inference/llm_engine_example.py
78
+ - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
79
+ - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
80
+ # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
81
+ - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
82
+
83
+- label: Metrics, Tracing (2 GPUs)
84
+ timeout_in_minutes: 20
85
+ num_gpus: 2
86
+ source_file_dependencies:
87
+ - vllm/
88
+ - tests/v1/tracing
89
+ commands:
90
+ - "pip install \
91
+ 'opentelemetry-sdk>=1.26.0' \
92
+ 'opentelemetry-api>=1.26.0' \
93
+ 'opentelemetry-exporter-otlp>=1.26.0' \
94
+ 'opentelemetry-semantic-conventions-ai>=0.4.1'"
95
+ - pytest -v -s v1/tracing
96
+
97
+- label: Python-only Installation
98
+ depends_on: ~
99
+ timeout_in_minutes: 20
100
+ source_file_dependencies:
101
+ - tests/standalone_tests/python_only_compile.sh
102
+ - setup.py
103
+ commands:
104
+ - bash standalone_tests/python_only_compile.sh
105
+
106
+- label: Async Engine, Inputs, Utils, Worker
107
+ timeout_in_minutes: 50
108
+ source_file_dependencies:
109
+ - vllm/
110
+ - tests/multimodal
111
+ - tests/utils_
112
+ commands:
113
+ - pytest -v -s -m 'not cpu_test' multimodal
114
+ - pytest -v -s utils_
115
+
116
+- label: Async Engine, Inputs, Utils, Worker, Config (CPU)
117
+ depends_on: ~
118
+ timeout_in_minutes: 20
119
+ source_file_dependencies:
120
+ - vllm/
121
+ - tests/test_inputs.py
122
+ - tests/test_outputs.py
123
+ - tests/multimodal
124
+ - tests/standalone_tests/lazy_imports.py
125
+ - tests/tokenizers_
126
+ - tests/transformers_utils
127
+ - tests/config
128
+ no_gpu: true
129
+ commands:
130
+ - python3 standalone_tests/lazy_imports.py
131
+ - pytest -v -s test_inputs.py
132
+ - pytest -v -s test_outputs.py
133
+ - pytest -v -s -m 'cpu_test' multimodal
134
+ - pytest -v -s tokenizers_
135
+ - pytest -v -s transformers_utils
136
+ - pytest -v -s config
137
+
138
+- label: GPT-OSS Eval (B200)
139
+ timeout_in_minutes: 60
140
+ working_dir: "/vllm-workspace/"
141
+ gpu: b200
142
+ optional: true
143
+ source_file_dependencies:
144
+ - tests/evals/gpt_oss
145
+ - vllm/model_executor/models/gpt_oss.py
146
+ - vllm/model_executor/layers/quantization/mxfp4.py
147
+ - vllm/v1/attention/backends/flashinfer.py
148
+ commands:
149
+ - uv pip install --system 'gpt-oss[eval]==0.0.5'
150
+ - pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58
151
+
152
+- label: Batch Invariance (H100)
153
+ timeout_in_minutes: 25
154
+ gpu: h100
155
+ source_file_dependencies:
156
+ - vllm/v1/attention
157
+ - vllm/model_executor/layers
158
+ - tests/v1/determinism/
159
+ commands:
160
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
161
+ - pip install pytest-timeout pytest-forked
162
+ - pytest -v -s v1/determinism/test_batch_invariance.py
163
+ - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
\ No newline at end of file
.buildkite/test_areas/model_executor.yaml
new
+17
@@ -0,0 +1,17 @@
1
+group: Model Executor
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Model Executor
6
+ timeout_in_minutes: 35
7
+ source_file_dependencies:
8
+ - vllm/engine/arg_utils.py
9
+ - vllm/config/model.py
10
+ - vllm/model_executor
11
+ - tests/model_executor
12
+ - tests/entrypoints/openai/test_tensorizer_entrypoint.py
13
+ commands:
14
+ - apt-get update && apt-get install -y curl libsodium23
15
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
16
+ - pytest -v -s model_executor
17
+ - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py
.buildkite/test_areas/models_basic.yaml
new
+62
@@ -0,0 +1,62 @@
1
+group: Models - Basic
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Basic Models Tests (Initialization)
6
+ timeout_in_minutes: 45
7
+ mirror_hardwares: [amdexperimental]
8
+ torch_nightly: true
9
+ source_file_dependencies:
10
+ - vllm/
11
+ - tests/models/test_initialization.py
12
+ commands:
13
+ # Run a subset of model initialization tests
14
+ - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
15
+
16
+- label: Basic Models Tests (Extra Initialization) %N
17
+ timeout_in_minutes: 45
18
+ mirror_hardwares: [amdexperimental]
19
+ torch_nightly: true
20
+ source_file_dependencies:
21
+ - vllm/model_executor/models/
22
+ - tests/models/test_initialization.py
23
+ commands:
24
+ # Only when vLLM model source is modified - test initialization of a large
25
+ # subset of supported models (the complement of the small subset in the above
26
+ # test.) Also run if model initialization test file is modified
27
+ - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
28
+ parallelism: 2
29
+
30
+- label: Basic Models Tests (Other)
31
+ timeout_in_minutes: 45
32
+ source_file_dependencies:
33
+ - vllm/
34
+ - tests/models/test_transformers.py
35
+ - tests/models/test_registry.py
36
+ commands:
37
+ - pytest -v -s models/test_transformers.py models/test_registry.py
38
+
39
+- label: Basic Models Test (Other CPU) # 5min
40
+ timeout_in_minutes: 10
41
+ source_file_dependencies:
42
+ - vllm/
43
+ - tests/models/test_utils.py
44
+ - tests/models/test_vision.py
45
+ no_gpu: true
46
+ commands:
47
+ - pytest -v -s models/test_utils.py models/test_vision.py
48
+
49
+- label: Transformers Nightly Models
50
+ working_dir: "/vllm-workspace/"
51
+ optional: true
52
+ soft_fail: true
53
+ commands:
54
+ - pip install --upgrade git+https://github.com/huggingface/transformers
55
+ - pytest -v -s tests/models/test_initialization.py
56
+ - pytest -v -s tests/models/test_transformers.py
57
+ - pytest -v -s tests/models/multimodal/processing/
58
+ - pytest -v -s tests/models/multimodal/test_mapping.py
59
+ - python3 examples/offline_inference/basic/chat.py
60
+ - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
61
+ # Whisper needs spawn method to avoid deadlock
62
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
.buildkite/test_areas/models_distributed.yaml
new
+22
@@ -0,0 +1,22 @@
1
+group: Models - Distributed
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Distributed Model Tests (2 GPUs)
6
+ timeout_in_minutes: 50
7
+ working_dir: "/vllm-workspace/tests"
8
+ num_gpus: 2
9
+ source_file_dependencies:
10
+ - vllm/model_executor/model_loader/sharded_state_loader.py
11
+ - vllm/model_executor/models/
12
+ - tests/basic_correctness/
13
+ - tests/model_executor/model_loader/test_sharded_state_loader.py
14
+ - tests/models/
15
+ commands:
16
+ - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
17
+ - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py
18
+ # Avoid importing model tests that cause CUDA reinitialization error
19
+ - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
20
+ - pytest models/language -v -s -m 'distributed(num_gpus=2)'
21
+ - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py
22
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
.buildkite/test_areas/models_language.yaml
new
+91
@@ -0,0 +1,91 @@
1
+group: Models - Language
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Language Models Tests (Standard)
6
+ timeout_in_minutes: 25
7
+ mirror_hardwares: [amdexperimental]
8
+ torch_nightly: true
9
+ source_file_dependencies:
10
+ - vllm/
11
+ - tests/models/language
12
+ commands:
13
+ # Test standard language models, excluding a subset of slow tests
14
+ - pip freeze | grep -E 'torch'
15
+ - pytest -v -s models/language -m 'core_model and (not slow_test)'
16
+
17
+- label: Language Models Tests (Extra Standard) %N
18
+ timeout_in_minutes: 45
19
+ mirror_hardwares: [amdexperimental]
20
+ torch_nightly: true
21
+ source_file_dependencies:
22
+ - vllm/model_executor/models/
23
+ - tests/models/language/pooling/test_embedding.py
24
+ - tests/models/language/generation/test_common.py
25
+ - tests/models/language/pooling/test_classification.py
26
+ commands:
27
+ # Shard slow subset of standard language models tests. Only run when model
28
+ # source is modified, or when specified test files are modified
29
+ - pip freeze | grep -E 'torch'
30
+ - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
31
+ parallelism: 2
32
+
33
+- label: Language Models Tests (Hybrid) %N
34
+ timeout_in_minutes: 75
35
+ mirror_hardwares: [amdexperimental]
36
+ torch_nightly: true
37
+ source_file_dependencies:
38
+ - vllm/
39
+ - tests/models/language/generation
40
+ commands:
41
+ # Install fast path packages for testing against transformers
42
+ # Note: also needed to run plamo2 model in vLLM
43
+ - uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.2.5'
44
+ - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
45
+ # Shard hybrid language model tests
46
+ - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
47
+ parallelism: 2
48
+
49
+- label: Language Models Test (Extended Generation) # 80min
50
+ timeout_in_minutes: 110
51
+ mirror_hardwares: [amdexperimental]
52
+ optional: true
53
+ source_file_dependencies:
54
+ - vllm/
55
+ - tests/models/language/generation
56
+ commands:
57
+ # Install fast path packages for testing against transformers
58
+ # Note: also needed to run plamo2 model in vLLM
59
+ - uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.2.5'
60
+ - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
61
+ - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
62
+
63
+- label: Language Models Test (PPL)
64
+ timeout_in_minutes: 110
65
+ mirror_hardwares: [amdexperimental]
66
+ optional: true
67
+ source_file_dependencies:
68
+ - vllm/
69
+ - tests/models/language/generation_ppl_test
70
+ commands:
71
+ - pytest -v -s models/language/generation_ppl_test
72
+
73
+- label: Language Models Test (Extended Pooling) # 36min
74
+ timeout_in_minutes: 50
75
+ mirror_hardwares: [amdexperimental]
76
+ optional: true
77
+ source_file_dependencies:
78
+ - vllm/
79
+ - tests/models/language/pooling
80
+ commands:
81
+ - pytest -v -s models/language/pooling -m 'not core_model'
82
+
83
+- label: Language Models Test (MTEB)
84
+ timeout_in_minutes: 110
85
+ mirror_hardwares: [amdexperimental]
86
+ optional: true
87
+ source_file_dependencies:
88
+ - vllm/
89
+ - tests/models/language/pooling_mteb_test
90
+ commands:
91
+ - pytest -v -s models/language/pooling_mteb_test
.buildkite/test_areas/models_multimodal.yaml
new
+79
@@ -0,0 +1,79 @@
1
+group: Models - Multimodal
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Multi-Modal Models (Standard) # 60min
6
+ timeout_in_minutes: 80
7
+ source_file_dependencies:
8
+ - vllm/
9
+ - tests/models/multimodal
10
+ commands:
11
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
12
+ - pip freeze | grep -E 'torch'
13
+ - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing
14
+ - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
15
+
16
+- label: Multi-Modal Processor Test (CPU)
17
+ timeout_in_minutes: 60
18
+ source_file_dependencies:
19
+ - vllm/
20
+ - tests/models/multimodal
21
+ no_gpu: true
22
+ commands:
23
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
24
+ - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
25
+
26
+- label: Multi-Modal Processor # 44min
27
+ timeout_in_minutes: 60
28
+ source_file_dependencies:
29
+ - vllm/
30
+ - tests/models/multimodal
31
+ commands:
32
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
33
+ - pytest -v -s models/multimodal/processing/test_tensor_schema.py
34
+
35
+- label: Multi-Modal Accuracy Eval (Small Models) # 50min
36
+ timeout_in_minutes: 70
37
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
38
+ source_file_dependencies:
39
+ - vllm/multimodal/
40
+ - vllm/inputs/
41
+ - vllm/v1/core/
42
+ commands:
43
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
44
+
45
+- label: Multi-Modal Models (Extended) 1
46
+ optional: true
47
+ source_file_dependencies:
48
+ - vllm/
49
+ - tests/models/multimodal
50
+ commands:
51
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
52
+ - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing
53
+
54
+- label: Multi-Modal Models (Extended) 2
55
+ optional: true
56
+ source_file_dependencies:
57
+ - vllm/
58
+ - tests/models/multimodal
59
+ commands:
60
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
61
+ - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
62
+
63
+- label: Multi-Modal Models (Extended) 3
64
+ optional: true
65
+ source_file_dependencies:
66
+ - vllm/
67
+ - tests/models/multimodal
68
+ commands:
69
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
70
+ - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
71
+
72
+# This test is used only in PR development phase to test individual models and should never run on main
73
+- label: Custom Models
74
+ optional: true
75
+ commands:
76
+ - echo 'Testing custom models...'
77
+ # PR authors can temporarily add commands below to test individual models
78
+ # e.g. pytest -v -s models/encoder_decoder/vision_language/test_mllama.py
79
+ # *To avoid merge conflicts, remember to REMOVE (not just comment out) them before merging the PR*
.buildkite/test_areas/plugins.yaml
new
+34
@@ -0,0 +1,34 @@
1
+group: Plugins
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Plugin Tests (2 GPUs)
6
+ timeout_in_minutes: 60
7
+ working_dir: "/vllm-workspace/tests"
8
+ num_gpus: 2
9
+ source_file_dependencies:
10
+ - vllm/plugins/
11
+ - tests/plugins/
12
+ commands:
13
+ # begin platform plugin and general plugin tests, all the code in-between runs on dummy platform
14
+ - pip install -e ./plugins/vllm_add_dummy_platform
15
+ - pytest -v -s plugins_tests/test_platform_plugins.py
16
+ - pip uninstall vllm_add_dummy_platform -y
17
+ # end platform plugin tests
18
+ # begin io_processor plugins test, all the code in between uses the prithvi_io_processor plugin
19
+ - pip install -e ./plugins/prithvi_io_processor_plugin
20
+ - pytest -v -s plugins_tests/test_io_processor_plugins.py
21
+ - pip uninstall prithvi_io_processor_plugin -y
22
+ # end io_processor plugins test
23
+ # begin stat_logger plugins test
24
+ - pip install -e ./plugins/vllm_add_dummy_stat_logger
25
+ - pytest -v -s plugins_tests/test_stats_logger_plugins.py
26
+ - pip uninstall dummy_stat_logger -y
27
+ # end stat_logger plugins test
28
+ # other tests continue here:
29
+ - pytest -v -s plugins_tests/test_scheduler_plugins.py
30
+ - pip install -e ./plugins/vllm_add_dummy_model
31
+ - pytest -v -s distributed/test_distributed_oot.py
32
+ - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
33
+ - pytest -v -s models/test_oot_registration.py # it needs a clean process
34
+ - pytest -v -s plugins/lora_resolvers # unit tests for in-tree lora resolver plugins
.buildkite/test_areas/pytorch.yaml
new
+50
@@ -0,0 +1,50 @@
1
+group: PyTorch
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: PyTorch Compilation Unit Tests
6
+ timeout_in_minutes: 30
7
+ source_file_dependencies:
8
+ - vllm/
9
+ - tests/compile
10
+ commands:
11
+ # Run unit tests defined directly under compile/,
12
+ # not including subdirectories, which are usually heavier
13
+ # tests covered elsewhere.
14
+ # Use `find` to launch multiple instances of pytest so that
15
+ # they do not suffer from https://github.com/vllm-project/vllm/issues/28965
16
+ - "find compile/ -maxdepth 1 -name 'test_*.py' -exec pytest -s -v {} \\;"
17
+
18
+- label: PyTorch Fullgraph Smoke Test
19
+ timeout_in_minutes: 30
20
+ source_file_dependencies:
21
+ - vllm/
22
+ - tests/compile
23
+ commands:
24
+ # Run smoke tests under fullgraph directory, except test_full_graph.py
25
+ # as it is a heavy test that is covered in other steps.
26
+ # Use `find` to launch multiple instances of pytest so that
27
+ # they do not suffer from https://github.com/vllm-project/vllm/issues/28965
28
+ - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -exec pytest -s -v {} \\;"
29
+
30
+- label: PyTorch Fullgraph
31
+ timeout_in_minutes: 40
32
+ source_file_dependencies:
33
+ - vllm/
34
+ - tests/compile
35
+ commands:
36
+ # fp8 kv scales not supported on sm89, tested on Blackwell instead
37
+ - pytest -v -s compile/fullgraph/test_full_graph.py -k 'not test_fp8_kv_scale_compile'
38
+ # Limit to no custom ops to reduce running time
39
+ # Wrap with quotes to escape yaml and avoid starting -k string with a -
40
+ - "pytest -v -s compile/distributed/test_fusions_e2e.py -k 'TRITON and not +quant_fp8 and not Llama-4'"
41
+
42
+- label: Pytorch Nightly Dependency Override Check # 2min
43
+ # if this test fails, it means the nightly torch version is not compatible with some
44
+ # of the dependencies. Please check the error message and add the package to whitelist
45
+ # in /vllm/tools/pre_commit/generate_nightly_torch_test.py
46
+ soft_fail: true
47
+ source_file_dependencies:
48
+ - requirements/nightly_torch_test.txt
49
+ commands:
50
+ - bash standalone_tests/pytorch_nightly_dependency.sh
\ No newline at end of file
.buildkite/test_areas/quantization.yaml
new
+46
@@ -0,0 +1,46 @@
1
+group: Quantization
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Quantization
6
+ timeout_in_minutes: 90
7
+ source_file_dependencies:
8
+ - csrc/
9
+ - vllm/model_executor/layers/quantization
10
+ - tests/quantization
11
+ commands:
12
+ # temporary install here since we need nightly, will move to requirements/test.in
13
+ # after torchao 0.12 release, and pin a working version of torchao nightly here
14
+
15
+ # since torchao nightly is only compatible with torch nightly currently
16
+ # https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now
17
+ # we can only upgrade after this is resolved
18
+ # TODO(jerryzh168): resolve the above comment
19
+ - uv pip install --system torchao==0.13.0 --index-url https://download.pytorch.org/whl/cu129
20
+ - uv pip install --system conch-triton-kernels
21
+ - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
22
+
23
+- label: Quantized MoE Test (B200)
24
+ timeout_in_minutes: 60
25
+ working_dir: "/vllm-workspace/"
26
+ gpu: b200
27
+ source_file_dependencies:
28
+ - tests/quantization/test_blackwell_moe.py
29
+ - vllm/model_executor/models/deepseek_v2.py
30
+ - vllm/model_executor/models/gpt_oss.py
31
+ - vllm/model_executor/models/llama4.py
32
+ - vllm/model_executor/layers/fused_moe
33
+ - vllm/model_executor/layers/quantization/compressed_tensors
34
+ - vllm/model_executor/layers/quantization/modelopt.py
35
+ - vllm/model_executor/layers/quantization/mxfp4.py
36
+ - vllm/v1/attention/backends/flashinfer.py
37
+ commands:
38
+ - pytest -s -v tests/quantization/test_blackwell_moe.py
39
+
40
+- label: Quantized Models Test
41
+ timeout_in_minutes: 60
42
+ source_file_dependencies:
43
+ - vllm/model_executor/layers/quantization
44
+ - tests/models/quantization
45
+ commands:
46
+ - pytest -v -s models/quantization
.buildkite/test_areas/samplers.yaml
new
+14
@@ -0,0 +1,14 @@
1
+group: Samplers
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Samplers Test
6
+ timeout_in_minutes: 75
7
+ source_file_dependencies:
8
+ - vllm/model_executor/layers
9
+ - vllm/sampling_metadata.py
10
+ - tests/samplers
11
+ - tests/conftest.py
12
+ commands:
13
+ - pytest -v -s samplers
14
+ - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
.buildkite/test_areas/tool_use.yaml
new
+23
@@ -0,0 +1,23 @@
1
+group: Tool use
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: OpenAI-Compatible Tool Use
6
+ timeout_in_minutes: 35
7
+ mirror_hardwares: [amdexperimental]
8
+ fast_check: false
9
+ source_file_dependencies:
10
+ - vllm/
11
+ - tests/tool_use
12
+ commands:
13
+ - pytest -v -s -m 'not cpu_test' tool_use
14
+
15
+- label: OpenAI-Compatible Tool Use (CPU)
16
+ depends_on: ~
17
+ timeout_in_minutes: 10
18
+ source_file_dependencies:
19
+ - vllm/
20
+ - tests/tool_use
21
+ no_gpu: true
22
+ commands:
23
+ - pytest -v -s -m 'cpu_test' tool_use
.buildkite/test_areas/weight_loading.yaml
new
+25
@@ -0,0 +1,25 @@
1
+group: Weight Loading
2
+depends_on:
3
+ - image-build
4
+steps:
5
+- label: Weight Loading Multiple GPU # 33min
6
+ timeout_in_minutes: 45
7
+ working_dir: "/vllm-workspace/tests"
8
+ num_gpus: 2
9
+ optional: true
10
+ source_file_dependencies:
11
+ - vllm/
12
+ - tests/weight_loading
13
+ commands:
14
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
15
+
16
+- label: Weight Loading Multiple GPU - Large Models # optional
17
+ working_dir: "/vllm-workspace/tests"
18
+ num_gpus: 2
19
+ gpu: a100
20
+ optional: true
21
+ source_file_dependencies:
22
+ - vllm/
23
+ - tests/weight_loading
24
+ commands:
25
+ - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt