A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 06:15 UTC
[ROCm][CI] Refine gating tests (#37243)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
May 6, 2026 at 00:05 UTC
91740ca5eaedbcdd960c3629206087d20fb43a00
9 files changed
+40
-65
.buildkite/scripts/hardware_ci/run-amd-test.sh
+11
-3
@@ -378,9 +378,11 @@ HF_MOUNT="/root/.cache/huggingface"
378
# double-quotes will have been stripped by the calling shell.
379
if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
380
commands="${VLLM_TEST_COMMANDS}"
381
+ commands_source="env"
382
echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
383
else
384
commands="$*"
385
+ commands_source="argv"
386
if [[ -z "$commands" ]]; then
387
echo "Error: No test commands provided." >&2
388
echo "Usage:" >&2
@@ -397,9 +399,15 @@ fi
399
400
echo "Raw commands: $commands"
401
400
-# Fix quoting before ROCm overrides (so overrides see correct structure)
401
-commands=$(re_quote_pytest_markers "$commands")
402
-echo "After re-quoting: $commands"
402
+# Only try to repair stripped pytest -m/-k quoting in legacy argv mode.
403
+# VLLM_TEST_COMMANDS preserves inner quoting already, and re-quoting that path
404
+# can corrupt embedded echo strings or otherwise well-formed shell fragments.
405
+if [[ "$commands_source" == "argv" ]]; then
406
+ commands=$(re_quote_pytest_markers "$commands")
407
+ echo "After re-quoting: $commands"
408
+else
409
+ echo "Skipping re-quoting for VLLM_TEST_COMMANDS input"
410
+fi
411
412
commands=$(apply_rocm_test_overrides "$commands")
413
echo "Final commands: $commands"
.buildkite/test-amd.yaml
+10
-1
@@ -621,6 +621,7 @@ steps:
621
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
622
agent_pool: mi250_1
623
torch_nightly: true
624
+ optional: true
625
working_dir: "/vllm-workspace/tests"
626
source_file_dependencies:
627
- vllm/
@@ -1803,6 +1804,7 @@ steps:
1804
timeout_in_minutes: 180
1805
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1806
agent_pool: mi300_1
1807
+ optional: true
1808
working_dir: "/vllm-workspace/tests"
1809
source_file_dependencies:
1810
- vllm/
@@ -1844,6 +1846,7 @@ steps:
1846
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1847
agent_pool: mi300_1
1848
torch_nightly: true
1849
+ optional: true
1850
working_dir: "/vllm-workspace/tests"
1851
source_file_dependencies:
1852
- vllm/
@@ -2204,7 +2207,6 @@ steps:
2207
timeout_in_minutes: 180
2208
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
2209
agent_pool: mi300_1
2207
- optional: true
2210
working_dir: "/vllm-workspace/tests"
2211
source_file_dependencies:
2212
- vllm/
@@ -2628,6 +2630,7 @@ steps:
2630
agent_pool: mi325_1
2631
torch_nightly: true
2632
parallelism: 2
2633
+ optional: true
2634
working_dir: "/vllm-workspace/tests"
2635
source_file_dependencies:
2636
- vllm/
@@ -2653,6 +2656,7 @@ steps:
2656
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
2657
agent_pool: mi325_1
2658
torch_nightly: true
2659
+ optional: true
2660
working_dir: "/vllm-workspace/tests"
2661
source_file_dependencies:
2662
- vllm/
@@ -2749,6 +2753,7 @@ steps:
2753
agent_pool: mi355_1
2754
fast_check: true
2755
torch_nightly: true
2756
+ optional: true
2757
working_dir: "/vllm-workspace/tests"
2758
source_file_dependencies:
2759
- vllm/
@@ -2764,6 +2769,7 @@ steps:
2769
agent_pool: mi355_1
2770
fast_check: true
2771
torch_nightly: true
2772
+ optional: true
2773
working_dir: "/vllm-workspace/tests"
2774
source_file_dependencies:
2775
- vllm/
@@ -3060,6 +3066,7 @@ steps:
3066
timeout_in_minutes: 180
3067
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3068
agent_pool: mi355_1
3069
+ optional: true
3070
working_dir: "/vllm-workspace/tests"
3071
source_file_dependencies:
3072
- vllm/
@@ -3259,6 +3266,7 @@ steps:
3266
timeout_in_minutes: 60
3267
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3268
agent_pool: mi355_1
3269
+ optional: true
3270
working_dir: "/vllm-workspace/tests"
3271
source_file_dependencies:
3272
- vllm/
@@ -3284,6 +3292,7 @@ steps:
3292
timeout_in_minutes: 60
3293
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
3294
agent_pool: mi355_1
3295
+ optional: true
3296
working_dir: "/vllm-workspace/tests"
3297
source_file_dependencies:
3298
- vllm/
.buildkite/test_areas/engine.yaml
-10
@@ -57,11 +57,6 @@ steps:
57
commands:
58
# Only run tests that need exactly 2 GPUs
59
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
60
- mirror:
61
- amd:
62
- device: mi325_2
63
- depends_on:
64
- - image-build-amd
60
61
- label: V1 e2e (4 GPUs)
62
key: v1-e2e-4-gpus
@@ -74,11 +69,6 @@ steps:
69
commands:
70
# Only run tests that need 4 GPUs
71
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle_correctness_heavy"
77
- mirror:
78
- amd:
79
- device: mi325_4
80
- depends_on:
81
- - image-build-amd
72
73
- label: V1 e2e (4xH100)
74
key: v1-e2e-4xh100
.buildkite/test_areas/entrypoints.yaml
+5
-10
@@ -26,6 +26,11 @@ steps:
26
- pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
27
- pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
28
- pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
29
+ mirror:
30
+ amd:
31
+ device: mi300_1
32
+ depends_on:
33
+ - image-build-amd
34
35
- label: Entrypoints Integration (API Server openai - Part 1)
36
key: entrypoints-integration-api-server-openai-part-1
@@ -38,11 +43,6 @@ steps:
43
commands:
44
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
45
- pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
41
- mirror:
42
- amd:
43
- device: mi325_1
44
- depends_on:
45
- - image-build-amd
46
47
48
- label: Entrypoints Integration (API Server openai - Part 2)
@@ -57,11 +57,6 @@ steps:
57
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
58
- pytest -v -s entrypoints/openai/speech_to_text/
59
- pytest -v -s entrypoints/test_chat_utils.py
60
- mirror:
61
- amd:
62
- device: mi325_1
63
- depends_on:
64
- - image-build-amd
60
61
- label: Entrypoints Integration (API Server openai - Part 3)
62
key: entrypoints-integration-api-server-openai-part-3
.buildkite/test_areas/misc.yaml
+1
-11
@@ -12,11 +12,6 @@ steps:
12
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
13
# TODO: create another `optional` test group for slow tests
14
- pytest -v -s -m 'not slow_test' v1/spec_decode
15
- mirror:
16
- amd:
17
- device: mi325_1
18
- depends_on:
19
- - image-build-amd
15
16
- label: V1 Sample + Logits
17
key: v1-sample-logits
@@ -38,7 +33,7 @@ steps:
33
- pytest -v -s v1/test_outputs.py
34
mirror:
35
amd:
41
- device: mi325_1
36
+ device: mi300_1
37
depends_on:
38
- image-build-amd
39
@@ -67,11 +62,6 @@ steps:
62
# Integration test for streaming correctness (requires special branch).
63
- pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
64
- pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
70
- mirror:
71
- amd:
72
- device: mi325_1
73
- depends_on:
74
- - image-build-amd
65
66
- label: V1 Others (CPU)
67
key: v1-others-cpu
.buildkite/test_areas/models_basic.yaml
-6
@@ -42,12 +42,6 @@ steps:
42
- tests/models/test_registry.py
43
commands:
44
- pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
45
- mirror:
46
- amd:
47
- device: mi325_1
48
- depends_on:
49
- - image-build-amd
50
-
45
46
- label: Basic Models Test (Other CPU) # 5min
47
key: basic-models-test-other-cpu
.buildkite/test_areas/models_language.yaml
+8
-14
@@ -48,6 +48,14 @@ steps:
48
parallelism: 2
49
mirror:
50
torch_nightly: {}
51
+ amd:
52
+ device: mi300_1
53
+ depends_on:
54
+ - image-build-amd
55
+ commands:
56
+ - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
57
+ - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
58
+ - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
59
60
- label: Language Models Test (Extended Generation) # 80min
61
key: language-models-test-extended-generation
@@ -62,15 +70,6 @@ steps:
70
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
71
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
72
- pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
65
- mirror:
66
- amd:
67
- device: mi325_1
68
- depends_on:
69
- - image-build-amd
70
- commands:
71
- - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
72
- - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
73
- - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
73
74
- label: Language Models Test (PPL)
75
key: language-models-test-ppl
@@ -92,11 +91,6 @@ steps:
91
- tests/models/language/pooling
92
commands:
93
- pytest -v -s models/language/pooling -m 'not core_model'
95
- mirror:
96
- amd:
97
- device: mi325_1
98
- depends_on:
99
- - image-build-amd
94
95
- label: Language Models Test (MTEB)
96
key: language-models-test-mteb
.buildkite/test_areas/models_multimodal.yaml
+4
-9
@@ -15,7 +15,7 @@ steps:
15
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
16
mirror:
17
amd:
18
- device: mi325_1
18
+ device: mi300_1
19
depends_on:
20
- image-build-amd
21
@@ -33,7 +33,7 @@ steps:
33
- pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
34
mirror:
35
amd:
36
- device: mi325_1
36
+ device: mi300_1
37
depends_on:
38
- image-build-amd
39
@@ -49,7 +49,7 @@ steps:
49
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
50
mirror:
51
amd:
52
- device: mi325_1
52
+ device: mi300_1
53
depends_on:
54
- image-build-amd
55
@@ -64,11 +64,6 @@ steps:
64
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
65
- pytest models/multimodal/generation/test_memory_leak.py -m core_model
66
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
67
- mirror:
68
- amd:
69
- device: mi325_1
70
- depends_on:
71
- - image-build-amd
67
68
- label: Multi-Modal Processor (CPU)
69
key: multi-modal-processor-cpu
@@ -120,7 +115,7 @@ steps:
115
- pytest -v -s models/multimodal/test_mapping.py
116
mirror:
117
amd:
123
- device: mi325_1
118
+ device: mi300_1
119
depends_on:
120
- image-build-amd
121
.buildkite/test_areas/samplers.yaml
+1
-1
@@ -17,7 +17,7 @@ steps:
17
- VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
18
mirror:
19
amd:
20
- device: mi325_1
20
+ device: mi250_1
21
depends_on:
22
- image-build-amd
23
commands: