[CI] Shard LoRA TP distributed tests (#52350)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Kevin H. Luu committed
Sep 2, 2026 at 01:16 UTC
f5711fa138d06cfff6be8ccb8a4d98eafa8a3b50
1 file changed
+24
-18
.buildkite/test_areas/lora.yaml
+24
-18
@@ -27,11 +27,12 @@ steps:
27
- image-build-amd
28
29
30
-- label: ":nvidia: (L4) LoRA TP (Distributed)"
30
+- label: ":nvidia: (L4) LoRA TP (Distributed) %N"
31
key: lora-tp-distributed
32
- timeout_in_minutes: 60
32
+ timeout_in_minutes: 25
33
device: l4
34
num_devices: 4
35
+ parallelism: 4
36
source_file_dependencies:
37
- vllm/lora
38
- vllm/model_executor/layers/fused_moe/
@@ -44,20 +45,22 @@ steps:
45
- export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
46
# There is some Tensor Parallelism related processing logic in LoRA that
47
# requires multi-GPU testing for validation.
47
- - pytest -v -s -x lora/test_chatglm3_tp.py
48
- - pytest -v -s -x lora/test_llama_tp.py
49
- - pytest -v -s -x lora/test_qwen3_with_multi_loras.py
50
- - pytest -v -s -x lora/test_olmoe_tp.py
51
- - pytest -v -s -x lora/test_gptoss_tp.py
52
- - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
53
- - pytest -v -s -x lora/test_gemma4_tp.py
48
+ # Timing-balanced static split from per-file walls in builds 83921/83936.
49
+ - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
50
+ - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py
51
+ - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py
52
+ - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py
53
+ - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py
54
+ - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py
55
+ - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py
56
+ - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py
57
mirror:
58
amd:
56
- label: ":amd: (MI300) LoRA TP (Distributed)"
59
+ label: ":amd: (MI300) LoRA TP (Distributed) %N"
60
dind: false
61
device: mi300_4
62
working_dir: "/vllm-workspace/tests"
60
- timeout_in_minutes: 55
63
+ timeout_in_minutes: 25
64
depends_on:
65
- image-build-amd
66
source_file_dependencies:
@@ -66,10 +69,13 @@ steps:
69
- vllm/platforms/rocm.py
70
- tests/lora
71
commands:
69
- - pytest -v -s -x lora/test_chatglm3_tp.py
70
- - pytest -v -s -x lora/test_llama_tp.py
71
- - pytest -v -s -x lora/test_qwen3_with_multi_loras.py
72
- - pytest -v -s -x lora/test_olmoe_tp.py
73
- - pytest -v -s -x lora/test_gptoss_tp.py
74
- - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
75
- - pytest -v -s -x lora/test_gemma4_tp.py
72
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
73
+ - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
74
+ - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
75
+ - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py
76
+ - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py
77
+ - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py
78
+ - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py
79
+ - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py
80
+ - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py
81
+ - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py