@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
yaml 83 lines 4 KB
Raw
1 group: LoRA
2 depends_on:
3 - image-build
4 steps:
5 - label: ":nvidia: (H200 MIG 35GB) LoRA Shard %N"
6 device: h200_35gb
7 key: lora
8 timeout_in_minutes: 40
9 source_file_dependencies:
10 - vllm/lora
11 - tests/lora
12 commands:
13 - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py
14 parallelism: 4
15 mirror:
16 amd:
17 label: ":amd: (MI355 DPX) LoRA Shard %N"
18 dind: false
19 device: mi355_dpx
20 working_dir: "/vllm-workspace/tests"
21 timeout_in_minutes: 85
22 source_file_dependencies:
23 - vllm/lora
24 - tests/lora
25 - vllm/platforms/rocm.py
26 depends_on:
27 - image-build-amd
28
29
30 - label: ":nvidia: (L4) LoRA TP (Distributed) %N"
31 key: lora-tp-distributed
32 timeout_in_minutes: 25
33 device: l4
34 num_devices: 4
35 parallelism: 4
36 source_file_dependencies:
37 - vllm/lora
38 - vllm/model_executor/layers/fused_moe/
39 - tests/lora
40 commands:
41 # FIXIT: find out which code initialize cuda before running the test
42 # before the fix, we need to use spawn to test it
43 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
44 # Alot of these tests are on the edge of OOMing
45 - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
46 # There is some Tensor Parallelism related processing logic in LoRA that
47 # requires multi-GPU testing for validation.
48 # Timing-balanced static split from per-file walls in builds 83921/83936.
49 - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
50 - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_chatglm3_tp.py
51 - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_llama_tp.py
52 - test "$$BUILDKITE_PARALLEL_JOB" != "3" || pytest -v -s -x lora/test_qwen3_with_multi_loras.py
53 - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_olmoe_tp.py
54 - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py
55 - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py
56 - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py
57 - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp
58 mirror:
59 amd:
60 label: ":amd: (MI300) LoRA TP (Distributed) %N"
61 dind: false
62 device: mi300_4
63 working_dir: "/vllm-workspace/tests"
64 timeout_in_minutes: 65
65 depends_on:
66 - image-build-amd
67 source_file_dependencies:
68 - vllm/lora
69 - vllm/model_executor/layers/fused_moe/
70 - vllm/platforms/rocm.py
71 - tests/lora
72 commands:
73 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
74 # expandable_segments breaks custom all-reduce IPC on ROCm.
75 - case "$$BUILDKITE_PARALLEL_JOB" in 0|1|2|3) ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" && exit 1;; esac
76 - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_chatglm3_tp.py; fi
77 - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_llama_tp.py; fi
78 - if [ "$$BUILDKITE_PARALLEL_JOB" = "3" ]; then pytest -v -s -x lora/test_qwen3_with_multi_loras.py; fi
79 - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_olmoe_tp.py; fi
80 - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi
81 - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi
82 - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi
83 - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp; fi