2a7d5b7324 · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -139,19 +139,6 @@ steps:

139 - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py

140 - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'

141

142 -#-------------------------------------------------------- mi250 · benchmarks ---------------------------------------------------------#

143 -

144 -- label: Benchmarks # TBD

145 - timeout_in_minutes: 180

146 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]

147 - agent_pool: mi250_1

148 - working_dir: "/vllm-workspace/.buildkite"

149 - source_file_dependencies:

150 - - benchmarks/

151 - - vllm/platforms/rocm.py

152 - commands:

153 - - bash scripts/run-benchmarks.sh

154 -

142 #---------------------------------------------------------- mi250 · compile ----------------------------------------------------------#

143

144 - label: PyTorch Compilation Unit Tests # TBD

@@ -485,7 +472,7 @@ steps:

472 - pytest -v -s model_executor -m '(not slow_test)'

473 - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py

474

488 -#---------------------------------------------------------- mi250 · models -----------------------------------------------------------#

475 +#------------------------------------------------------ mi250 · models / basic -------------------------------------------------------#

476

477 - label: Basic Models Test (Other CPU) # TBD

478 timeout_in_minutes: 180

@@ -546,6 +533,8 @@ steps:

533 commands:

534 - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py

535

536 +#----------------------------------------------------- mi250 · models / language -----------------------------------------------------#

537 +

538 - label: Language Models Test (MTEB) # TBD

539 timeout_in_minutes: 180

540 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]

@@ -590,6 +579,8 @@ steps:

579 - pip freeze | grep -E 'torch'

580 - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB

581

582 +#---------------------------------------------------- mi250 · models / multimodal ----------------------------------------------------#

583 +

584 - label: Multi-Modal Models (Extended Generation 2) # TBD

585 timeout_in_minutes: 180

586 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]

@@ -976,18 +967,6 @@ steps:

967

968 #-------------------------------------------------------- mi300 · benchmarks ---------------------------------------------------------#

969

979 -- label: Benchmarks # TBD

980 - timeout_in_minutes: 180

981 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]

982 - agent_pool: mi300_1

983 - optional: true

984 - working_dir: "/vllm-workspace/.buildkite"

985 - source_file_dependencies:

986 - - benchmarks/

987 - - vllm/platforms/rocm.py

988 - commands:

989 - - bash scripts/run-benchmarks.sh

990 -

970 - label: Benchmarks CLI Test # TBD

971 timeout_in_minutes: 180

972 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]

@@ -1759,7 +1738,7 @@ steps:

1738 - pytest -v -s -x lora/test_gptoss_tp.py

1739 - pytest -v -s -x lora/test_qwen35_densemodel_lora.py

1740

1762 -#---------------------------------------------------------- mi300 · models -----------------------------------------------------------#

1741 +#----------------------------------------------------- mi300 · models / language -----------------------------------------------------#

1742

1743 - label: Language Models Test (Extended Pooling) # TBD

1744 timeout_in_minutes: 180

@@ -1787,6 +1766,8 @@ steps:

1766 - pip freeze | grep -E 'torch'

1767 - pytest -v -s models/language -m 'core_model and (not slow_test)'

1768

1769 +#---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------#

1770 +

1771 - label: Multi-Modal Models (Extended Generation 1) # TBD

1772 timeout_in_minutes: 180

1773 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]

@@ -1892,10 +1873,11 @@ steps:

1873 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

1874 - pytest -v -s models/multimodal/processing/test_tensor_schema.py

1875

1895 -- label: Multi-Modal Processor (CPU) # TBD

1876 +- label: Multi-Modal Processor (CPU) %N # TBD

1877 timeout_in_minutes: 180

1878 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]

1879 agent_pool: mi300_1

1880 + parallelism: 4

1881 no_gpu: true

1882 optional: true

1883 working_dir: "/vllm-workspace/tests"

@@ -1905,7 +1887,9 @@ steps:

1887 - tests/models/registry.py

1888 commands:

1889 - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git

1908 - - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py

1890 + - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB

1891 +

1892 +#----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------#

1893

1894 - label: Quantized Models Test # TBD

1895 timeout_in_minutes: 180

@@ -1921,7 +1905,31 @@ steps:

1905 commands:

1906 - pytest -v -s models/quantization

1907

1924 -- label: Transformers Nightly Models # TBD

1908 +#-------------------------------------------------- mi300 · models / transformers ---------------------------------------------------#

1909 +

1910 +- label: Transformers Nightly Models (Shardable) %N # TBD

1911 + timeout_in_minutes: 180

1912 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]

1913 + agent_pool: mi300_1

1914 + parallelism: 4

1915 + optional: true

1916 + working_dir: "/vllm-workspace/"

1917 + source_file_dependencies:

1918 + - vllm/model_executor/models/

1919 + - vllm/model_executor/model_loader/

1920 + - vllm/multimodal/

1921 + - vllm/model_executor/layers/

1922 + - vllm/v1/attention/backends/

1923 + - vllm/v1/attention/selector.py

1924 + - vllm/_aiter_ops.py

1925 + - vllm/platforms/rocm.py

1926 + - tests/models/

1927 + commands:

1928 + - pip install --upgrade git+https://github.com/huggingface/transformers

1929 + - pytest -v -s tests/models/test_initialization.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB

1930 + - pytest -v -s tests/models/multimodal/processing/ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB

1931 +

1932 +- label: Transformers Nightly Models (Single) # TBD

1933 timeout_in_minutes: 180

1934 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]

1935 agent_pool: mi300_1

@@ -1940,9 +1948,7 @@ steps:

1948 - examples/

1949 commands:

1950 - pip install --upgrade git+https://github.com/huggingface/transformers

1943 - - pytest -v -s tests/models/test_initialization.py

1951 - pytest -v -s tests/models/test_transformers.py

1945 - - pytest -v -s tests/models/multimodal/processing/

1952 - pytest -v -s tests/models/multimodal/test_mapping.py

1953 - python3 examples/basic/offline_inference/chat.py

1954 - python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl

@@ -2391,7 +2397,7 @@ steps:

2397 - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt

2398 - DP_EP=1 ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

2399

2394 -- label: Hyrbid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD

2400 +- label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD

2401 timeout_in_minutes: 180

2402 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]

2403 agent_pool: mi300_4

@@ -2593,7 +2599,7 @@ steps:

2599 - export VLLM_WORKER_MULTIPROC_METHOD=spawn

2600 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8

2601

2596 -#---------------------------------------------------------- mi325 · models -----------------------------------------------------------#

2602 +#----------------------------------------------------- mi325 · models / language -----------------------------------------------------#

2603

2604 - label: Language Models Test (Extended Generation) # TBD

2605 timeout_in_minutes: 180

@@ -2624,6 +2630,8 @@ steps:

2630 - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'

2631 - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB

2632

2633 +#---------------------------------------------------- mi325 · models / multimodal ----------------------------------------------------#

2634 +

2635 - label: Multi-Modal Models (Extended Pooling) # TBD

2636 timeout_in_minutes: 180

2637 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]

@@ -3043,7 +3051,7 @@ steps:

3051 commands:

3052 - pytest -v -s kernels/moe/test_deepep_moe.py

3053

3046 -#---------------------------------------------------------- mi355 · models -----------------------------------------------------------#

3054 +#----------------------------------------------------- mi355 · models / language -----------------------------------------------------#

3055

3056 - label: Language Models Test (Extended Generation) # TBD

3057 timeout_in_minutes: 180

@@ -3111,6 +3119,8 @@ steps:

3119 - pip freeze | grep -E 'torch'

3120 - pytest -v -s models/language -m 'core_model and (not slow_test)'

3121

3122 +#---------------------------------------------------- mi355 · models / multimodal ----------------------------------------------------#

3123 +

3124 - label: Multi-Modal Models (Extended Generation 1) # TBD

3125 timeout_in_minutes: 180

3126 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]

@@ -3182,6 +3192,8 @@ steps:

3192 - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model

3193 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model

3194

3195 +#----------------------------------------------------- mi355 · models / quantized -----------------------------------------------------#

3196 +

3197 - label: Quantized Models Test # TBD

3198 timeout_in_minutes: 180

3199 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]

[ROCm][CI] Remove benchmarks test group and shard long test groups (#41669)