@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bcf43ab1f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 23:45 UTC

[CI/Build][AMD] Add Llama4 Maverick FP8 to AMD CI (#28695)

Signed-off-by: zhewenli <zhewenli@meta.com>

Zhewen Li committed Dec 4, 2025 at 16:07 UTC bcf43ab1f380208ea33769c49d116ea83f915080
4 files changed +158 -112
.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml
+1
@@ -8,3 +8,4 @@ tasks:
8 value: 0.80
9 limit: 250 # will run on 250 * 14 subjects = 3500 samples
10 num_fewshot: 5
11 +rtol: 0.05
.buildkite/lm-eval-harness/configs/models-large-rocm.txt new
+1
@@ -0,0 +1 @@
1 +Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+55 -20
@@ -9,11 +9,40 @@ pytest -s -v test_lm_eval_correctness.py \
9 --tp-size=1
10 """
11
12 +import os
13 +from contextlib import contextmanager
14 +
15 import lm_eval
16 import numpy as np
17 import yaml
18
16 -RTOL = 0.08
19 +DEFAULT_RTOL = 0.08
20 +
21 +
22 +@contextmanager
23 +def scoped_env_vars(new_env: dict[str, str]):
24 + if not new_env:
25 + # Fast path: nothing to do
26 + yield
27 + return
28 +
29 + old_values = {}
30 + new_keys = []
31 +
32 + try:
33 + for key, value in new_env.items():
34 + if key in os.environ:
35 + old_values[key] = os.environ[key]
36 + else:
37 + new_keys.append(key)
38 + os.environ[key] = str(value)
39 + yield
40 + finally:
41 + # Restore / clean up
42 + for key, value in old_values.items():
43 + os.environ[key] = value
44 + for key in new_keys:
45 + os.environ.pop(key, None)
46
47
48 def launch_lm_eval(eval_config, tp_size):
@@ -32,23 +61,26 @@ def launch_lm_eval(eval_config, tp_size):
61 f"trust_remote_code={trust_remote_code},"
62 f"max_model_len={max_model_len},"
63 )
35 - results = lm_eval.simple_evaluate(
36 - model=backend,
37 - model_args=model_args,
38 - tasks=[task["name"] for task in eval_config["tasks"]],
39 - num_fewshot=eval_config["num_fewshot"],
40 - limit=eval_config["limit"],
41 - # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help
42 - # text models. however, this is regressing measured strict-match for
43 - # existing text models in CI, so only apply it for mm, or explicitly set
44 - apply_chat_template=eval_config.get(
45 - "apply_chat_template", backend == "vllm-vlm"
46 - ),
47 - fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", False),
48 - # Forward decoding and early-stop controls (e.g., max_gen_toks, until=...)
49 - gen_kwargs=eval_config.get("gen_kwargs"),
50 - batch_size=batch_size,
51 - )
64 +
65 + env_vars = eval_config.get("env_vars", None)
66 + with scoped_env_vars(env_vars):
67 + results = lm_eval.simple_evaluate(
68 + model=backend,
69 + model_args=model_args,
70 + tasks=[task["name"] for task in eval_config["tasks"]],
71 + num_fewshot=eval_config["num_fewshot"],
72 + limit=eval_config["limit"],
73 + # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help
74 + # text models. however, this is regressing measured strict-match for
75 + # existing text models in CI, so only apply it for mm, or explicitly set
76 + apply_chat_template=eval_config.get(
77 + "apply_chat_template", backend == "vllm-vlm"
78 + ),
79 + fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", False),
80 + # Forward decoding and early-stop controls (e.g., max_gen_toks, until=...)
81 + gen_kwargs=eval_config.get("gen_kwargs"),
82 + batch_size=batch_size,
83 + )
84 return results
85
86
@@ -57,6 +89,8 @@ def test_lm_eval_correctness_param(config_filename, tp_size):
89
90 results = launch_lm_eval(eval_config, tp_size)
91
92 + rtol = eval_config.get("rtol", DEFAULT_RTOL)
93 +
94 success = True
95 for task in eval_config["tasks"]:
96 for metric in task["metrics"]:
@@ -64,8 +98,9 @@ def test_lm_eval_correctness_param(config_filename, tp_size):
98 measured_value = results["results"][task["name"]][metric["name"]]
99 print(
100 f"{task['name']} | {metric['name']}: "
67 - f"ground_truth={ground_truth} | measured={measured_value}"
101 + f"ground_truth={ground_truth:.3f} | "
102 + f"measured={measured_value:.3f} | rtol={rtol}"
103 )
69 - success = success and np.isclose(ground_truth, measured_value, rtol=RTOL)
104 + success = success and np.isclose(ground_truth, measured_value, rtol=rtol)
105
106 assert success
.buildkite/test-amd.yaml
+101 -92
@@ -718,17 +718,6 @@ steps:
718 - uv pip install --system conch-triton-kernels
719 - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
720
721 -- label: LM Eval Small Models # 15min
722 - timeout_in_minutes: 20
723 - mirror_hardwares: [amdexperimental, amdproduction]
724 - agent_pool: mi325_1
725 - # grade: Blocking
726 - source_file_dependencies:
727 - - csrc/
728 - - vllm/model_executor/layers/quantization
729 - commands:
730 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
731 -
721 - label: OpenAI API correctness # 10min
722 timeout_in_minutes: 15
723 mirror_hardwares: [amdexperimental, amdproduction]
@@ -974,19 +963,6 @@ steps:
963 - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing
964 - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
965
977 -- label: Multi-Modal Accuracy Eval (Small Models) # 10min
978 - timeout_in_minutes: 70
979 - mirror_hardwares: [amdexperimental, amdproduction]
980 - agent_pool: mi325_1
981 - # grade: Blocking
982 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
983 - source_file_dependencies:
984 - - vllm/multimodal/
985 - - vllm/inputs/
986 - - vllm/v1/core/
987 - commands:
988 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
989 -
966 - label: Multi-Modal Models Test (Extended) 1 # 60min
967 timeout_in_minutes: 120
968 mirror_hardwares: [amdexperimental]
@@ -1162,21 +1138,6 @@ steps:
1138 # Run all e2e fusion tests
1139 - pytest -v -s tests/compile/distributed/test_fusions_e2e.py
1140
1165 -- label: ROCm GPT-OSS Eval
1166 - timeout_in_minutes: 60
1167 - working_dir: "/vllm-workspace/"
1168 - agent_pool: mi325_1
1169 - mirror_hardwares: [amdexperimental, amdproduction]
1170 - optional: true # run on nightlies
1171 - source_file_dependencies:
1172 - - tests/evals/gpt_oss
1173 - - vllm/model_executor/models/gpt_oss.py
1174 - - vllm/model_executor/layers/quantization/mxfp4.py
1175 - - vllm/v1/attention/backends/flashinfer.py
1176 - commands:
1177 - - uv pip install --system 'gpt-oss[eval]==0.0.5'
1178 - - VLLM_ROCM_USE_AITER_MHA=0 VLLM_ROCM_USE_AITER=1 VLLM_USE_AITER_UNIFIED_ATTENTION=1 pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58
1179 -
1141 - label: Blackwell Quantized MoE Test
1142 timeout_in_minutes: 60
1143 working_dir: "/vllm-workspace/"
@@ -1194,16 +1155,6 @@ steps:
1155 commands:
1156 - pytest -s -v tests/quantization/test_blackwell_moe.py
1157
1197 -- label: Blackwell LM Eval Small Models
1198 - timeout_in_minutes: 120
1199 - gpu: b200
1200 - optional: true # run on nightlies
1201 - source_file_dependencies:
1202 - - csrc/
1203 - - vllm/model_executor/layers/quantization
1204 - commands:
1205 - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt --tp-size=1
1206 -
1158 ##### 1 GPU test #####
1159 ##### multi gpus test #####
1160
@@ -1380,7 +1331,7 @@ steps:
1331 - pytest -v -s -x lora/test_llm_with_multi_loras.py
1332 - pytest -v -s -x lora/test_olmoe_tp.py
1333
1383 - # Disabled for now because MXFP4 backend on non-cuda platform
1334 + # Disabled for now because MXFP4 backend on non-cuda platform
1335 # doesn't support LoRA yet
1336 #- pytest -v -s -x lora/test_gptoss_tp.py
1337
@@ -1446,37 +1397,6 @@ steps:
1397 - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
1398 - pytest -v -s -x lora/test_mixtral.py
1399
1449 -- label: LM Eval Large Models # optional
1450 - mirror_hardwares: [amdexperimental, amdproduction]
1451 - agent_pool: mi325_4
1452 - # grade: Blocking
1453 - gpu: a100
1454 - optional: true
1455 - num_gpus: 4
1456 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1457 - source_file_dependencies:
1458 - - csrc/
1459 - - vllm/model_executor/layers/quantization
1460 - commands:
1461 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1462 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
1463 -
1464 -##### H100 test #####
1465 -- label: LM Eval Large Models (H100) # optional
1466 - mirror_hardwares: [amdexperimental, amdproduction]
1467 - agent_pool: mi325_4
1468 - # grade: Blocking
1469 - gpu: h100
1470 - optional: true
1471 - num_gpus: 4
1472 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1473 - source_file_dependencies:
1474 - - csrc/
1475 - - vllm/model_executor/layers/quantization
1476 - commands:
1477 - - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
1478 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
1479 -
1400 ##### H200 test #####
1401 - label: Distributed Tests (H200) # optional
1402 mirror_hardwares: [amdexperimental]
@@ -1508,20 +1428,94 @@ steps:
1428 - pytest -v -s tests/distributed/test_nccl_symm_mem_allreduce.py
1429 - pytest -v -s tests/v1/distributed/test_dbo.py
1430
1511 -##### RL Integration Tests #####
1512 -- label: Prime-RL Integration Test # 15min
1513 - mirror_hardwares: [amdexperimental]
1514 - agent_pool: mi325_2
1431 +##### E2E Eval Tests #####
1432 +- label: LM Eval Small Models (1 Card) # 15min
1433 + timeout_in_minutes: 20
1434 + mirror_hardwares: [amdexperimental, amdproduction]
1435 + agent_pool: mi325_1
1436 # grade: Blocking
1516 - timeout_in_minutes: 30
1437 + source_file_dependencies:
1438 + - csrc/
1439 + - vllm/model_executor/layers/quantization
1440 + commands:
1441 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
1442 +
1443 +- label: Blackwell LM Eval Small Models
1444 + timeout_in_minutes: 120
1445 + gpu: b200
1446 + optional: true # run on nightlies
1447 + source_file_dependencies:
1448 + - csrc/
1449 + - vllm/model_executor/layers/quantization
1450 + commands:
1451 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt --tp-size=1
1452 +
1453 +- label: Multi-Modal Accuracy Eval (Small Models) # 10min
1454 + timeout_in_minutes: 70
1455 + mirror_hardwares: [amdexperimental, amdproduction]
1456 + agent_pool: mi325_1
1457 + # grade: Blocking
1458 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1459 + source_file_dependencies:
1460 + - vllm/multimodal/
1461 + - vllm/inputs/
1462 + - vllm/v1/core/
1463 + commands:
1464 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
1465 +
1466 +- label: LM Eval Large Models (4 Card)
1467 + mirror_hardwares: [amdexperimental, amdproduction]
1468 + agent_pool: mi325_4
1469 + # grade: Blocking
1470 + gpu: a100
1471 optional: true
1518 - num_gpus: 2
1519 - working_dir: "/vllm-workspace"
1472 + num_gpus: 4
1473 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1474 source_file_dependencies:
1521 - - vllm/
1522 - - .buildkite/scripts/run-prime-rl-test.sh
1475 + - csrc/
1476 + - vllm/model_executor/layers/quantization
1477 commands:
1524 - - bash .buildkite/scripts/run-prime-rl-test.sh
1478 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1479 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
1480 +
1481 +- label: LM Eval Large Models (H100) # optional
1482 + mirror_hardwares: [amdexperimental, amdproduction]
1483 + agent_pool: mi325_4
1484 + # grade: Blocking
1485 + gpu: h100
1486 + optional: true
1487 + num_gpus: 4
1488 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1489 + source_file_dependencies:
1490 + - csrc/
1491 + - vllm/model_executor/layers/quantization
1492 + commands:
1493 + - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
1494 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
1495 +
1496 +- label: ROCm LM Eval Large Models (8 Card)
1497 + mirror_hardwares: [amdproduction]
1498 + agent_pool: mi325_8
1499 + num_gpus: 8
1500 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1501 + commands:
1502 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1503 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
1504 +
1505 +- label: ROCm GPT-OSS Eval
1506 + timeout_in_minutes: 60
1507 + working_dir: "/vllm-workspace/"
1508 + agent_pool: mi325_1
1509 + mirror_hardwares: [amdexperimental, amdproduction]
1510 + optional: true # run on nightlies
1511 + source_file_dependencies:
1512 + - tests/evals/gpt_oss
1513 + - vllm/model_executor/models/gpt_oss.py
1514 + - vllm/model_executor/layers/quantization/mxfp4.py
1515 + - vllm/v1/attention/backends/flashinfer.py
1516 + commands:
1517 + - uv pip install --system 'gpt-oss[eval]==0.0.5'
1518 + - VLLM_ROCM_USE_AITER_MHA=0 VLLM_ROCM_USE_AITER=1 VLLM_USE_AITER_UNIFIED_ATTENTION=1 pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58
1519
1520 - label: DeepSeek V2-Lite Accuracy
1521 mirror_hardwares: [amdexperimental, amdproduction]
@@ -1554,4 +1548,19 @@ steps:
1548 num_gpus: 2
1549 working_dir: "/vllm-workspace"
1550 commands:
1557 - - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
\ No newline at end of file
1551 + - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
1552 +
1553 +##### RL Integration Tests #####
1554 +- label: Prime-RL Integration Test # 15min
1555 + mirror_hardwares: [amdexperimental]
1556 + agent_pool: mi325_2
1557 + # grade: Blocking
1558 + timeout_in_minutes: 30
1559 + optional: true
1560 + num_gpus: 2
1561 + working_dir: "/vllm-workspace"
1562 + source_file_dependencies:
1563 + - vllm/
1564 + - .buildkite/scripts/run-prime-rl-test.sh
1565 + commands:
1566 + - bash .buildkite/scripts/run-prime-rl-test.sh