A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 00:30 UTC
[CI/Build]Add eval config for Qwen3-235B-A22B-Instruct-2507-FP8 (#27113)
Signed-off-by: Huamin Li <3ericli@gmail.com>
Huamin Li committed
Oct 30, 2025 at 00:50 UTC
5be1bed79058ddc1016f2639c52dfb5b597bf39c
5 files changed
+39
-4
.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Instruct-2507-FP8.yaml
new
+14
@@ -0,0 +1,14 @@
1
+model_name: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8"
2
+tasks:
3
+ - name: "mmlu_pro"
4
+ metrics:
5
+ - name: "exact_match,custom-extract"
6
+ value: 0.82
7
+limit: 250 # will run on 250 * 14 subjects = 3500 samples
8
+num_fewshot: 5
9
+enforce_eager: false # we use false to speed up the eval process
10
+kv_cache_dtype: fp8 # we use fp8 to speed up the eval process
11
+max_model_len: 40960
12
+apply_chat_template: true
13
+fewshot_as_multiturn: true
14
+gen_kwargs: "temperature=0,top_p=1,top_k=0,max_gen_toks=5632,until=<|ENDANSWER|>"
.buildkite/lm-eval-harness/configs/models-large-h100.txt
deleted
-1
@@ -1 +0,0 @@
1
-Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml
.buildkite/lm-eval-harness/configs/models-large-hopper.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen3-235B-A22B-Instruct-2507-FP8.yaml
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+11
-3
@@ -21,10 +21,13 @@ def launch_lm_eval(eval_config, tp_size):
21
max_model_len = eval_config.get("max_model_len", 4096)
22
batch_size = eval_config.get("batch_size", "auto")
23
backend = eval_config.get("backend", "vllm")
24
+ enforce_eager = eval_config.get("enforce_eager", "true")
25
+ kv_cache_dtype = eval_config.get("kv_cache_dtype", "auto")
26
model_args = (
27
f"pretrained={eval_config['model_name']},"
28
f"tensor_parallel_size={tp_size},"
27
- f"enforce_eager=true,"
29
+ f"enforce_eager={enforce_eager},"
30
+ f"kv_cache_dtype={kv_cache_dtype},"
31
f"add_bos_token=true,"
32
f"trust_remote_code={trust_remote_code},"
33
f"max_model_len={max_model_len},"
@@ -37,8 +40,13 @@ def launch_lm_eval(eval_config, tp_size):
40
limit=eval_config["limit"],
41
# TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help
42
# text models. however, this is regressing measured strict-match for
40
- # existing text models in CI, so only apply it for mm.
41
- apply_chat_template=backend == "vllm-vlm",
43
+ # existing text models in CI, so only apply it for mm, or explicitly set
44
+ apply_chat_template=eval_config.get(
45
+ "apply_chat_template", backend == "vllm-vlm"
46
+ ),
47
+ fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", False),
48
+ # Forward decoding and early-stop controls (e.g., max_gen_toks, until=...)
49
+ gen_kwargs=eval_config.get("gen_kwargs"),
50
batch_size=batch_size,
51
)
52
return results
.buildkite/test-pipeline.yaml
+13
@@ -1186,6 +1186,19 @@ steps:
1186
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1187
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
1188
1189
+##### H100 test #####
1190
+- label: LM Eval Large Models (H100) # optional
1191
+ gpu: h100
1192
+ optional: true
1193
+ num_gpus: 4
1194
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
1195
+ source_file_dependencies:
1196
+ - csrc/
1197
+ - vllm/model_executor/layers/quantization
1198
+ commands:
1199
+ - export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
1200
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
1201
+
1202
##### H200 test #####
1203
- label: Distributed Tests (H200) # optional
1204
gpu: h200