A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 08:45 UTC
[CI] Generalize gsm8k test args and add Qwen3-Next MTP B200 test (#30723)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Dec 16, 2025 at 14:28 UTC
10ee1c64cfa7c0b7f68e9ee793435c9cafbf821a
14 files changed
+78
-57
.buildkite/test-pipeline.yaml
+2
-2
@@ -654,7 +654,7 @@ steps:
654
- vllm/model_executor/layers/quantization
655
autorun_on_main: true
656
commands:
657
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
657
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
658
659
- label: OpenAI API correctness # 22min
660
timeout_in_minutes: 30
@@ -1064,7 +1064,7 @@ steps:
1064
- csrc/
1065
- vllm/model_executor/layers/quantization
1066
commands:
1067
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt --tp-size=1
1067
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
1068
1069
##### 1 GPU test #####
1070
##### multi gpus test #####
tests/evals/gsm8k/README.md
+9
-4
@@ -7,9 +7,8 @@ This directory contains a replacement for the lm-eval-harness GSM8K evaluation,
7
### Run tests with pytest (like buildkite)
8
9
```bash
10
-pytest -s -v tests/gsm8k/test_gsm8k_correctness.py \
11
- --config-list-file=configs/models-small.txt \
12
- --tp-size=1
10
+pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \
11
+ --config-list-file=configs/models-small.txt
12
```
13
14
### Run standalone evaluation script
@@ -31,5 +30,11 @@ model_name: "Qwen/Qwen2.5-1.5B-Instruct"
30
accuracy_threshold: 0.54 # Minimum expected accuracy
31
num_questions: 1319 # Number of questions (default: full test set)
32
num_fewshot: 5 # Few-shot examples from train set
34
-max_model_len: 4096 # Model context length
33
+server_args: "--max-model-len 4096 --tensor-parallel-size 2" # Server arguments
34
+env: # Environment variables (optional)
35
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
36
```
37
+
38
+The `server_args` field accepts any arguments that can be passed to `vllm serve`.
39
+
40
+The `env` field accepts a dictionary of environment variables to set for the server process.
tests/evals/gsm8k/configs/DeepSeek-V2-Lite-Instruct-FP8.yaml
+1
-2
@@ -2,5 +2,4 @@ model_name: "RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8"
2
accuracy_threshold: 0.72
3
num_questions: 1319
4
num_fewshot: 5
5
-max_model_len: 4096
6
-
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Llama-3-8B-Instruct-nonuniform-CT.yaml
+1
-1
@@ -2,4 +2,4 @@ model_name: "nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test"
2
accuracy_threshold: 0.74
3
num_questions: 1319
4
num_fewshot: 5
5
-max_model_len: 4096
\ No newline at end of file
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Llama-3.2-1B-Instruct-INT8-CT.yaml
+1
-1
@@ -2,4 +2,4 @@ model_name: "RedHatAI/Llama-3.2-1B-Instruct-quantized.w8a8"
2
accuracy_threshold: 0.31
3
num_questions: 1319
4
num_fewshot: 5
5
-max_model_len: 4096
\ No newline at end of file
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Qwen1.5-MoE-W4A16-CT.yaml
+1
-1
@@ -2,4 +2,4 @@ model_name: "nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16"
2
accuracy_threshold: 0.45
3
num_questions: 1319
4
num_fewshot: 5
5
-max_model_len: 4096
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
+1
-1
@@ -2,4 +2,4 @@ model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic"
2
accuracy_threshold: 0.60
3
num_questions: 1319
4
num_fewshot: 5
5
-max_model_len: 4096
\ No newline at end of file
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Qwen3-0.6B-FP8.yaml
+1
-1
@@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-0.6B-FP8"
2
accuracy_threshold: 0.375
3
num_questions: 1319
4
num_fewshot: 5
5
-max_model_len: 4096
\ No newline at end of file
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Qwen3-30B-A3B-NVFP4.yaml
+1
-2
@@ -2,5 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-FP4"
2
accuracy_threshold: 0.89
3
num_questions: 1319
4
num_fewshot: 5
5
-max_model_len: 4096
6
-
5
+server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2.yaml
new
+12
@@ -0,0 +1,12 @@
1
+model_name: "nm-testing/Qwen3-Next-80B-A3B-Instruct-NVFP4"
2
+accuracy_threshold: 0.75
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --enforce-eager
7
+ --max-model-len 4096
8
+ --tensor-parallel-size 2
9
+ --enable-expert-parallel
10
+ --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}'
11
+env:
12
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
tests/evals/gsm8k/configs/models-blackwell.txt
+1
@@ -3,3 +3,4 @@ Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
3
Qwen1.5-MoE-W4A16-CT.yaml
4
DeepSeek-V2-Lite-Instruct-FP8.yaml
5
Qwen3-30B-A3B-NVFP4.yaml
6
+Qwen3-Next-80B-A3B-NVFP4-EP2.yaml
tests/evals/gsm8k/conftest.py
+3
-5
@@ -11,14 +11,12 @@ def pytest_addoption(parser):
11
default="configs/models-small.txt",
12
help="File containing list of config files to test",
13
)
14
- parser.addoption("--tp-size", default=1, type=int, help="Tensor parallel size")
14
15
16
def pytest_generate_tests(metafunc):
17
"""Generate test parameters from config files."""
18
if "config_filename" in metafunc.fixturenames:
19
config_list_file = metafunc.config.getoption("--config-list-file")
21
- tp_size = metafunc.config.getoption("--tp-size")
20
21
# Handle both relative and absolute paths
22
config_list_path = Path(config_list_file)
@@ -55,9 +53,9 @@ def pytest_generate_tests(metafunc):
53
# Generate test parameters
54
if config_files:
55
metafunc.parametrize(
58
- ["config_filename", "tp_size"],
59
- [(config_file, int(tp_size)) for config_file in config_files],
60
- ids=[f"{config_file.stem}-tp{tp_size}" for config_file in config_files],
56
+ "config_filename",
57
+ config_files,
58
+ ids=[config_file.stem for config_file in config_files],
59
)
60
else:
61
print("No config files found, test will be skipped")
tests/evals/gsm8k/test_gsm8k_correctness.py
+41
-29
@@ -5,30 +5,31 @@ GSM8K evaluation using vLLM server and isolated GSM8K script.
5
Replacement for lm-eval-harness with better performance and control.
6
7
Usage:
8
-pytest -s -v test_gsm8k_correctness.py \
9
- --config-list-file=configs/models-small.txt \
10
- --tp-size=1
8
+pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \
9
+ --config-list-file=configs/models-small.txt
10
"""
11
12
+import shlex
13
+
14
import yaml
15
16
from tests.utils import RemoteOpenAIServer
17
18
from .gsm8k_eval import evaluate_gsm8k
19
19
-RTOL = 0.08 # Relative tolerance for accuracy comparison
20
+TOL = 0.08 # Absolute tolerance for accuracy comparison
21
22
22
-def launch_gsm8k_eval(eval_config, server_url, tp_size):
23
- """Launch GSM8K evaluation using our isolated script."""
23
+def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
24
+ """Run GSM8K evaluation using our isolated script."""
25
# Extract host and port from server URL
26
if "://" in server_url:
27
server_url = server_url.split("://")[1]
28
29
host_port = server_url.split("/")[0] # Remove path if present
30
if ":" in host_port:
30
- host, port = host_port.split(":")
31
- port = int(port)
31
+ host, p = host_port.split(":")
32
+ port = int(p)
33
else:
34
host = host_port
35
port = 8000
@@ -48,46 +49,57 @@ def launch_gsm8k_eval(eval_config, server_url, tp_size):
49
return results
50
51
51
-def test_gsm8k_correctness_param(config_filename, tp_size):
52
+def test_gsm8k_correctness(config_filename):
53
"""Test GSM8K correctness for a given model configuration."""
54
eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
55
55
- # Server arguments
56
- server_args = [
57
- "--max-model-len",
58
- str(eval_config.get("max_model_len", 4096)),
59
- "--enforce-eager",
60
- "--trust-remote-code",
61
- "--tensor-parallel-size",
62
- str(tp_size),
63
- ]
56
+ # Parse server arguments from config (use shlex to handle quoted strings)
57
+ server_args_str = eval_config.get("server_args", "")
58
+ server_args = shlex.split(server_args_str) if server_args_str else []
59
+
60
+ # Add standard server arguments
61
+ server_args.extend(
62
+ [
63
+ "--trust-remote-code",
64
+ ]
65
+ )
66
67
env_dict = eval_config.get("env", None)
68
69
+ print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}")
70
+ print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
71
+ print(f"Number of questions: {eval_config['num_questions']}")
72
+ print(f"Number of few-shot examples: {eval_config['num_fewshot']}")
73
+ print(f"Server args: {' '.join(server_args)}")
74
+
75
# Launch server and run evaluation
76
with RemoteOpenAIServer(
69
- eval_config["model_name"], server_args, env_dict=env_dict, max_wait_seconds=480
77
+ eval_config["model_name"],
78
+ server_args,
79
+ env_dict=env_dict,
80
+ max_wait_seconds=600,
81
) as remote_server:
82
server_url = remote_server.url_for("v1")
83
+ print(f"Server started at: {server_url}")
84
73
- results = launch_gsm8k_eval(eval_config, server_url, tp_size)
85
+ results = run_gsm8k_eval(eval_config, server_url)
86
75
- # Check accuracy against threshold
76
- measured_accuracy = results["accuracy"]
77
- expected_accuracy = eval_config["accuracy_threshold"]
87
+ measured_metric = results["accuracy"]
88
+ expected_metric = eval_config["accuracy_threshold"]
89
90
print(f"GSM8K Results for {eval_config['model_name']}:")
80
- print(f" Accuracy: {measured_accuracy:.3f}")
81
- print(f" Expected: {expected_accuracy:.3f}")
91
+ print(f" Measured metric: {measured_metric:.4f}")
92
+ print(f" Expected metric: {expected_metric:.4f}")
93
+ print(f" Tolerance: {TOL:.4f}")
94
print(f" Questions: {results['num_questions']}")
95
print(f" Invalid rate: {results['invalid_rate']:.3f}")
96
print(f" Latency: {results['latency']:.1f}s")
97
print(f" QPS: {results['questions_per_second']:.1f}")
98
87
- # Verify accuracy is within tolerance
88
- assert measured_accuracy >= expected_accuracy - RTOL, (
89
- f"Accuracy too low: {measured_accuracy:.3f} < "
90
- f"{expected_accuracy:.3f} - {RTOL:.3f}"
99
+ # Verify metric is within tolerance
100
+ assert measured_metric >= expected_metric - TOL, (
101
+ f"GSM8K metric too low: {measured_metric:.4f} < "
102
+ f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}"
103
)
104
105
print(f"✅ GSM8K test passed for {eval_config['model_name']}")
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+3
-8
@@ -626,17 +626,11 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
626
apply_router_weight_on_input=layer.apply_router_weight_on_input,
627
)
628
else:
629
+ # If no modular kernel is provided, use cutlass_moe_fp4 for TP case
630
+ # only (no EP).
631
from vllm.model_executor.layers.fused_moe.cutlass_moe import cutlass_moe_fp4
632
631
- assert layer.expert_map is None, (
632
- "Expert Parallelism / expert_map "
633
- "is currently not supported for "
634
- "CompressedTensorsW4A4Nvfp4MoEMethod."
635
- )
633
assert self.moe_quant_config is not None
637
-
638
- # Cutlass moe takes in activations in BF16/Half precision
639
- # and fp4 quantized weights loaded from the checkpoint
634
return cutlass_moe_fp4(
635
a=x,
636
w1_fp4=layer.w13_weight,
@@ -644,6 +638,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
638
topk_weights=topk_weights,
639
topk_ids=topk_ids,
640
quant_config=self.moe_quant_config,
641
+ expert_map=layer.expert_map,
642
apply_router_weight_on_input=layer.apply_router_weight_on_input,
643
# TODO(bnell): derive these from arguments
644
m=x.shape[0],