A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 00:00 UTC
[CI] Add Qwen3-Next-FP8 to Blackwell model tests (#31049)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com>
Vadim Gimpelson committed
Dec 24, 2025 at 05:21 UTC
bc0a5a0c089844b17cb93f3294348f411e523586
4 files changed
+14
tests/evals/gsm8k/configs/Qwen3-Next-FP8-EP2.yaml
new
+11
@@ -0,0 +1,11 @@
1
+model_name: "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: >-
6
+ --max-model-len 4096
7
+ --tensor-parallel-size 2
8
+ --enable-expert-parallel
9
+ --async-scheduling
10
+env:
11
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
tests/evals/gsm8k/configs/models-blackwell.txt
+1
@@ -4,3 +4,4 @@ Qwen1.5-MoE-W4A16-CT.yaml
4
DeepSeek-V2-Lite-Instruct-FP8.yaml
5
Qwen3-30B-A3B-NVFP4.yaml
6
Qwen3-Next-80B-A3B-NVFP4-EP2.yaml
7
+Qwen3-Next-FP8-EP2.yaml
tests/evals/gsm8k/test_gsm8k_correctness.py
+1
@@ -71,6 +71,7 @@ def test_gsm8k_correctness(config_filename):
71
print(f"Number of questions: {eval_config['num_questions']}")
72
print(f"Number of few-shot examples: {eval_config['num_fewshot']}")
73
print(f"Server args: {' '.join(server_args)}")
74
+ print(f"Environment variables: {env_dict}")
75
76
# Launch server and run evaluation
77
with RemoteOpenAIServer(
tests/utils.py
+1
@@ -106,6 +106,7 @@ class RemoteOpenAIServer:
106
env.update(env_dict)
107
serve_cmd = ["vllm", "serve", model, *vllm_serve_args]
108
print(f"Launching RemoteOpenAIServer with: {' '.join(serve_cmd)}")
109
+ print(f"Environment variables: {env}")
110
self.proc: subprocess.Popen = subprocess.Popen(
111
serve_cmd,
112
env=env,