[CI][ROCm] Restore Wikitext coverage for Qwen OCP-MX (#55410)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Sep 4, 2026 at 21:50 UTC
ea5f41edd7f157a7c24bc86b59d8ae3b55b6dcf5
5 files changed
+38
-14
tests/evals/gsm8k/configs/Qwen-1.5-MOE-W-MXFP4-A-MXFP6.yaml
deleted
-5
@@ -1,5 +0,0 @@
1
-model_name: "fxmarty/qwen1.5_moe_a2.7b_chat_w_fp4_a_fp6_e2m3"
2
-accuracy_threshold: 0.4405
3
-num_questions: 50
4
-num_fewshot: 5
5
-server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/Qwen-1.5-MOE-W-MXFP6-A-MXFP6.yaml
deleted
-5
@@ -1,5 +0,0 @@
1
-model_name: "fxmarty/qwen1.5_moe_a2.7b_chat_w_fp6_e3m2_a_fp6_e3m2"
2
-accuracy_threshold: 0.5049
3
-num_questions: 50
4
-num_fewshot: 5
5
-server_args: "--enforce-eager --max-model-len 4096"
tests/evals/gsm8k/configs/models-gfx950-large.txt
-2
@@ -7,5 +7,3 @@
7
# DeepSeek-V4-Pro-NVFP4.yaml
8
DeepSeek-V4-Flash-NVFP4.yaml
9
Qwen3-30B-A3B-NVFP4-quark.yaml
10
-Qwen-1.5-MOE-W-MXFP4-A-MXFP6.yaml
11
-Qwen-1.5-MOE-W-MXFP6-A-MXFP6.yaml
\ No newline at end of file
tests/evals/gsm8k/configs/models-mi3xx.txt
-2
@@ -6,5 +6,3 @@ Qwen3-30B-A3B-NVFP4.yaml
6
Qwen3-30B-A3B-NVFP4-quark.yaml
7
Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml
8
Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml
9
-Qwen-1.5-MOE-W-MXFP4-A-MXFP6.yaml
10
-Qwen-1.5-MOE-W-MXFP6-A-MXFP6.yaml
\ No newline at end of file
tests/quantization/test_quark.py
+38
@@ -1188,6 +1188,44 @@ class AccuracyTestConfig:
1188
return model_args
1189
1190
1191
+WIKITEXT_ACCURACY_CONFIGS = [
1192
+ AccuracyTestConfig(
1193
+ model_name="fxmarty/qwen1.5_moe_a2.7b_chat_w_fp4_a_fp6_e2m3",
1194
+ excepted_value=11.3,
1195
+ ),
1196
+ AccuracyTestConfig(
1197
+ model_name="fxmarty/qwen1.5_moe_a2.7b_chat_w_fp6_e3m2_a_fp6_e3m2",
1198
+ excepted_value=10.6,
1199
+ ),
1200
+]
1201
+
1202
+
1203
+@pytest.mark.skipif(
1204
+ not QUARK_MXFP4_AVAILABLE,
1205
+ reason=f"amd-quark>={QUARK_MXFP4_MIN_VERSION} is not available",
1206
+)
1207
+@pytest.mark.parametrize(
1208
+ "config", WIKITEXT_ACCURACY_CONFIGS, ids=lambda config: config.model_name
1209
+)
1210
+@pytest.mark.parametrize("tp_size", [1, 2])
1211
+def test_ocp_mx_wikitext_correctness(config: AccuracyTestConfig, tp_size: int):
1212
+ device_count = torch.accelerator.device_count()
1213
+ if device_count < tp_size:
1214
+ pytest.skip(f"This test requires >={tp_size} gpus, got only {device_count}")
1215
+
1216
+ results = lm_eval.simple_evaluate(
1217
+ model="vllm",
1218
+ model_args=config.get_model_args(
1219
+ tp_size=tp_size, kwargs={"cudagraph_capture_sizes": [16]}
1220
+ ),
1221
+ tasks="wikitext",
1222
+ batch_size=64,
1223
+ )
1224
+
1225
+ measured_value = results["results"]["wikitext"]["word_perplexity,none"]
1226
+ assert measured_value == pytest.approx(config.excepted_value, abs=0.1)
1227
+
1228
+
1229
GSM8K_ACCURACY_CONFIGS = [
1230
# Private model.
1231
AccuracyTestConfig(