A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 01:15 UTC
[Bugfix] Add regression test for MoE quant_config under torch.compile (#34335)
Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
Matthias Gehre committed
Feb 20, 2026 at 06:27 UTC
4e2c7caf2d11444ce6c1e4895bc921c93610bd7c
1 file changed
+23
tests/quantization/test_compressed_tensors.py
+23
@@ -816,3 +816,26 @@ def test_compressed_tensors_moe_ignore_with_model(vllm_runner):
816
# Verify the model can generate output
817
output = llm.generate_greedy("Hello, my name is", max_tokens=4)
818
assert output
819
+
820
+
821
+def test_w4a16_moe_torch_compile(vllm_runner):
822
+ """Regression test: MoE quant_config must be initialized inside the
823
+ moe_forward custom op, not just in forward_native which is compiled by
824
+ Dynamo (attribute mutations are not replayed at runtime).
825
+
826
+ Without the fix in _moe_forward/_moe_forward_shared, this hits:
827
+ AssertionError: Hidden size mismatch 2048 != 1024
828
+ because use_int4_w4a16 is False (moe_quant_config stays None).
829
+ """
830
+ model_path = "nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable"
831
+
832
+ with vllm_runner(
833
+ model_path,
834
+ enforce_eager=False,
835
+ max_model_len=256,
836
+ compilation_config={
837
+ "cudagraph_mode": "NONE",
838
+ },
839
+ ) as llm:
840
+ output = llm.generate_greedy("Hi", max_tokens=1)
841
+ assert output