test_auto_round.py · tduncan/Nvidia-Optimization-Stack-VLLM

@tduncan / Nvidia-Optimization-Stack-VLLM

Code Commits Issues Pulls Cloud Agent CI/CD

main

py 2,604 lines 83.4 KB

Raw

1	# SPDX-License-Identifier: Apache-2.0
2	# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3	"""Test model set-up and inference for quantized HF models supported
4	on the AutoRound.
5
6	Validating the configuration and printing results for manual checking.
7
8	Run `pytest tests/quantization/test_auto_round.py`.
9	"""
10
11	from types import SimpleNamespace
12	from typing import Any, cast
13
14	import pytest
15	import torch
16
17	from vllm.model_executor.layers.fused_moe import RoutedExperts
18	from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
19	from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
20	from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
21	from vllm.model_executor.layers.quantization.inc import INCConfig
22	from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
23	from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
24	from vllm.model_executor.layers.quantization.inc.schemes import (
25	INCFp8Scheme,
26	INCMxfp4Scheme,
27	INCMxfp8Scheme,
28	INCWna16Scheme,
29	resolve_scheme,
30	)
31	from vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear import (
32	INCFp8LinearScheme,
33	)
34	from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import (
35	INCMxfp8LinearScheme,
36	)
37	from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
38	INCLinearScheme,
39	)
40	from vllm.model_executor.layers.quantization.inc.schemes.inc_w4a8_linear import (
41	INCXPUW4A8LinearMethod,
42	)
43	from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear import (
44	INCARKLinearMethod,
45	INCWNA16LinearScheme,
46	INCXPULinearMethod,
47	)
48	from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_scheme import (
49	_resolve_awq_moe,
50	_resolve_gptq_moe,
51	)
52	from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
53	from vllm.platforms import current_platform
54
55	MODELS = [
56	pytest.param(
57	"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
58	id="auto_round:auto_gptq",
59	),
60	pytest.param(
61	"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound",
62	marks=pytest.mark.skipif(
63	not (current_platform.is_cuda() or current_platform.is_xpu()),
64	reason="AWQ AutoRound model only supports CUDA/XPU backend for now.",
65	),
66	id="auto_round:auto_awq",
67	),
68	pytest.param(
69	"Intel/Qwen3-8B-w2g64-for-ut",
70	marks=pytest.mark.skipif(
71	not (current_platform.is_xpu()),
72	reason="INC int2 on XPU requires the ARK backend.",
73	),
74	id="auto_round:auto_gptq_int2_tp2",
75	),
76	pytest.param(
77	"INC4AI/Qwen3-8B-MXFP8-AR",
78	marks=pytest.mark.skipif(
79	not (current_platform.is_cuda() or current_platform.is_xpu()),
80	reason="MXFP8 AutoRound model only supports CUDA/XPU backend for now.",
81	),
82	id="auto_round:llm_compressor_mxfp8",
83	),
84	pytest.param(
85	"INC4AI/Qwen3-8B-fp-w8g128x128-for-ut",
86	marks=pytest.mark.skipif(
87	not (current_platform.is_cuda()),
88	reason="Block-wise AutoRound model only supports CUDA backend for now.",
89	),
90	id="auto_round:block_wise_fp8_on_cuda",
91	),
92	]
93
94	QWEN3_AUTOROUND_MODELS = [
95	pytest.param(
96	"INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4",
97	marks=pytest.mark.skipif(
98	not (current_platform.is_cuda() or current_platform.is_xpu()),
99	reason="Qwen3-1.7B MXFP4 AutoRound model requires CUDA/XPU.",
100	),
101	id="auto_round:mxfp4:qwen3-1p7b",
102	),
103	pytest.param(
104	"INCModel/Qwen3-30B-A3B-12L-W4A16-test",
105	marks=pytest.mark.skipif(
106	not (current_platform.is_cuda() or current_platform.is_xpu()),
107	reason="Qwen3-30B-A3B W4A16 AutoRound model requires CUDA/XPU.",
108	),
109	id="auto_round:w4a16:qwen3-30b-a3b",
110	),
111	pytest.param(
112	"INCModel/Qwen3-30B-A3B-12L-MXFP4-test",
113	marks=pytest.mark.skipif(
114	not (current_platform.is_cuda() or current_platform.is_xpu()),
115	reason="Qwen3-30B-A3B MXFP4 AutoRound model requires CUDA/XPU.",
116	),
117	id="auto_round:mxfp4:qwen3-30b-a3b",
118	),
119	pytest.param(
120	"INCModel/Qwen3-30B-A3B-12L-MXFP8-test",
121	marks=pytest.mark.skipif(
122	not (current_platform.is_cuda() or current_platform.is_xpu()),
123	reason="Qwen3-30B-A3B MXFP8 AutoRound model requires CUDA/XPU.",
124	),
125	id="auto_round:mxfp8:qwen3-30b-a3b",
126	),
127	]
128
129	MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
130	"INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4": {"enforce_eager": True},
131	"INCModel/Qwen3-30B-A3B-12L-MXFP4-test": {"enforce_eager": True},
132	"INCModel/Qwen3-30B-A3B-12L-MXFP8-test": {"enforce_eager": True},
133	"Intel/Qwen3-8B-w2g64-for-ut": {
134	"block_size": 64,
135	"gpu_memory_utilization": 0.8,
136	"max_model_len": 512,
137	},
138	"INC4AI/Qwen3-8B-MXFP8-AR": {
139	"block_size": 64,
140	"gpu_memory_utilization": 0.8,
141	"max_model_len": 512,
142	},
143	"INC4AI/Qwen3-8B-fp-w8g128x128-for-ut": {
144	"gpu_memory_utilization": 0.8,
145	"max_model_len": 512,
146	},
147	}
148
149	# These checkpoints fail during XPU graph capture, so run them eagerly there.
150	XPU_EAGER_ONLY_MODELS = {
151	"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
152	"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound",
153	"Intel/Qwen3-8B-w2g64-for-ut",
154	"INCModel/Qwen3-30B-A3B-12L-W4A16-test",
155	}
156
157
158	def _runner_kwargs(model: str) -> dict[str, Any]:
159	kwargs = dict(MODEL_RUNNER_KWARGS.get(model, {}))
160	if current_platform.is_xpu() and model in XPU_EAGER_ONLY_MODELS:
161	kwargs["enforce_eager"] = True
162	return kwargs
163
164
165	@pytest.mark.skipif(
166	not (
167	current_platform.is_cpu()
168	or current_platform.is_xpu()
169	or current_platform.is_cuda()
170	),
171	reason="Only supports CPU/XPU/CUDA backend.",
172	)
173	@pytest.mark.parametrize("model", MODELS + QWEN3_AUTOROUND_MODELS)
174	def test_auto_round_model(vllm_runner, model):
175	with vllm_runner(model, **_runner_kwargs(model)) as llm:
176	output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
177
178	assert output
179	print(output[0][1])
180
181
182	# ---------------------------------------------------------------------------
183	# Unit tests for INCConfig and related classes
184	# ---------------------------------------------------------------------------
185
186
187	class DummyLayer:
188	pass
189
190
191	class DummyFusedMoE:
192	pass
193
194
195	def make_config(**overrides) -> INCConfig:
196	kwargs: dict[str, Any] = {
197	"weight_bits": 4,
198	"group_size": 128,
199	"sym": True,
200	"packing_format": "auto_round:auto_gptq",
201	"block_name_to_quantize": None,
202	"extra_config": None,
203	"data_type": "int",
204	"backend": "auto",
205	}
206	kwargs.update(overrides)
207	return INCConfig(**kwargs)
208
209
210	def make_layer_config(**overrides) -> INCLayerConfig:
211	kwargs: dict[str, Any] = {
212	"bits": 4,
213	"group_size": 128,
214	"sym": True,
215	"packing_format": "auto_round:auto_gptq",
216	"backend": "auto",
217	"data_type": "int",
218	"quantized": True,
219	}
220	kwargs.update(overrides)
221	return INCLayerConfig(**kwargs)
222
223
224	def make_block_fp8_raw_config(**overrides) -> dict[str, object]:
225	kwargs = {
226	"bits": 8,
227	"group_size": [128, 128],
228	"sym": True,
229	"packing_format": "auto_round:fp8",
230	"quant_method": "auto-round",
231	"data_type": "fp",
232	"act_bits": 8,
233	"act_data_type": "fp",
234	"act_group_size": 128,
235	"act_sym": True,
236	"act_dynamic": True,
237	"activation_scheme": "dynamic",
238	"fmt": "e4m3",
239	"weight_block_size": [128, 128],
240	}
241	kwargs.update(overrides)
242	return kwargs
243
244
245	def make_qwen3_autoround_config(kind: str) -> INCConfig:
246	configs = {
247	"qwen3_1p7b_mxfp4": {
248	"quant_method": "auto-round",
249	"bits": 4,
250	"group_size": 32,
251	"sym": True,
252	"packing_format": "auto_round:llm_compressor",
253	"data_type": "mx_fp",
254	"extra_config": {
255	"model.layers.0.self_attn.q_proj": {
256	"bits": 16,
257	"data_type": "float",
258	},
259	},
260	},
261	"qwen3_30b_a3b_w4a16": {
262	"quant_method": "auto-round",
263	"bits": 4,
264	"group_size": 32,
265	"sym": True,
266	"packing_format": "auto_round:auto_gptq",
267	"data_type": "int",
268	"extra_config": {
269	"model.layers.0.mlp.gate": {
270	"bits": 16,
271	"data_type": "float",
272	},
273	},
274	},
275	"qwen3_30b_a3b_mxfp4": {
276	"quant_method": "auto-round",
277	"bits": 4,
278	"group_size": 32,
279	"sym": True,
280	"packing_format": "auto_round:llm_compressor",
281	"data_type": "mx_fp",
282	"act_bits": 4,
283	"act_group_size": 32,
284	"act_data_type": "mx_fp",
285	"extra_config": {
286	"model.layers.0.mlp.gate": {
287	"bits": 16,
288	"data_type": "float",
289	},
290	"model.layers.0.self_attn.q_proj": {
291	"bits": 16,
292	"data_type": "float",
293	},
294	},
295	},
296	"qwen3_30b_a3b_mxfp8": {
297	"quant_method": "auto-round",
298	"bits": 8,
299	"group_size": 32,
300	"sym": True,
301	"packing_format": "auto_round:llm_compressor",
302	"data_type": "mx_fp",
303	"act_bits": 8,
304	"act_group_size": 32,
305	"act_data_type": "mx_fp",
306	"extra_config": {
307	"model.layers.0.mlp.gate": {
308	"bits": 16,
309	"data_type": "float",
310	},
311	},
312	},
313	}
314	try:
315	config = configs[kind]
316	except KeyError as err:
317	raise AssertionError(f"unknown qwen3 autoround config: {kind}") from err
318	return INCConfig.from_config(config)
319
320
321	def test_inc_config_parser_exact_match() -> None:
322	config = make_config(
323	extra_config={
324	"layers.0.self_attn.q_proj": {
325	"bits": 8,
326	"group_size": 64,
327	"sym": False,
328	}
329	}
330	)
331
332	layer_config = config.config_parser.resolve(
333	DummyLayer(), "layers.0.self_attn.q_proj"
334	)
335
336	assert layer_config.bits == 8
337	assert layer_config.group_size == 64
338	assert layer_config.sym is False
339	assert layer_config.quantized is True
340
341
342	def test_inc_model_prefix_early_exit() -> None:
343	"""extra_config keys with model. prefix trigger early unquantized return."""
344	config = make_config(
345	extra_config={
346	"model.layers.1.mlp.gate_proj": {
347	"bits": 16,
348	},
349	}
350	)
351	layer = object.__new__(LinearBase)
352
353	# get_quant_method checks model. prefix for unquantized early-exit
354	result = config.get_quant_method(layer, "layers.1.mlp.gate_proj")
355	assert isinstance(result, UnquantizedLinearMethod)
356
357
358	def test_inc_get_quant_method_unquantized_parallel_lm_head_returns_unquantized() -> (
359	None
360	):
361	"""ParallelLMHead with bits >= 16 returns UnquantizedLinearMethod via the
362	early-exit path (parallel_lm_head is explicitly handled alongside LinearBase).
363	"""
364	config = make_config(extra_config={"lm_head": {"bits": 16}})
365	layer = object.__new__(ParallelLMHead)
366
367	method = config.get_quant_method(layer, "lm_head")
368
369	assert isinstance(method, UnquantizedLinearMethod)
370
371
372	def test_inc_get_quant_method_unquantized_unknown_layer_returns_none() -> None:
373	"""Regression test: non-Linear/non-MoE layers with bits >= 16 must return
374	None from the early-exit path instead of being misclassified as linear.
375	"""
376	config = make_config(extra_config={"layer": {"bits": 16}})
377
378	method = config.get_quant_method(DummyLayer(), "layer")
379
380	assert method is None
381
382
383	@pytest.mark.parametrize(
384	"layer_factory",
385	[
386	pytest.param(
387	lambda: object.__new__(LinearBase),
388	id="LinearBase",
389	),
390	pytest.param(
391	lambda: object.__new__(ParallelLMHead),
392	id="ParallelLMHead",
393	),
394	],
395	)
396	def test_inc_get_quant_method_unquantized_layer_with_model_prefix(
397	layer_factory,
398	) -> None:
399	"""``model.``-prefixed extra_config entries trigger early exit for both
400	LinearBase and ParallelLMHead layers."""
401	config = make_config(extra_config={"model.layer": {"bits": 16}})
402	layer = layer_factory()
403
404	method = config.get_quant_method(layer, "layer")
405
406	assert isinstance(method, UnquantizedLinearMethod)
407
408
409	def test_inc_get_quant_method_unquantized_routed_experts_with_model_prefix(
410	monkeypatch,
411	) -> None:
412	"""``model.``-prefixed extra_config entries trigger early exit for
413	RoutedExperts layers, returning UnquantizedFusedMoEMethod."""
414
415	class DummyUnquantizedFusedMoEMethod:
416	def __init__(self, moe_config) -> None:
417	self.moe_config = moe_config
418
419	monkeypatch.setattr(
420	"vllm.model_executor.layers.quantization.inc.inc.UnquantizedFusedMoEMethod",
421	DummyUnquantizedFusedMoEMethod,
422	)
423
424	config = make_config(extra_config={"model.moe_layer": {"bits": 16}})
425	layer = object.__new__(RoutedExperts)
426	layer.moe_config = None
427
428	method = config.get_quant_method(layer, "moe_layer")
429
430	assert isinstance(method, DummyUnquantizedFusedMoEMethod)
431	assert method.moe_config is None
432
433
434	def test_inc_get_quant_method_unknown_layer_with_model_prefix_returns_none() -> None:
435	"""``model.``-prefixed extra_config entries return None for unhandled
436	layer types (non-Linear, non-MoE)."""
437	config = make_config(extra_config={"model.unknown": {"bits": 16}})
438
439	method = config.get_quant_method(DummyLayer(), "unknown")
440
441	assert method is None
442
443
444	def test_inc_config_parser_regex_match() -> None:
445	config = make_config(
446	extra_config={
447	r"layers\.\d+\.self_attn\.(q\|k\|v)_proj": {
448	"bits": 8,
449	"group_size": 64,
450	"sym": False,
451	}
452	}
453	)
454
455	layer_config = config.config_parser.resolve(
456	DummyLayer(), "layers.3.self_attn.q_proj"
457	)
458
459	assert layer_config.bits == 8
460	assert layer_config.group_size == 64
461	assert layer_config.sym is False
462
463
464	def test_inc_config_parser_invalid_regex_ignored() -> None:
465	config = make_config(
466	extra_config={
467	"[invalid": {
468	"bits": 8,
469	"group_size": 64,
470	"sym": False,
471	}
472	}
473	)
474
475	layer_config = config.config_parser.resolve(
476	DummyLayer(), "layers.0.self_attn.q_proj"
477	)
478
479	assert layer_config.bits == 4
480	assert layer_config.group_size == 128
481	assert layer_config.sym is True
482
483
484	def test_inc_config_parser_block_name_to_quantize_marks_unquantized() -> None:
485	config = make_config(block_name_to_quantize=["layers.1"])
486
487	layer_config = config.config_parser.resolve(
488	DummyLayer(), "layers.0.self_attn.q_proj"
489	)
490
491	assert layer_config.bits == 16
492	assert layer_config.group_size == -1
493	assert layer_config.sym is True
494	assert layer_config.quantized is False
495
496
497	def test_inc_config_parser_parallel_lm_head_defaults_to_unquantized() -> None:
498	layer = object.__new__(ParallelLMHead)
499	config = make_config()
500
501	layer_config = config.config_parser.resolve(layer, "lm_head")
502
503	assert layer_config.quantized is False
504	assert layer_config.bits == 16
505
506
507	def test_inc_config_parser_suffix_match_for_lm_head() -> None:
508	"""Short extra_config key should match fully-qualified lm_head layer name."""
509	layer = object.__new__(ParallelLMHead)
510	config = make_config(
511	extra_config={
512	"lm_head": {
513	"bits": 4,
514	"group_size": 128,
515	"sym": True,
516	}
517	}
518	)
519
520	layer_config = config.config_parser.resolve(layer, "model.language_model.lm_head")
521
522	assert layer_config.quantized is True
523	assert layer_config.bits == 4
524	assert layer_config.group_size == 128
525	assert layer_config.sym is True
526
527
528	def test_inc_config_parser_fused_moe_requires_consistent_configs() -> None:
529	config = make_config(
530	extra_config={
531	"layers.0.block_sparse_moe.experts.0.w1": {
532	"bits": 4,
533	"group_size": 128,
534	"sym": True,
535	},
536	"layers.0.block_sparse_moe.experts.0.w2": {
537	"bits": 8,
538	"group_size": 128,
539	"sym": True,
540	},
541	}
542	)
543
544	with pytest.raises(ValueError, match="requires consistent quant config"):
545	config.config_parser.resolve(DummyFusedMoE(), "layers.0.block_sparse_moe")
546
547
548	def test_inc_config_parser_fused_module_requires_consistent_configs() -> None:
549	config = make_config(
550	extra_config={
551	"layers.0.self_attn.q_proj": {
552	"bits": 4,
553	"group_size": 128,
554	"sym": True,
555	},
556	"layers.0.self_attn.k_proj": {
557	"bits": 8,
558	"group_size": 128,
559	"sym": True,
560	},
561	"layers.0.self_attn.v_proj": {
562	"bits": 4,
563	"group_size": 128,
564	"sym": True,
565	},
566	}
567	)
568	config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
569
570	with pytest.raises(ValueError, match="requires consistent quant config"):
571	config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj")
572
573
574	def test_inc_mxfp8() -> None:
575	config = make_config(
576	weight_bits=8,
577	group_size=32,
578	sym=True,
579	packing_format="auto_round:llm_compressor",
580	data_type="mx_fp",
581	)
582
583	assert config.weight_bits == 8
584	assert config.group_size == 32
585	assert config.data_type == "mx_fp"
586	assert config.packing_format == "auto_round:llm_compressor"
587
588
589	def test_inc_config_rejects_invalid_mxfp8_activation_config() -> None:
590	with pytest.raises(AssertionError, match="act_dynamic=True"):
591	INCConfig.from_config(
592	{
593	"bits": 8,
594	"group_size": 32,
595	"sym": True,
596	"packing_format": "auto_round:llm_compressor",
597	"data_type": "mx_fp",
598	"act_bits": 8,
599	"act_data_type": "mx_fp",
600	"act_group_size": 32,
601	"act_sym": True,
602	"act_dynamic": False,
603	}
604	)
605
606
607	def test_inc_layer_config_mx_fp_helpers() -> None:
608	layer_config = INCLayerConfig(
609	bits=4,
610	group_size=32,
611	sym=True,
612	packing_format="",
613	backend="",
614	data_type="mx_fp",
615	quantized=True,
616	)
617
618	assert layer_config.is_mxfp4 is True
619	assert layer_config.is_mxfp8 is False
620
621
622	def test_inc_config_accepts_block_fp8() -> None:
623	config = INCConfig.from_config(make_block_fp8_raw_config())
624
625	assert config.weight_bits == 8
626	assert config.group_size == (128, 128)
627	assert config.sym is True
628	assert config.data_type == "fp"
629	assert config.packing_format == "auto_round:fp8"
630
631
632	def test_inc_config_rejects_invalid_block_fp8_activation_config() -> None:
633	with pytest.raises(AssertionError, match="act_dynamic=True"):
634	INCConfig.from_config(make_block_fp8_raw_config(act_dynamic=False))
635
636
637	def test_inc_layer_config_fp8_block_helper() -> None:
638	layer_config = INCLayerConfig(
639	bits=8,
640	group_size=(128, 128),
641	sym=True,
642	packing_format="auto_round:fp8",
643	backend="auto",
644	data_type="fp",
645	quantized=True,
646	)
647
648	assert layer_config.is_fp8_block is True
649	assert layer_config.is_wna16_int is False
650
651
652	def test_inc_resolve_scheme_selects_fp8_block() -> None:
653	layer_config = INCLayerConfig(
654	bits=8,
655	group_size=(128, 128),
656	sym=True,
657	packing_format="auto_round:fp8",
658	backend="auto",
659	data_type="fp",
660	quantized=True,
661	)
662
663	scheme = resolve_scheme(layer_config)
664
665	assert isinstance(scheme, INCFp8Scheme)
666
667
668	def test_inc_resolve_scheme_selects_wna16() -> None:
669	layer_config = INCLayerConfig(
670	bits=4,
671	group_size=128,
672	sym=True,
673	packing_format="auto_round:auto_gptq",
674	backend="auto",
675	data_type="int",
676	quantized=True,
677	)
678
679	scheme = resolve_scheme(layer_config)
680
681	assert isinstance(scheme, INCWna16Scheme)
682
683
684	@pytest.mark.parametrize("bits", [2, 3])
685	@pytest.mark.parametrize(
686	"packing_format", ["auto_round:auto_gptq", "auto_round:auto_awq"]
687	)
688	def test_wna16_cuda_low_bit_linear_routes_to_humming(
689	monkeypatch, bits, packing_format
690	) -> None:
691	expected_method = object()
692	captured = {}
693
694	monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
695	monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
696	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
697	monkeypatch.setattr(
698	"vllm.model_executor.layers.quantization.inc.schemes."
699	"inc_wna16_scheme._build_humming_linear_method",
700	lambda layer_config: captured.update({"layer_config": layer_config})
701	or expected_method,
702	)
703
704	layer_config = make_layer_config(bits=bits, packing_format=packing_format)
705	method = INCWna16Scheme().get_linear_method(
706	make_config(), object(), "model.layers.0.mlp.down_proj", layer_config
707	)
708
709	assert method is expected_method
710	assert captured["layer_config"] is layer_config
711
712
713	@pytest.mark.parametrize("bits", [2, 3])
714	def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
715	expected_method = object()
716	captured = {}
717
718	class DummyMoeConfig:
719	pass
720
721	monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
722	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
723	monkeypatch.setattr(
724	"vllm.model_executor.layers.quantization.inc.schemes."
725	"inc_wna16_scheme._build_humming_moe_method",
726	lambda layer, layer_config: captured.update(
727	{"layer": layer, "layer_config": layer_config}
728	)
729	or expected_method,
730	)
731
732	layer = object.__new__(RoutedExperts)
733	layer.moe_config = DummyMoeConfig()
734	layer_config = make_layer_config(bits=bits)
735	method = INCWna16Scheme().get_moe_method(
736	make_config(), layer, "model.layers.0.mlp", layer_config
737	)
738
739	assert method is expected_method
740	assert captured["layer"] is layer
741	assert captured["layer_config"] is layer_config
742
743
744	@pytest.mark.skipif(
745	not current_platform.is_cuda(),
746	reason="This test only exercises the CUDA Marlin path.",
747	)
748	@pytest.mark.parametrize("bits", [4, 8])
749	def test_wna16_cuda_high_bit_skips_humming(monkeypatch, bits) -> None:
750	"""4/8-bit int stays on the Marlin/GPTQ/AWQ path even on CUDA so a single
751	model can mix high-bit Marlin and low-bit humming layers."""
752	called = {"humming": False}
753
754	monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
755	monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
756	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
757	monkeypatch.setattr(
758	"vllm.model_executor.layers.quantization.inc.schemes."
759	"inc_wna16_scheme._build_humming_linear_method",
760	lambda layer_config: called.update({"humming": True}),
761	)
762
763	method = INCWna16Scheme().get_linear_method(
764	make_config(),
765	object(),
766	"model.layers.0.mlp.down_proj",
767	make_layer_config(bits=bits),
768	)
769
770	assert called["humming"] is False
771	assert isinstance(method, INCLinearMethod)
772
773
774	def test_inc_config_accepts_mxfp_family_llm_compressor() -> None:
775	config = INCConfig.from_config(
776	{
777	"quant_method": "auto-round",
778	"bits": 4,
779	"group_size": 32,
780	"sym": True,
781	"packing_format": "auto_round:llm_compressor",
782	"data_type": "mx_fp4e2m1",
783	}
784	)
785
786	layer_config = config.config_parser.resolve(
787	DummyLayer(), "model.layers.0.mlp.down_proj"
788	)
789
790	assert config.sym is True
791	assert layer_config.is_mxfp4 is True
792	assert isinstance(resolve_scheme(layer_config), INCMxfp4Scheme)
793
794
795	def test_qwen3_1p7b_mxfp4_autoround_uses_mxfp4_linear_scheme(
796	monkeypatch,
797	) -> None:
798	class DummyKernel:
799	pass
800
801	monkeypatch.setattr(
802	"vllm.model_executor.layers.quantization.inc.schemes."
803	"inc_mxfp4_linear.init_mxfp4_linear_kernel",
804	lambda **kwargs: DummyKernel(),
805	)
806
807	from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_linear import ( # noqa: E501
808	INCMxfp4LinearMethod,
809	)
810
811	config = make_qwen3_autoround_config("qwen3_1p7b_mxfp4")
812
813	assert (
814	INCConfig.override_quantization_method(
815	{"quant_method": "auto-round"}, user_quant=None
816	)
817	== "inc"
818	)
819	ignored_method = config.get_quant_method(
820	object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
821	)
822	layer_config = config.config_parser.resolve(
823	DummyLayer(), "model.layers.0.mlp.gate_proj"
824	)
825	method = INCMxfp4Scheme().get_linear_method(
826	config,
827	object.__new__(LinearBase),
828	"model.layers.0.mlp.gate_proj",
829	layer_config,
830	)
831
832	assert isinstance(ignored_method, UnquantizedLinearMethod)
833	assert layer_config.bits == 4
834	assert layer_config.group_size == 32
835	assert layer_config.is_mxfp4 is True
836	assert isinstance(resolve_scheme(layer_config), INCMxfp4Scheme)
837	assert isinstance(method, INCLinearMethod)
838	assert isinstance(method.scheme, INCMxfp4LinearMethod)
839	assert isinstance(method.scheme.kernel, DummyKernel)
840
841
842	def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe(
843	monkeypatch,
844	) -> None:
845	captured = {}
846	expected_method = object()
847
848	class DummyMoeConfig:
849	pass
850
851	def fake_resolve_gptq_moe(layer, layer_config):
852	captured["layer"] = layer
853	captured["layer_config"] = layer_config
854	return expected_method
855
856	monkeypatch.setattr(
857	"vllm.model_executor.layers.quantization.inc.schemes."
858	"inc_wna16_scheme._resolve_gptq_moe",
859	fake_resolve_gptq_moe,
860	)
861
862	config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16")
863	layer = object.__new__(RoutedExperts)
864	layer.moe_config = DummyMoeConfig()
865
866	method = config.get_quant_method(layer, "model.layers.0.mlp")
867
868	assert method is expected_method
869	assert captured["layer"] is layer
870	assert captured["layer_config"].bits == 4
871	assert captured["layer_config"].group_size == 32
872	assert captured["layer_config"].is_gptq is True
873	assert captured["layer_config"].is_wna16_int is True
874
875
876	def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
877	monkeypatch,
878	) -> None:
879	class DummyMoeConfig:
880	pass
881
882	class DummyMxfp4MoEMethod:
883	def __init__(self, moe_config) -> None:
884	self.moe_config = moe_config
885
886	monkeypatch.setattr(
887	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe.INCMxfp4MoEMethod",
888	DummyMxfp4MoEMethod,
889	)
890
891	config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4")
892	layer = object.__new__(RoutedExperts)
893	layer.moe_config = DummyMoeConfig()
894
895	ignored_method = config.get_quant_method(
896	object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
897	)
898	method = config.get_quant_method(layer, "model.layers.0.mlp")
899	layer_config = config.config_parser.resolve(DummyLayer(), "model.layers.0.mlp")
900
901	assert isinstance(ignored_method, UnquantizedLinearMethod)
902	assert layer_config.bits == 4
903	assert layer_config.group_size == 32
904	assert layer_config.is_mxfp4 is True
905	assert isinstance(resolve_scheme(layer_config), INCMxfp4Scheme)
906	assert isinstance(method, DummyMxfp4MoEMethod)
907	assert method.moe_config is layer.moe_config
908
909
910	def test_qwen3_30b_a3b_mxfp8_autoround_routes_to_mxfp8_moe(
911	monkeypatch,
912	) -> None:
913	expected_backend = object()
914	expected_experts_cls = object()
915	captured = {}
916
917	def fake_select_mxfp8_moe_backend(*, config):
918	captured["moe_config"] = config
919	return expected_backend, expected_experts_cls
920
921	monkeypatch.setattr(
922	"vllm.model_executor.layers.quantization.inc.schemes."
923	"inc_mxfp8_moe.select_mxfp8_moe_backend",
924	fake_select_mxfp8_moe_backend,
925	)
926
927	config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp8")
928	layer = object.__new__(RoutedExperts)
929	layer.moe_config = cast(Any, "moe-config")
930
931	linear_method = config.get_quant_method(
932	object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
933	)
934	moe_method = config.get_quant_method(layer, "model.layers.0.mlp")
935	gate_method = config.get_quant_method(
936	object.__new__(LinearBase), "model.layers.0.mlp.gate"
937	)
938
939	from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
940	INCMxfp8MoEMethod,
941	)
942
943	assert isinstance(linear_method, INCLinearMethod)
944	assert isinstance(linear_method.scheme, INCMxfp8LinearScheme)
945	assert isinstance(moe_method, INCMxfp8MoEMethod)
946	assert moe_method.mxfp8_backend is expected_backend
947	assert moe_method.experts_cls is expected_experts_cls
948	assert captured["moe_config"] == layer.moe_config
949	assert isinstance(gate_method, UnquantizedLinearMethod)
950
951
952	def test_inc_mxfp8_moe_method_registers_block_scaled_weights(monkeypatch) -> None:
953	expected_backend = object()
954	expected_experts_cls = object()
955
956	monkeypatch.setattr(
957	"vllm.model_executor.layers.quantization.inc.schemes."
958	"inc_mxfp8_moe.select_mxfp8_moe_backend",
959	lambda *, config: (expected_backend, expected_experts_cls),
960	)
961	monkeypatch.setattr(
962	"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
963	lambda: 0,
964	)
965	monkeypatch.setattr(
966	"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
967	lambda: 1,
968	)
969
970	from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
971	INCMxfp8MoEMethod,
972	)
973
974	class DummyMoeConfig:
975	is_act_and_mul = True
976
977	method = INCMxfp8MoEMethod(cast(Any, DummyMoeConfig()))
978	layer = torch.nn.Module()
979	layer.hidden_size = 64
980	layer.intermediate_size_per_partition = 32
981
982	method.create_weights(
983	layer,
984	num_experts=2,
985	hidden_size=64,
986	intermediate_size_per_partition=32,
987	params_dtype=torch.bfloat16,
988	)
989
990	assert method.mxfp8_backend is expected_backend
991	assert method.experts_cls is expected_experts_cls
992	assert method.weight_block_size == [1, 32]
993	assert layer.w13_weight.shape == (2, 64, 64)
994	assert layer.w2_weight.shape == (2, 64, 32)
995	assert layer.w13_weight.dtype is torch.float8_e4m3fn
996	assert layer.w2_weight.dtype is torch.float8_e4m3fn
997	assert layer.w13_weight_scale.shape == (2, 64, 2)
998	assert layer.w2_weight_scale.shape == (2, 64, 1)
999	assert layer.w13_weight_scale.dtype is torch.uint8
1000	assert layer.w2_weight_scale.dtype is torch.uint8
1001	assert layer.w13_weight_scale.quant_method == "block"
1002	assert layer.w2_weight_scale.quant_method == "block"
1003
1004	misaligned_layer = torch.nn.Module()
1005	misaligned_layer.hidden_size = 65
1006	misaligned_layer.intermediate_size_per_partition = 32
1007	with pytest.raises(ValueError, match="hidden_size divisible by 32"):
1008	method.create_weights(
1009	misaligned_layer,
1010	num_experts=2,
1011	hidden_size=65,
1012	intermediate_size_per_partition=32,
1013	params_dtype=torch.bfloat16,
1014	)
1015
1016
1017	def test_inc_mxfp4_linear_method_registers_and_processes_weights(
1018	monkeypatch,
1019	) -> None:
1020	captured = {}
1021
1022	class DummyKernel:
1023	def process_weights_after_loading(self, layer) -> None:
1024	captured["processed_layer"] = layer
1025
1026	monkeypatch.setattr(
1027	"vllm.model_executor.layers.quantization.inc.schemes."
1028	"inc_mxfp4_linear.init_mxfp4_linear_kernel",
1029	lambda **kwargs: DummyKernel(),
1030	)
1031	monkeypatch.setattr(
1032	"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
1033	lambda: 0,
1034	)
1035	monkeypatch.setattr(
1036	"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
1037	lambda: 1,
1038	)
1039
1040	from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_linear import ( # noqa: E501
1041	INCMxfp4LinearMethod,
1042	)
1043
1044	layer = torch.nn.Module()
1045	method = INCMxfp4LinearMethod(
1046	make_layer_config(group_size=32, data_type="mx_fp4e2m1")
1047	)
1048
1049	method.create_weights(
1050	layer,
1051	input_size_per_partition=64,
1052	output_partition_sizes=[16, 32],
1053	input_size=64,
1054	output_size=48,
1055	params_dtype=torch.bfloat16,
1056	)
1057
1058	assert layer.weight_packed.shape == (48, 32)
1059	assert layer.weight_packed.dtype is torch.uint8
1060	assert layer.weight_scale.shape == (48, 2)
1061	assert layer.weight_scale.dtype is torch.uint8
1062	assert layer.logical_widths == [16, 32]
1063	assert layer.input_size_per_partition == 64
1064	assert layer.output_size_per_partition == 48
1065
1066	packed_data = layer.weight_packed.data
1067	method.process_weights_after_loading(layer)
1068
1069	assert layer.weight.data.data_ptr() == packed_data.data_ptr()
1070	assert not hasattr(layer, "weight_packed")
1071	assert captured["processed_layer"] is layer
1072
1073
1074	@pytest.mark.parametrize(
1075	("moe_backend", "is_xpu", "mxfp4_backend"),
1076	[
1077	("auto", True, Mxfp4MoeBackend.XPU),
1078	("b12x", False, Mxfp4MoeBackend.B12X_MXFP4_MXFP8),
1079	],
1080	)
1081	def test_inc_mxfp4_moe_method_preserves_checkpoint_packing(
1082	monkeypatch,
1083	moe_backend: str,
1084	is_xpu: bool,
1085	mxfp4_backend: Mxfp4MoeBackend,
1086	) -> None:
1087	captured = {}
1088	expected_quant_config = object()
1089	expected_kernel = SimpleNamespace(
1090	fused_experts=SimpleNamespace(
1091	process_weights_after_loading=lambda layer: captured.update(
1092	{"processed_layer": layer}
1093	)
1094	)
1095	)
1096	expected_experts_cls = object()
1097
1098	monkeypatch.setattr(
1099	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
1100	"CutlassExpertsMxfp4._supports_current_device",
1101	lambda: False,
1102	)
1103	monkeypatch.setattr(current_platform, "is_xpu", lambda: is_xpu)
1104	monkeypatch.setattr(
1105	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
1106	"select_mxfp4_moe_backend",
1107	lambda moe: (mxfp4_backend, expected_experts_cls),
1108	)
1109	monkeypatch.setattr(
1110	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
1111	"prepare_moe_fp4_layer_for_marlin",
1112	lambda layer: pytest.fail("packed backends must not use Marlin packing"),
1113	)
1114	monkeypatch.setattr(
1115	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
1116	"make_mxfp4_moe_quant_config",
1117	lambda **kwargs: captured.update({"quant_config_kwargs": kwargs})
1118	or expected_quant_config,
1119	)
1120	monkeypatch.setattr(
1121	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
1122	"make_mxfp4_moe_kernel",
1123	lambda **kwargs: captured.update({"kernel_kwargs": kwargs}) or expected_kernel,
1124	)
1125
1126	from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe import (
1127	INCMxfp4MoEMethod,
1128	)
1129
1130	expected_moe_config = SimpleNamespace(
1131	w13_num_shards=2,
1132	moe_backend=moe_backend,
1133	)
1134	method = INCMxfp4MoEMethod(moe=cast(Any, expected_moe_config))
1135	layer = torch.nn.Module()
1136	layer._expert_routing_tables = lambda: "routing-tables"
1137
1138	method.create_weights(
1139	layer,
1140	num_experts=2,
1141	hidden_size=64,
1142	intermediate_size_per_partition=32,
1143	params_dtype=torch.bfloat16,
1144	)
1145
1146	assert method.experts_cls is expected_experts_cls
1147	assert layer.w13_weight_packed.shape == (2, 64, 32)
1148	assert layer.w2_weight_packed.shape == (2, 64, 16)
1149	assert layer.w13_weight_scale.shape == (2, 64, 2)
1150	assert layer.w2_weight_scale.shape == (2, 64, 1)
1151
1152	w13_packed_data = layer.w13_weight_packed.data
1153	w2_packed_data = layer.w2_weight_packed.data
1154	method.process_weights_after_loading(layer)
1155
1156	assert layer.w13_weight.data.data_ptr() == w13_packed_data.data_ptr()
1157	assert layer.w2_weight.data.data_ptr() == w2_packed_data.data_ptr()
1158	assert not hasattr(layer, "w13_weight_packed")
1159	assert not hasattr(layer, "w2_weight_packed")
1160	assert captured["quant_config_kwargs"]["w1_scale"] is layer.w13_weight_scale
1161	assert captured["quant_config_kwargs"]["w2_scale"] is layer.w2_weight_scale
1162	assert captured["kernel_kwargs"]["moe_quant_config"] is expected_quant_config
1163	assert captured["kernel_kwargs"]["moe_config"] is expected_moe_config
1164	assert captured["kernel_kwargs"]["experts_cls"] is expected_experts_cls
1165	assert captured["kernel_kwargs"]["routing_tables"] == "routing-tables"
1166	assert captured["processed_layer"] is layer
1167	assert method.moe_kernel is expected_kernel
1168
1169
1170	def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None:
1171	captured = {}
1172	expected_method = object()
1173
1174	class DummyMoeConfig:
1175	pass
1176
1177	monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
1178	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
1179	monkeypatch.setattr(
1180	"vllm.model_executor.layers.quantization.inc.schemes."
1181	"inc_wna16_scheme._resolve_gptq_moe",
1182	lambda layer, layer_config: captured.update(
1183	{"layer": layer, "layer_config": layer_config}
1184	)
1185	or expected_method,
1186	)
1187
1188	layer = object.__new__(RoutedExperts)
1189	layer.moe_config = DummyMoeConfig()
1190	method = INCWna16Scheme().get_moe_method(
1191	make_config(),
1192	layer,
1193	"model.layers.0.mlp",
1194	make_layer_config(group_size=32),
1195	)
1196
1197	assert method is expected_method
1198	assert captured["layer"] is layer
1199	assert captured["layer_config"].is_gptq is True
1200
1201
1202	def test_inc_resolve_scheme_selects_mxfp8() -> None:
1203	layer_config = INCLayerConfig(
1204	bits=8,
1205	group_size=32,
1206	sym=True,
1207	packing_format="auto_round:llm_compressor",
1208	backend="auto",
1209	data_type="mx_fp",
1210	quantized=True,
1211	)
1212
1213	scheme = resolve_scheme(layer_config)
1214
1215	assert isinstance(scheme, INCMxfp8Scheme)
1216
1217
1218	class DummyLinearScheme(INCLinearScheme):
1219	def __init__(self) -> None:
1220	self.calls: list[tuple] = []
1221
1222	@classmethod
1223	def get_min_capability(cls) -> int:
1224	return 0
1225
1226	def create_weights(self, args, *kwargs) -> None:
1227	self.calls.append(("create_weights", args, kwargs))
1228
1229	def process_weights_after_loading(self, layer) -> None:
1230	self.calls.append(("process_weights_after_loading", layer))
1231
1232	def apply_weights(self, layer, x, bias=None):
1233	self.calls.append(("apply_weights", layer, x, bias))
1234	return "applied"
1235
1236
1237	def test_inc_mxfp8_linear_scheme_delegates_to_kernel(monkeypatch) -> None:
1238	class DummyKernel:
1239	def __init__(self) -> None:
1240	self.calls: list[tuple] = []
1241
1242	def process_weights_after_loading(self, layer) -> None:
1243	self.calls.append(("process", layer))
1244
1245	def apply_weights(self, layer, x, bias=None):
1246	self.calls.append(("apply", layer, x, bias))
1247	return "applied"
1248
1249	kernel = DummyKernel()
1250	monkeypatch.setattr(
1251	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
1252	lambda weight_shape: kernel,
1253	)
1254	monkeypatch.setattr(
1255	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.ModelWeightParameter",
1256	lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
1257	)
1258	monkeypatch.setattr(
1259	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.GroupQuantScaleParameter",
1260	lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
1261	)
1262
1263	scheme = INCMxfp8LinearScheme()
1264	layer = torch.nn.Module()
1265
1266	scheme.create_weights(
1267	layer=layer,
1268	input_size_per_partition=64,
1269	output_partition_sizes=[48, 16],
1270	input_size=64,
1271	output_size=64,
1272	params_dtype=torch.bfloat16,
1273	weight_loader=lambda args, *kwargs: None,
1274	)
1275
1276	assert layer.weight.shape == (64, 64)
1277	assert layer.weight.dtype == torch.float8_e4m3fn
1278	assert layer.weight_scale.shape == (64, 2)
1279	assert layer.weight_scale.dtype == torch.uint8
1280
1281	scheme.process_weights_after_loading(layer)
1282	result = scheme.apply_weights(layer, torch.randn(1, 64), None)
1283
1284	assert result == "applied"
1285	assert [call[0] for call in kernel.calls] == ["process", "apply"]
1286
1287
1288	def test_inc_mxfp8_linear_scheme_requires_block_32_input(monkeypatch) -> None:
1289	monkeypatch.setattr(
1290	"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
1291	lambda weight_shape: object(),
1292	)
1293	scheme = INCMxfp8LinearScheme()
1294
1295	with pytest.raises(ValueError, match="divisible by 32"):
1296	scheme.create_weights(
1297	layer=torch.nn.Module(),
1298	input_size_per_partition=48,
1299	output_partition_sizes=[32],
1300	input_size=48,
1301	output_size=32,
1302	params_dtype=torch.bfloat16,
1303	weight_loader=lambda args, *kwargs: None,
1304	)
1305
1306
1307	def test_inc_linear_method_delegates() -> None:
1308	scheme = DummyLinearScheme()
1309	method = INCLinearMethod(scheme)
1310	layer = DummyLayer()
1311
1312	method.create_weights(
1313	layer,
1314	input_size_per_partition=1,
1315	output_partition_sizes=[2],
1316	input_size=1,
1317	output_size=2,
1318	params_dtype=None,
1319	)
1320	method.process_weights_after_loading(layer)
1321	result = method.apply(layer, "x", "b")
1322
1323	assert result == "applied"
1324	assert [call[0] for call in scheme.calls] == [
1325	"create_weights",
1326	"process_weights_after_loading",
1327	"apply_weights",
1328	]
1329
1330
1331	def test_inc_fp8_linear_scheme_delegates_to_fp8_linear_method(
1332	monkeypatch,
1333	) -> None:
1334	captured = {}
1335
1336	class DummyMethod:
1337	def __init__(self, quant_config) -> None:
1338	captured["quant_config"] = quant_config
1339	captured["method"] = self
1340	self.calls: list[tuple] = []
1341	self.marlin_input_dtype = None
1342
1343	def create_weights(
1344	self,
1345	layer,
1346	input_size_per_partition,
1347	output_partition_sizes,
1348	input_size,
1349	output_size,
1350	params_dtype,
1351	**extra_weight_attrs,
1352	) -> None:
1353	self.calls.append(
1354	(
1355	"create_weights",
1356	layer,
1357	input_size_per_partition,
1358	output_partition_sizes,
1359	input_size,
1360	output_size,
1361	params_dtype,
1362	extra_weight_attrs,
1363	)
1364	)
1365
1366	def process_weights_after_loading(self, layer) -> None:
1367	self.calls.append(("process_weights_after_loading", layer))
1368
1369	def apply(self, layer, x, bias=None):
1370	self.calls.append(("apply", layer, x, bias))
1371	return "applied"
1372
1373	monkeypatch.setattr(
1374	"vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
1375	"Fp8LinearMethod",
1376	DummyMethod,
1377	)
1378	monkeypatch.setattr(
1379	"vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
1380	"get_marlin_input_dtype",
1381	lambda _prefix: "fp8-marlin",
1382	)
1383
1384	scheme = INCFp8LinearScheme(
1385	prefix="layers.0.mlp.down_proj",
1386	weight_block_size=(128, 128),
1387	)
1388	layer = torch.nn.Module()
1389
1390	scheme.create_weights(
1391	layer=layer,
1392	input_size_per_partition=128,
1393	output_partition_sizes=[64, 64],
1394	input_size=128,
1395	output_size=128,
1396	params_dtype=torch.bfloat16,
1397	weight_loader=lambda args, *kwargs: None,
1398	)
1399	scheme.process_weights_after_loading(layer)
1400	result = scheme.apply_weights(layer, torch.randn(1, 128), None)
1401
1402	assert captured["quant_config"].is_checkpoint_fp8_serialized is True
1403	assert captured["quant_config"].activation_scheme == "dynamic"
1404	assert captured["quant_config"].weight_block_size == [128, 128]
1405	assert captured["method"].marlin_input_dtype == "fp8-marlin"
1406	assert result == "applied"
1407	assert [call[0] for call in captured["method"].calls] == [
1408	"create_weights",
1409	"process_weights_after_loading",
1410	"apply",
1411	]
1412
1413
1414	def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
1415	class DummyQuantLinear:
1416	pass
1417
1418	monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
1419	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
1420	monkeypatch.setattr(
1421	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
1422	lambda: (True, None, object(), DummyQuantLinear),
1423	)
1424
1425	method = INCWna16Scheme().get_linear_method(
1426	make_config(),
1427	object(),
1428	"layer",
1429	make_layer_config(),
1430	)
1431
1432	assert isinstance(method, INCLinearMethod)
1433	assert isinstance(method.scheme, INCARKLinearMethod)
1434
1435
1436	def test_inc_config_from_config_accepts_xpu_int2() -> None:
1437	def _make_int2_raw_config(**overrides) -> dict[str, object]:
1438	kwargs = {
1439	"bits": 2,
1440	"group_size": 64,
1441	"sym": True,
1442	"data_type": "int",
1443	"quant_method": "auto-round",
1444	}
1445	kwargs.update(overrides)
1446
1447	return kwargs
1448
1449	config = INCConfig.from_config(_make_int2_raw_config())
1450
1451	assert config.weight_bits == 2
1452	assert config.group_size == 64
1453	assert config.sym is True
1454	assert config.data_type == "int"
1455	assert config.packing_format == "auto_round:auto_gptq"
1456	assert config.backend == "auto"
1457
1458
1459	def test_wna16_xpu_int2_prefers_ark_when_available(monkeypatch) -> None:
1460	class DummyQuantLinear:
1461	pass
1462
1463	monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
1464	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
1465	monkeypatch.setattr(
1466	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
1467	lambda: (True, None, object(), DummyQuantLinear),
1468	)
1469
1470	method = INCWna16Scheme().get_linear_method(
1471	make_config(weight_bits=2, group_size=64),
1472	object(),
1473	"layer",
1474	make_layer_config(bits=2, group_size=64),
1475	)
1476
1477	assert isinstance(method, INCLinearMethod)
1478	assert isinstance(method.scheme, INCARKLinearMethod)
1479
1480
1481	def test_wna16_xpu_int2_requires_ark_when_unavailable(monkeypatch) -> None:
1482	monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
1483	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
1484	monkeypatch.setattr(
1485	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
1486	lambda: (False, "missing", None, None),
1487	)
1488
1489	with pytest.raises(
1490	NotImplementedError,
1491	match="INC int2 on XPU requires the ARK backend",
1492	):
1493	INCWna16Scheme().get_linear_method(
1494	make_config(weight_bits=2, group_size=64),
1495	object(),
1496	"layer",
1497	make_layer_config(bits=2, group_size=64),
1498	)
1499
1500
1501	def test_wna16_xpu_int2_unsupported_config_still_raises(monkeypatch) -> None:
1502	monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
1503	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
1504
1505	with pytest.raises(NotImplementedError, match="unsupported config"):
1506	INCWna16Scheme().get_linear_method(
1507	make_config(weight_bits=2, sym=False),
1508	object(),
1509	"layer",
1510	make_layer_config(bits=2, sym=False),
1511	)
1512
1513
1514	def test_inc_ark_linear_method_xpu_int2_create_weights(monkeypatch) -> None:
1515	monkeypatch.setattr(
1516	"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
1517	lambda: 0,
1518	)
1519	monkeypatch.setattr(
1520	"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
1521	lambda: 1,
1522	)
1523
1524	class DummyQuantLinear:
1525	pass
1526
1527	class DummyLayer(torch.nn.Module):
1528	pass
1529
1530	monkeypatch.setattr(
1531	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
1532	lambda: (True, None, object(), DummyQuantLinear),
1533	)
1534
1535	layer = DummyLayer()
1536	method = INCARKLinearMethod(make_layer_config(bits=2, group_size=64))
1537
1538	method.create_weights(
1539	layer=layer,
1540	input_size_per_partition=64,
1541	output_partition_sizes=[32, 32],
1542	input_size=64,
1543	output_size=64,
1544	params_dtype=torch.bfloat16,
1545	weight_loader=lambda args, *kwargs: None,
1546	)
1547
1548	assert method.pack_factor == 16
1549	assert layer.qweight.shape == (4, 64)
1550	assert layer.qweight.dtype == torch.int32
1551	assert layer.scales.shape == (1, 64)
1552	assert layer.scales.dtype == torch.bfloat16
1553	assert layer.qzeros.shape == (1, 4)
1554	assert layer.qzeros.dtype == torch.int32
1555	assert not hasattr(layer, "g_idx")
1556	assert layer.in_features == 64
1557	assert layer.out_features == 64
1558	assert layer.params_dtype == torch.bfloat16
1559
1560
1561	def test_wna16_xpu_falls_back_when_ark_unavailable(monkeypatch) -> None:
1562	monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
1563	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
1564	monkeypatch.setattr(
1565	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
1566	lambda: (False, "missing", None, None),
1567	)
1568
1569	method = INCWna16Scheme().get_linear_method(
1570	make_config(),
1571	object(),
1572	"layer",
1573	make_layer_config(),
1574	)
1575
1576	assert isinstance(method, INCLinearMethod)
1577	assert isinstance(method.scheme, INCXPULinearMethod)
1578
1579
1580	def test_wna16_cpu_gptq_prefers_ark_when_available(monkeypatch) -> None:
1581	class DummyQuantLinear:
1582	pass
1583
1584	monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
1585	monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
1586	monkeypatch.setattr(
1587	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
1588	lambda: (True, None, object(), DummyQuantLinear),
1589	)
1590
1591	method = INCWna16Scheme().get_linear_method(
1592	make_config(),
1593	object(),
1594	"layer",
1595	make_layer_config(),
1596	)
1597
1598	assert isinstance(method, INCLinearMethod)
1599	assert isinstance(method.scheme, INCARKLinearMethod)
1600
1601
1602	def test_wna16_cpu_gptq_raises_when_ark_and_marlin_unavailable(
1603	monkeypatch,
1604	) -> None:
1605	monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
1606	monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
1607	monkeypatch.setattr(
1608	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
1609	lambda: (False, "missing", None, None),
1610	)
1611	monkeypatch.setattr(
1612	"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.check_marlin_supported",
1613	lambda args, *kwargs: False,
1614	)
1615
1616	with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
1617	INCWna16Scheme().get_linear_method(
1618	make_config(),
1619	object(),
1620	"layer",
1621	make_layer_config(),
1622	)
1623
1624
1625	def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None:
1626	captured = {}
1627
1628	class DummyMethod:
1629	def __init__(self, cfg):
1630	captured["cfg"] = cfg
1631
1632	monkeypatch.setattr(
1633	"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear."
1634	"check_marlin_supported",
1635	lambda args, *kwargs: True,
1636	)
1637	monkeypatch.setattr(
1638	"vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQLinearMethod",
1639	DummyMethod,
1640	)
1641
1642	scheme = INCWNA16LinearScheme(make_layer_config())
1643
1644	assert isinstance(scheme.linear_method, DummyMethod)
1645	assert isinstance(captured["cfg"], AutoGPTQConfig)
1646	assert captured["cfg"].weight_bits == 4
1647	assert captured["cfg"].group_size == 128
1648	assert captured["cfg"].is_sym is True
1649
1650
1651	def test_wna16_linear_gptq_unsupported_config_raises() -> None:
1652	with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
1653	INCWNA16LinearScheme(make_layer_config(sym=False))
1654
1655
1656	def test_inc_get_quant_method_unquantized_linear_returns_unquantized() -> None:
1657	config = make_config(extra_config={"layer": {"bits": 16}})
1658	layer = object.__new__(LinearBase)
1659
1660	method = config.get_quant_method(layer, "layer")
1661
1662	assert isinstance(method, UnquantizedLinearMethod)
1663
1664
1665	def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
1666	monkeypatch,
1667	) -> None:
1668	"""Early-exit returns UnquantizedFusedMoEMethod for FusedMoE layers
1669	when extra_config has bits >= 16."""
1670	config = make_config(extra_config={"layer": {"bits": 16}})
1671	layer = object.__new__(RoutedExperts)
1672	layer.moe_config = None
1673
1674	class DummyUnquantizedFusedMoEMethod:
1675	def __init__(self, moe_config) -> None:
1676	self.moe_config = moe_config
1677
1678	monkeypatch.setattr(
1679	"vllm.model_executor.layers.quantization.inc.inc.UnquantizedFusedMoEMethod",
1680	DummyUnquantizedFusedMoEMethod,
1681	)
1682
1683	method = config.get_quant_method(layer, "layer")
1684
1685	assert isinstance(method, DummyUnquantizedFusedMoEMethod)
1686	assert method.moe_config is None
1687
1688
1689	def test_inc_get_quant_method_linear_uses_resolved_scheme(monkeypatch) -> None:
1690	config = make_config()
1691	layer = object.__new__(LinearBase)
1692	sentinel = object()
1693
1694	class DummyScheme:
1695	def get_linear_method(self, _config, _layer, _prefix, _layer_config):
1696	return sentinel
1697
1698	monkeypatch.setattr(
1699	"vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
1700	lambda _layer_config: DummyScheme(),
1701	)
1702
1703	method = config.get_quant_method(layer, "layer")
1704
1705	assert method is sentinel
1706
1707
1708	def test_inc_get_quant_method_lm_head_uses_suffix_match(monkeypatch) -> None:
1709	"""lm_head extra_config should apply to fully-qualified prefix."""
1710	config = make_config(
1711	extra_config={
1712	"lm_head": {
1713	"bits": 4,
1714	"group_size": 128,
1715	"sym": True,
1716	}
1717	}
1718	)
1719	layer = object.__new__(ParallelLMHead)
1720	sentinel = object()
1721
1722	class DummyScheme:
1723	def get_linear_method(self, _config, _layer, _prefix, _layer_config):
1724	return sentinel
1725
1726	monkeypatch.setattr(
1727	"vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
1728	lambda _layer_config: DummyScheme(),
1729	)
1730
1731	method = config.get_quant_method(layer, "model.language_model.lm_head")
1732
1733	assert method is sentinel
1734
1735
1736	def test_inc_get_quant_method_moe_uses_resolved_scheme(monkeypatch) -> None:
1737	config = make_config()
1738	layer = object.__new__(RoutedExperts)
1739	sentinel = object()
1740
1741	class DummyScheme:
1742	def get_moe_method(self, _config, _layer, _prefix, _layer_config):
1743	return sentinel
1744
1745	monkeypatch.setattr(
1746	"vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
1747	lambda _layer_config: DummyScheme(),
1748	)
1749
1750	method = config.get_quant_method(layer, "layer")
1751
1752	assert method is sentinel
1753
1754
1755	def test_resolve_gptq_moe_falls_back_to_moe_wna16(monkeypatch) -> None:
1756	captured = {}
1757
1758	class DummyMoeConfig:
1759	pass
1760
1761	class DummyLayer:
1762	moe_config = DummyMoeConfig()
1763
1764	class DummyBuiltConfig:
1765	pass
1766
1767	built_config = DummyBuiltConfig()
1768
1769	class DummyMethod:
1770	def __init__(self, cfg, moe):
1771	captured["cfg"] = cfg
1772	captured["moe"] = moe
1773
1774	monkeypatch.setattr(
1775	"vllm.model_executor.layers.quantization.utils.marlin_utils."
1776	"check_moe_marlin_supports_layer",
1777	lambda args, *kwargs: False,
1778	)
1779	monkeypatch.setattr(
1780	"vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Config.from_config",
1781	lambda cfg: captured.update({"from_config": cfg}) or built_config,
1782	)
1783	monkeypatch.setattr(
1784	"vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Method",
1785	DummyMethod,
1786	)
1787
1788	layer_config = INCLayerConfig(
1789	bits=4,
1790	group_size=128,
1791	sym=True,
1792	packing_format="auto_round:auto_gptq",
1793	backend="auto",
1794	data_type="int",
1795	quantized=True,
1796	)
1797
1798	_resolve_gptq_moe(DummyLayer(), layer_config)
1799
1800	assert captured["from_config"] == {
1801	"quant_method": "gptq",
1802	"bits": 4,
1803	"group_size": 128,
1804	"sym": True,
1805	"lm_head": False,
1806	}
1807	assert captured["cfg"] is built_config
1808	assert captured["moe"] is DummyLayer.moe_config
1809
1810
1811	def test_resolve_gptq_moe_uses_auto_gptq_when_supported(monkeypatch) -> None:
1812	captured = {}
1813
1814	class DummyMoeConfig:
1815	pass
1816
1817	class DummyLayer:
1818	moe_config = DummyMoeConfig()
1819
1820	class DummyMethod:
1821	def __init__(self, cfg, moe):
1822	captured["cfg"] = cfg
1823	captured["moe"] = moe
1824
1825	monkeypatch.setattr(
1826	"vllm.model_executor.layers.quantization.utils.marlin_utils."
1827	"check_moe_marlin_supports_layer",
1828	lambda args, *kwargs: True,
1829	)
1830	monkeypatch.setattr(
1831	"vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQMoEMethod",
1832	DummyMethod,
1833	)
1834
1835	_resolve_gptq_moe(DummyLayer(), make_layer_config())
1836
1837	assert isinstance(captured["cfg"], AutoGPTQConfig)
1838	assert captured["cfg"].weight_bits == 4
1839	assert captured["cfg"].group_size == 128
1840	assert captured["cfg"].is_sym is True
1841	assert captured["moe"] is DummyLayer.moe_config
1842
1843
1844	def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None:
1845	captured = {}
1846
1847	class DummyMoeConfig:
1848	pass
1849
1850	class DummyLayer:
1851	moe_config = DummyMoeConfig()
1852
1853	class DummyMethod:
1854	def __init__(self, cfg, moe):
1855	captured["cfg"] = cfg
1856	captured["moe"] = moe
1857
1858	monkeypatch.setattr(
1859	"vllm.model_executor.layers.quantization.utils.marlin_utils.check_moe_marlin_supports_layer",
1860	lambda args, *kwargs: True,
1861	)
1862	monkeypatch.setattr(
1863	"vllm.model_executor.layers.quantization.auto_awq.verify_marlin_supported",
1864	lambda args, *kwargs: None,
1865	)
1866	monkeypatch.setattr(
1867	"vllm.model_executor.layers.quantization.auto_awq.AutoAWQMoEMethod",
1868	DummyMethod,
1869	)
1870
1871	layer_config = INCLayerConfig(
1872	bits=4,
1873	group_size=128,
1874	sym=False,
1875	packing_format="auto_round:auto_awq",
1876	backend="auto",
1877	data_type="int",
1878	quantized=True,
1879	)
1880
1881	_resolve_awq_moe(DummyLayer(), layer_config)
1882
1883	assert captured["cfg"].weight_bits == 4
1884	assert captured["cfg"].zero_point is True
1885	assert captured["moe"] is DummyLayer.moe_config
1886
1887
1888	# ---------------------------------------------------------------------------
1889	# Tests for get_layer_config step 4 (fused QKV / packed_modules_mapping)
1890	# ---------------------------------------------------------------------------
1891
1892
1893	class TestGetLayerConfigFusedQKV:
1894	"""Tests for step-4 (fused QKV / packed_modules_mapping) logic.
1895
1896	Focused on preventing false-positive substring matches.
1897	"""
1898
1899	def test_exact_fusion_key_match(self):
1900	"""A layer whose name contains 'qkv' maps to its extra_config entry."""
1901	config = make_config(
1902	extra_config={
1903	"model.layers.0.self_attn.qkv_proj": {"bits": 8},
1904	}
1905	)
1906	config.packed_modules_mapping = {
1907	"qkv_proj": ["q_proj", "k_proj", "v_proj"],
1908	}
1909	bits, _, _ = config.get_layer_config(
1910	DummyLayer(), "model.layers.0.self_attn.qkv_proj"
1911	)
1912	assert bits == 8
1913
1914	def test_false_substring_match_does_not_override(self):
1915	"""Regression test for the false-substring-match bug.
1916
1917	Scenario (Qwen3.6-35B-A3B VLM):
1918	- packed_modules_mapping has "qkv" → ["qkv"] (from vision encoder).
1919	- The GDN text-attention layer is named "in_proj_qkvz".
1920	- "qkv" is a substring of "in_proj_qkvz", so old code would enter
1921	step 4 and generate sub_name "in_proj_qkvz" (replacing "qkv" with
1922	"qkv"). That name is NOT in extra_config, so get_config() falls
1923	back to the global default (bits=4), even though correct is 16.
1924	- Fix: skip the fusion key when none of the generated sub_names
1925	actually exist in extra_config.
1926	"""
1927	config = make_config(
1928	extra_config={
1929	"model.layers.0.in_proj_qkv": {"bits": 16},
1930	"model.layers.0.in_proj_z": {"bits": 16},
1931	}
1932	)
1933	config.packed_modules_mapping = {
1934	"qkv": ["qkv"],
1935	}
1936	bits, _, _ = config.get_layer_config(
1937	DummyLayer(), "model.layers.0.in_proj_qkvz"
1938	)
1939	# bits should be the global default (4) – no erroneous fusion match
1940	assert bits == 4
1941
1942	def test_real_qkv_fusion_key_still_resolves(self):
1943	"""The true "qkv" fusion (vision encoder) still resolves correctly."""
1944	config = make_config(
1945	extra_config={
1946	"vision_model.encoder.layers.0.self_attn.qkv": {"bits": 8},
1947	}
1948	)
1949	config.packed_modules_mapping = {
1950	"qkv": ["qkv"],
1951	}
1952	bits, _, _ = config.get_layer_config(
1953	DummyLayer(), "vision_model.encoder.layers.0.self_attn.qkv"
1954	)
1955	assert bits == 8
1956
1957	def test_mixed_fp16_and_int4_fused_layer(self):
1958	"""All sub-keys must agree; inconsistent configs raise ValueError."""
1959	config = make_config(
1960	extra_config={
1961	"model.layers.0.self_attn.q_proj": {"bits": 16},
1962	"model.layers.0.self_attn.k_proj": {"bits": 4},
1963	"model.layers.0.self_attn.v_proj": {"bits": 4},
1964	}
1965	)
1966	config.packed_modules_mapping = {
1967	"qkv_proj": ["q_proj", "k_proj", "v_proj"],
1968	}
1969	with pytest.raises(ValueError, match="consistent quant config"):
1970	config.get_layer_config(DummyLayer(), "model.layers.0.self_attn.qkv_proj")
1971
1972	def test_fusion_triggered_by_regex_configured_sub_name(self):
1973	"""Fusion step 4 is still triggered when sub_names match via regex.
1974
1975	Ensures the guard does not regress when extra_config uses regex
1976	patterns instead of exact keys to configure sub-modules.
1977	"""
1978	config = make_config(
1979	extra_config={
1980	r"model\.layers\.\d+\.self_attn\.(q\|k\|v)_proj": {"bits": 8},
1981	}
1982	)
1983	config.packed_modules_mapping = {
1984	"qkv_proj": ["q_proj", "k_proj", "v_proj"],
1985	}
1986	bits, _, _ = config.get_layer_config(
1987	DummyLayer(), "model.layers.0.self_attn.qkv_proj"
1988	)
1989	assert bits == 8
1990
1991
1992	# ---------------------------------------------------------------------------
1993	# INC XPU w4a8: int4 weights with dynamic per-token int8 activations
1994	#
1995	# The w4a8 path keeps int4 weights but dynamically quantizes activations to
1996	# per-token symmetric int8, so the GEMM stays on the int8 datapath instead of
1997	# upconverting weights to the activation dtype. It is opt-in through
1998	# VLLM_XPU_INC_WNA16_BACKEND because which kernel is fastest depends on the
1999	# device: ARK can use XMX int8 on some XPUs, and the default ("auto")
2000	# preference order is deliberately left unchanged.
2001	#
2002	# The backend-selection tests stub out ARK and the platform; the weight-creation
2003	# tests only need the TP-rank stubs. Both run anywhere. The apply_weights tests
2004	# stub out int4_gemm_w4a8 — what needs guarding there is the calling convention
2005	# (scale dtypes, argument order, weight reuse, batch-size gate), which is where
2006	# this path is easy to get silently wrong, since the kernel accepts bf16 scales
2007	# and returns plausible-looking garbage rather than raising. The ones that cross
2008	# the token threshold reach ``vllm._xpu_ops`` for the activation quantizer, so
2009	# they are XPU-image only; the small-batch fallback test returns before that
2010	# import and must stay unmarked to keep covering the gate off-XPU. The
2011	# end-to-end tests load a real int4 checkpoint and additionally need the op.
2012	# ---------------------------------------------------------------------------
2013
2014	_BACKEND_ENV = "VLLM_XPU_INC_WNA16_BACKEND"
2015	_ARK_STATE = (
2016	"vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state"
2017	)
2018	_QUANT_REF = "vllm._xpu_ops.xpu_ops.dynamic_per_token_int8_quant_ref"
2019
2020	IN_FEATURES = 128
2021	OUT_FEATURES = 64
2022
2023	# auto-round int4/sym/g128 checkpoints, which is what this backend serves. The
2024	# GPTQ- and AWQ-packed variants take different repack branches in
2025	# ``process_weights_after_loading``, so both are worth loading.
2026	E2E_MODELS = [
2027	pytest.param("OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", id="auto_round:auto_gptq"),
2028	pytest.param(
2029	"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", id="auto_round:auto_awq"
2030	),
2031	]
2032
2033
2034	def _has_xpu_ops() -> bool:
2035	"""True when ``vllm._xpu_ops`` is importable.
2036
2037	``apply_weights`` imports it for the activation quantizer, and the stubs
2038	below patch through it, so tests that exercise that path cannot even be set
2039	up on a build without vllm-xpu-kernels installed.
2040	"""
2041	if not current_platform.is_xpu():
2042	return False
2043	try:
2044	import vllm._xpu_ops # noqa: F401
2045	except ImportError:
2046	return False
2047	return True
2048
2049
2050	def _has_w4a8_kernel() -> bool:
2051	"""True when this build can actually run the w4a8 GEMM.
2052
2053	The op is registered as a side effect of importing ``vllm._xpu_ops``, so it
2054	is not visible on ``torch.ops._xpu_C`` until that import happens.
2055	"""
2056	return _has_xpu_ops() and hasattr(torch.ops._xpu_C, "int4_gemm_w4a8")
2057
2058
2059	requires_xpu_ops = pytest.mark.skipif(
2060	not _has_xpu_ops(),
2061	reason="needs an XPU image with vllm._xpu_ops available",
2062	)
2063
2064	requires_w4a8_kernel = pytest.mark.skipif(
2065	not _has_w4a8_kernel(),
2066	reason="needs an XPU with the int4_gemm_w4a8 op",
2067	)
2068
2069
2070	def _fake_xpu(monkeypatch, ark_available: bool = True) -> None:
2071	class DummyQuantLinear:
2072	pass
2073
2074	monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
2075	monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
2076	monkeypatch.setattr(
2077	_ARK_STATE,
2078	lambda: (
2079	(True, None, object(), DummyQuantLinear)
2080	if ark_available
2081	else (False, "missing", None, None)
2082	),
2083	)
2084
2085
2086	def _with_w4a8_kernel(monkeypatch) -> None:
2087	"""Present the op so the w4a8 support check passes off-XPU."""
2088
2089	class FakeOps:
2090	int4_gemm_w4a8 = object()
2091
2092	monkeypatch.setattr(torch.ops, "_xpu_C", FakeOps, raising=False)
2093
2094
2095	def _dispatch(layer_config=None):
2096	return INCWna16Scheme().get_linear_method(
2097	make_config(), object(), "layer", layer_config or make_layer_config()
2098	)
2099
2100
2101	@pytest.fixture
2102	def single_tp_rank(monkeypatch):
2103	"""Parameter creation reads the TP rank/size; there is no distributed init."""
2104	monkeypatch.setattr(
2105	"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
2106	lambda: 0,
2107	)
2108	monkeypatch.setattr(
2109	"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
2110	lambda: 1,
2111	)
2112
2113
2114	def _create_weights(
2115	method,
2116	input_size_per_partition: int = IN_FEATURES,
2117	output_partition_sizes: list[int] \| None = None,
2118	) -> torch.nn.Module:
2119	class DummyLayer(torch.nn.Module):
2120	pass
2121
2122	layer = DummyLayer()
2123	output_partition_sizes = (
2124	[OUT_FEATURES] if output_partition_sizes is None else output_partition_sizes
2125	)
2126	method.create_weights(
2127	layer=layer,
2128	input_size_per_partition=input_size_per_partition,
2129	output_partition_sizes=output_partition_sizes,
2130	input_size=input_size_per_partition,
2131	output_size=sum(output_partition_sizes),
2132	params_dtype=torch.bfloat16,
2133	weight_loader=lambda args, *kwargs: None,
2134	)
2135	return layer
2136
2137
2138	@pytest.fixture
2139	def w4a8_layer(single_tp_rank):
2140	"""A w4a8 method plus a layer whose weights are created but not processed."""
2141	method = INCXPUW4A8LinearMethod(make_layer_config())
2142	return method, _create_weights(method)
2143
2144
2145	def _stub_quant(monkeypatch, captured=None):
2146	def fake_quant(x, use_sym, bits):
2147	if captured is not None:
2148	captured["quant_args"] = (tuple(x.shape), use_sym, bits)
2149	m, k = x.shape
2150	return (
2151	torch.zeros(m, k, dtype=torch.int8),
2152	torch.ones(m, 1, dtype=x.dtype),
2153	torch.zeros(m, 1, dtype=torch.int32),
2154	)
2155
2156	monkeypatch.setattr(_QUANT_REF, fake_quant, raising=False)
2157
2158
2159	def _stub_gemm(monkeypatch, captured=None):
2160	def fake_gemm(*args):
2161	if captured is not None:
2162	captured["gemm_args"] = args
2163	# The real kernel always emits float16, whatever the model dtype.
2164	return torch.zeros(args[0].shape[0], OUT_FEATURES, dtype=torch.float16)
2165
2166	monkeypatch.setattr(torch.ops._xpu_C, "int4_gemm_w4a8", fake_gemm, raising=False)
2167
2168
2169	# ---------------------------------------------------------------------------
2170	# Backend selection
2171	# ---------------------------------------------------------------------------
2172
2173
2174	def test_default_backend_keeps_ark_preference(monkeypatch) -> None:
2175	"""The "auto" default must not change upstream behaviour.
2176
2177	ARK still wins whenever it is importable.
2178	"""
2179	monkeypatch.delenv(_BACKEND_ENV, raising=False)
2180	_fake_xpu(monkeypatch, ark_available=True)
2181	_with_w4a8_kernel(monkeypatch)
2182
2183	method = _dispatch()
2184
2185	assert isinstance(method, INCLinearMethod)
2186	assert isinstance(method.scheme, INCARKLinearMethod)
2187
2188
2189	def test_w4a8_backend_overrides_available_ark(monkeypatch) -> None:
2190	"""An explicit request wins even though ARK would otherwise be chosen."""
2191	monkeypatch.setenv(_BACKEND_ENV, "w4a8")
2192	_fake_xpu(monkeypatch, ark_available=True)
2193	_with_w4a8_kernel(monkeypatch)
2194
2195	method = _dispatch()
2196
2197	assert isinstance(method.scheme, INCXPUW4A8LinearMethod)
2198	assert not isinstance(method.scheme, INCARKLinearMethod)
2199	# w4a8 subclasses w4a16 so it can fall back for small batches.
2200	assert isinstance(method.scheme, INCXPULinearMethod)
2201
2202
2203	def test_w4a16_backend_overrides_available_ark(monkeypatch) -> None:
2204	monkeypatch.setenv(_BACKEND_ENV, "w4a16")
2205	_fake_xpu(monkeypatch, ark_available=True)
2206
2207	method = _dispatch()
2208
2209	assert isinstance(method.scheme, INCXPULinearMethod)
2210	assert not isinstance(method.scheme, INCXPUW4A8LinearMethod)
2211	assert not isinstance(method.scheme, INCARKLinearMethod)
2212
2213
2214	def test_ark_backend_requested_but_unavailable_raises(monkeypatch) -> None:
2215	"""Explicitly asking for ARK must fail loudly, not fall back silently."""
2216	monkeypatch.setenv(_BACKEND_ENV, "ark")
2217	_fake_xpu(monkeypatch, ark_available=False)
2218
2219	with pytest.raises(NotImplementedError, match="auto_round_kernel is unavailable"):
2220	_dispatch()
2221
2222
2223	@pytest.mark.parametrize("backend", ["w4a16", "w4a8"])
2224	def test_onednn_backends_reject_int2(monkeypatch, backend) -> None:
2225	"""The oneDNN int4 GEMMs cannot serve int2; ARK is the only int2 path."""
2226	monkeypatch.setenv(_BACKEND_ENV, backend)
2227	_fake_xpu(monkeypatch, ark_available=True)
2228	_with_w4a8_kernel(monkeypatch)
2229
2230	with pytest.raises(NotImplementedError, match="only supports int4"):
2231	_dispatch(make_layer_config(bits=2))
2232
2233
2234	@pytest.mark.parametrize("group_size", [48, -1, 0])
2235	def test_w4a8_rejects_unaligned_group_size(monkeypatch, group_size) -> None:
2236	"""The kernel needs a group size that is a positive multiple of 32."""
2237	monkeypatch.setenv(_BACKEND_ENV, "w4a8")
2238	_fake_xpu(monkeypatch, ark_available=True)
2239	_with_w4a8_kernel(monkeypatch)
2240
2241	with pytest.raises(NotImplementedError, match="multiple of 32"):
2242	_dispatch(make_layer_config(group_size=group_size))
2243
2244
2245	@pytest.mark.parametrize("group_size", [128, 64, 32])
2246	def test_w4a8_accepts_aligned_group_size(monkeypatch, group_size) -> None:
2247	monkeypatch.setenv(_BACKEND_ENV, "w4a8")
2248	_fake_xpu(monkeypatch, ark_available=True)
2249	_with_w4a8_kernel(monkeypatch)
2250
2251	method = _dispatch(make_layer_config(group_size=group_size))
2252
2253	assert isinstance(method.scheme, INCXPUW4A8LinearMethod)
2254
2255
2256	def test_w4a8_requires_the_kernel(monkeypatch) -> None:
2257	"""Older vllm-xpu-kernels builds lack the op entirely."""
2258	monkeypatch.setenv(_BACKEND_ENV, "w4a8")
2259	_fake_xpu(monkeypatch, ark_available=True)
2260
2261	class FakeOpsNoW4A8:
2262	pass
2263
2264	monkeypatch.setattr(torch.ops, "_xpu_C", FakeOpsNoW4A8, raising=False)
2265
2266	with pytest.raises(NotImplementedError, match="int4_gemm_w4a8"):
2267	_dispatch()
2268
2269
2270	def test_invalid_backend_value_raises(monkeypatch) -> None:
2271	import vllm.envs as envs
2272
2273	monkeypatch.setenv(_BACKEND_ENV, "onednn")
2274	with pytest.raises(ValueError, match=_BACKEND_ENV):
2275	_ = envs.VLLM_XPU_INC_WNA16_BACKEND
2276
2277
2278	# ---------------------------------------------------------------------------
2279	# Partition shape
2280	# ---------------------------------------------------------------------------
2281	#
2282	# ``XPUW4A8IntLinearKernel.can_implement`` rejects partition shapes whose in/out
2283	# dims are not multiples of 8, but that check is unreachable from the INC path:
2284	# the shapes only exist once ``create_weights`` is called, well after backend
2285	# selection. Unaligned dims must not reach the kernel — the int4 packing divides
2286	# both dims by 8, so an unaligned size is silently truncated into a weight tensor
2287	# that is too small rather than rejected.
2288
2289
2290	def test_rejects_unaligned_input_partition(single_tp_rank) -> None:
2291	method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
2292
2293	with pytest.raises(NotImplementedError, match=r"multiples of 8.*input=132"):
2294	_create_weights(method, input_size_per_partition=132)
2295
2296
2297	def test_rejects_unaligned_output_partition(single_tp_rank) -> None:
2298	method = INCXPUW4A8LinearMethod(make_layer_config())
2299
2300	with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=60"):
2301	_create_weights(method, output_partition_sizes=[60])
2302
2303
2304	def test_rejects_unaligned_sum_of_output_partitions(single_tp_rank) -> None:
2305	"""A fused QKV/MLP layer is only as aligned as the sum of its shards."""
2306	method = INCXPUW4A8LinearMethod(make_layer_config())
2307
2308	with pytest.raises(NotImplementedError, match=r"multiples of 8.*output=76"):
2309	_create_weights(method, output_partition_sizes=[OUT_FEATURES, 12])
2310
2311
2312	def test_reports_both_unaligned_dims(single_tp_rank) -> None:
2313	"""The message names every offending dim, not just the first."""
2314	method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
2315
2316	with pytest.raises(NotImplementedError) as excinfo:
2317	_create_weights(
2318	method, input_size_per_partition=132, output_partition_sizes=[60]
2319	)
2320
2321	assert "input=132" in str(excinfo.value)
2322	assert "output=60" in str(excinfo.value)
2323
2324
2325	def test_accepts_aligned_partition_shape(single_tp_rank) -> None:
2326	"""Multiples of 8 that are not powers of two are still fine."""
2327	method = INCXPUW4A8LinearMethod(make_layer_config(group_size=32))
2328
2329	layer = _create_weights(
2330	method, input_size_per_partition=96, output_partition_sizes=[24, 48]
2331	)
2332
2333	assert layer.qweight.shape == (96 // 8, 72)
2334
2335
2336	def test_w4a16_accepts_unaligned_partition_shape(single_tp_rank) -> None:
2337	"""The constraint is a w4a8 kernel requirement; w4a16 must stay unaffected."""
2338	method = INCXPULinearMethod(make_layer_config(group_size=32))
2339
2340	layer = _create_weights(method, output_partition_sizes=[60])
2341
2342	assert layer.scales.shape[1] == 60
2343
2344
2345	# ---------------------------------------------------------------------------
2346	# Weight processing
2347	# ---------------------------------------------------------------------------
2348
2349
2350	def test_keeps_fp16_scales(w4a8_layer) -> None:
2351	"""The kernel reads scales as fp16 regardless of activation dtype.
2352
2353	Passing bf16 scales does not raise — it silently returns wrong results — so
2354	an fp16 copy must exist, while the bf16 ``scales`` survive for the
2355	small-batch w4a16 fallback.
2356	"""
2357	method, layer = w4a8_layer
2358	layer.scales.data.fill_(0.125) # exactly representable in both dtypes
2359
2360	method.process_weights_after_loading(layer)
2361
2362	assert layer.scales_fp16.dtype is torch.float16
2363	assert layer.scales_fp16.shape == layer.scales.shape
2364	assert torch.equal(layer.scales_fp16.data.float(), layer.scales.data.float())
2365	# The w4a16 fallback still needs the original activation-dtype scales.
2366	assert layer.scales.dtype is torch.bfloat16
2367
2368
2369	def test_reuses_w4a16_weight_layout(w4a8_layer) -> None:
2370	"""w4a8 and w4a16 accept bit-identical weights, so there is no repacking."""
2371	method, layer = w4a8_layer
2372	qweight_before = layer.qweight.data.clone()
2373
2374	method.process_weights_after_loading(layer)
2375
2376	assert layer.qweight.dtype is torch.int32
2377	assert layer.qweight.shape == (IN_FEATURES // 8, OUT_FEATURES)
2378	assert torch.equal(layer.qweight.data, qweight_before)
2379	# Symmetric int4: the zero point is the constant 8, as for w4a16.
2380	assert layer.qzeros.dtype is torch.int8
2381	assert layer.qzeros.tolist() == [8]
2382
2383
2384	# ---------------------------------------------------------------------------
2385	# apply_weights
2386	# ---------------------------------------------------------------------------
2387
2388
2389	def test_falls_back_to_w4a16_for_small_batches(monkeypatch, w4a8_layer) -> None:
2390	"""Below the token threshold, per-token quant overhead dominates."""
2391	method, layer = w4a8_layer
2392	method.process_weights_after_loading(layer)
2393
2394	calls = []
2395
2396	def record_fallback(self, layer, x, bias=None):
2397	calls.append(tuple(x.shape))
2398	return torch.zeros(x.shape[:-1] + (OUT_FEATURES,), dtype=x.dtype)
2399
2400	monkeypatch.setattr(INCXPULinearMethod, "apply_weights", record_fallback)
2401	monkeypatch.setattr(
2402	torch.ops._xpu_C,
2403	"int4_gemm_w4a8",
2404	lambda *args: pytest.fail("small batches must not call the w4a8 kernel"),
2405	raising=False,
2406	)
2407
2408	tokens = method._MIN_TOKENS_FOR_INT8 - 1
2409	x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16)
2410	out = method.apply_weights(layer, x)
2411
2412	assert calls == [(tokens, IN_FEATURES)]
2413	assert out.shape == (tokens, OUT_FEATURES)
2414
2415
2416	@requires_xpu_ops
2417	def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
2418	"""At/above the threshold, activations are int8-quantized per token."""
2419	method, layer = w4a8_layer
2420	method.process_weights_after_loading(layer)
2421
2422	monkeypatch.setattr(
2423	INCXPULinearMethod,
2424	"apply_weights",
2425	lambda self, layer, x, bias=None: pytest.fail(
2426	"large batches must not take the w4a16 fallback"
2427	),
2428	)
2429	captured: dict[str, tuple] = {}
2430	_stub_quant(monkeypatch, captured)
2431	_stub_gemm(monkeypatch, captured)
2432
2433	tokens = method._MIN_TOKENS_FOR_INT8
2434	x = torch.zeros(tokens, IN_FEATURES, dtype=torch.bfloat16)
2435	out = method.apply_weights(layer, x)
2436
2437	# Symmetric 8-bit per-token quantization over the flattened activations.
2438	assert captured["quant_args"] == ((tokens, IN_FEATURES), True, 8)
2439
2440	(
2441	quant_x,
2442	x_scale,
2443	x_zero,
2444	qweight,
2445	w_scale,
2446	w_zp,
2447	group_size,
2448	group_idx,
2449	bias,
2450	) = captured["gemm_args"]
2451	assert quant_x.dtype is torch.int8
2452	# Both scales must reach the kernel as fp16, whatever the model dtype.
2453	assert x_scale.dtype is torch.float16
2454	assert w_scale.dtype is torch.float16
2455	assert w_scale is layer.scales_fp16
2456	assert x_zero.dtype is torch.int32
2457	assert qweight is layer.qweight
2458	assert w_zp is layer.qzeros
2459	assert group_size == 128
2460	assert group_idx is None
2461	assert bias is None
2462
2463	# The kernel emits fp16; the result is cast back to the activation dtype.
2464	assert out.dtype is torch.bfloat16
2465	assert out.shape == (tokens, OUT_FEATURES)
2466
2467
2468	@requires_xpu_ops
2469	@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
2470	def test_scales_are_fp16_for_any_activation_dtype(
2471	monkeypatch, w4a8_layer, act_dtype
2472	) -> None:
2473	"""fp16 scales are required for bf16 and fp16 activations alike."""
2474	method, layer = w4a8_layer
2475	layer.scales.data = layer.scales.data.to(act_dtype)
2476	method.process_weights_after_loading(layer)
2477
2478	captured: dict[str, tuple] = {}
2479	_stub_quant(monkeypatch, captured)
2480	_stub_gemm(monkeypatch, captured)
2481
2482	x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=act_dtype)
2483	out = method.apply_weights(layer, x)
2484
2485	assert captured["gemm_args"][1].dtype is torch.float16
2486	assert captured["gemm_args"][4].dtype is torch.float16
2487	assert out.dtype is act_dtype
2488
2489
2490	@requires_xpu_ops
2491	def test_forwards_bias_to_kernel(monkeypatch, w4a8_layer) -> None:
2492	"""Bias is applied by the kernel, not added afterwards."""
2493	method, layer = w4a8_layer
2494	method.process_weights_after_loading(layer)
2495
2496	captured: dict[str, tuple] = {}
2497	_stub_quant(monkeypatch)
2498	_stub_gemm(monkeypatch, captured)
2499
2500	bias = torch.zeros(OUT_FEATURES, dtype=torch.bfloat16)
2501	x = torch.zeros(method._MIN_TOKENS_FOR_INT8, IN_FEATURES, dtype=torch.bfloat16)
2502	method.apply_weights(layer, x, bias)
2503
2504	assert captured["gemm_args"][8] is bias
2505
2506
2507	@requires_xpu_ops
2508	def test_preserves_leading_dims(monkeypatch, w4a8_layer) -> None:
2509	"""3D activations must round-trip through the flatten/reshape."""
2510	method, layer = w4a8_layer
2511	method.process_weights_after_loading(layer)
2512
2513	_stub_quant(monkeypatch)
2514	_stub_gemm(monkeypatch)
2515
2516	tokens = method._MIN_TOKENS_FOR_INT8
2517	x = torch.zeros(4, tokens, IN_FEATURES, dtype=torch.bfloat16)
2518	out = method.apply_weights(layer, x)
2519
2520	assert out.shape == (4, tokens, OUT_FEATURES)
2521	assert out.dtype is torch.bfloat16
2522
2523
2524	@requires_xpu_ops
2525	def test_batch_gate_uses_flattened_token_count(monkeypatch, w4a8_layer) -> None:
2526	"""The gate counts total tokens, not the leading dimension."""
2527	method, layer = w4a8_layer
2528	method.process_weights_after_loading(layer)
2529
2530	monkeypatch.setattr(
2531	INCXPULinearMethod,
2532	"apply_weights",
2533	lambda self, layer, x, bias=None: pytest.fail(
2534	"flattened token count is at the threshold; must use w4a8"
2535	),
2536	)
2537	_stub_quant(monkeypatch)
2538	_stub_gemm(monkeypatch)
2539
2540	tokens = method._MIN_TOKENS_FOR_INT8
2541	# Each leading slice is below the threshold, but 4 * (tokens // 4) is not.
2542	x = torch.zeros(4, tokens // 4, IN_FEATURES, dtype=torch.bfloat16)
2543	out = method.apply_weights(layer, x)
2544
2545	assert out.shape == (4, tokens // 4, OUT_FEATURES)
2546
2547
2548	# ---------------------------------------------------------------------------
2549	# End to end on real weights
2550	# ---------------------------------------------------------------------------
2551
2552
2553	@requires_w4a8_kernel
2554	@pytest.mark.parametrize("model", E2E_MODELS)
2555	def test_e2e_generates_with_real_weights(vllm_runner, monkeypatch, model) -> None:
2556	"""The whole path — real int4 checkpoint, real kernel, real output.
2557
2558	The stubbed tests above cannot catch a wrong-but-plausible calling
2559	convention, because the fake kernel accepts anything. Greedy decoding of a
2560	factual prompt does: wrongly set scales or a bad weight layout turn the
2561	answer into noise rather than raising.
2562	"""
2563	monkeypatch.setenv(_BACKEND_ENV, "w4a8")
2564
2565	with vllm_runner(
2566	model, max_model_len=512, enforce_eager=True, gpu_memory_utilization=0.55
2567	) as vllm_model:
2568	out = vllm_model.generate_greedy(["The capital of France is"], max_tokens=8)
2569
2570	assert "Paris" in out[0][1]
2571
2572
2573	@requires_w4a8_kernel
2574	def test_e2e_selects_w4a8_for_every_linear(vllm_runner, monkeypatch) -> None:
2575	"""Guard against the validation silently becoming dead code.
2576
2577	If backend selection stopped routing here, the generation test above would
2578	still pass on the w4a16 path, so assert the method is actually in use and
2579	that every partition shape it accepted honours the alignment rule.
2580	"""
2581	monkeypatch.setenv(_BACKEND_ENV, "w4a8")
2582	# apply_model pickles the closure below.
2583	monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
2584
2585	with vllm_runner(
2586	E2E_MODELS[0].values[0],
2587	max_model_len=512,
2588	enforce_eager=True,
2589	gpu_memory_utilization=0.55,
2590	) as vllm_model:
2591
2592	def check_model(model):
2593	shapes = []
2594	for module in model.modules():
2595	scheme = getattr(getattr(module, "quant_method", None), "scheme", None)
2596	if isinstance(scheme, INCXPUW4A8LinearMethod):
2597	# qweight is the packed [in // 8, out] layout.
2598	packed_in, out = module.qweight.shape
2599	shapes.append((packed_in * 8, out))
2600
2601	assert shapes, "no linear layer used the w4a8 method"
2602	assert all(i % 8 == 0 and o % 8 == 0 for i, o in shapes), shapes
2603
2604	vllm_model.apply_model(check_model)