@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bcb518ad7a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[quant][autoround]Refactor INC quantization into package with INCScheme orchestrator (#40601)

Signed-off-by: yiliu30 <yi4.liu@intel.com> Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: n1ck-guo <heng.guo@intel.com> Co-authored-by: Zhenzhong1 <zhenzhong.xu@intel.com>

Yi Liu committed Jun 17, 2026 at 21:51 UTC bcb518ad7a2138ad51b5f9912c07c63af35cee86
13 files changed +1940 -803
.buildkite/intel_jobs/test-intel.yaml
+17
@@ -87,3 +87,20 @@ steps:
87 cd tests &&
88 pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
89 pytest -v -s benchmarks/test_serve_cli.py'
90 + - label: "XPU quantization test"
91 + depends_on:
92 + - image-build-xpu
93 + timeout_in_minutes: 30
94 + device: intel_gpu
95 + no_plugin: true
96 + env:
97 + REGISTRY: "public.ecr.aws/q9t5s3a7"
98 + REPO: "vllm-ci-test-repo"
99 + source_file_dependencies:
100 + - vllm/
101 + - .buildkite/intel_jobs/test-intel.yaml
102 + commands:
103 + - >-
104 + bash .buildkite/scripts/hardware_ci/run-intel-test.sh
105 + 'cd tests &&
106 + pytest -v -s quantization/test_auto_round.py'
\ No newline at end of file
requirements/xpu.txt
+1 -1
@@ -16,5 +16,5 @@ torch==2.12.0
16 torchaudio
17 torchvision
18
19 -auto_round_lib>=0.13.0
19 +auto_round_lib>=0.13.3
20 vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.9.1/vllm_xpu_kernels-0.1.9.1-cp38-abi3-manylinux_2_28_x86_64.whl
tests/quantization/test_auto_round.py
+750 -8
@@ -10,23 +10,765 @@ Run `pytest tests/quantization/test_auto_round.py`.
10
11 import pytest
12
13 +from vllm.model_executor.layers.fused_moe import RoutedExperts
14 +from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
15 +from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
16 +from vllm.model_executor.layers.quantization.inc import INCConfig
17 +from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
18 +from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
19 +from vllm.model_executor.layers.quantization.inc.schemes import (
20 + INCWna16Scheme,
21 + resolve_scheme,
22 +)
23 +from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
24 + INCLinearScheme,
25 +)
26 +from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear import (
27 + INCARKLinearMethod,
28 + INCWNA16LinearScheme,
29 + INCXPULinearMethod,
30 +)
31 +from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_scheme import (
32 + _resolve_awq_moe,
33 + _resolve_gptq_moe,
34 +)
35 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36 from vllm.platforms import current_platform
37
38 MODELS = [
16 - "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", ##auto_round:auto_gptq
17 - "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", ##auto_round:auto_awq
39 + pytest.param(
40 + "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
41 + id="auto_round:auto_gptq",
42 + ),
43 + pytest.param(
44 + "Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound",
45 + marks=pytest.mark.skipif(
46 + not current_platform.is_cuda(),
47 + reason="AWQ AutoRound model only supports CUDA backend for now.",
48 + ),
49 + id="auto_round:auto_awq",
50 + ),
51 ]
52
53
54 @pytest.mark.skipif(
22 - not current_platform.is_cpu()
23 - and not current_platform.is_xpu()
24 - and not current_platform.is_cuda(),
25 - reason="only supports CPU/XPU/CUDA backend.",
55 + not (
56 + current_platform.is_cpu()
57 + or current_platform.is_xpu()
58 + or current_platform.is_cuda()
59 + ),
60 + reason="Only supports CPU/XPU/CUDA backend.",
61 )
62 @pytest.mark.parametrize("model", MODELS)
28 -def test_auto_round(vllm_runner, model):
63 +def test_auto_round_model(vllm_runner, model):
64 with vllm_runner(model, enforce_eager=True) as llm:
65 output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
66 +
67 assert output
32 - print(f"{output[0][1]}")
68 + print(output[0][1])
69 +
70 +
71 +# ---------------------------------------------------------------------------
72 +# Unit tests for INCConfig and related classes
73 +# ---------------------------------------------------------------------------
74 +
75 +
76 +class DummyLayer:
77 + pass
78 +
79 +
80 +class DummyFusedMoE:
81 + pass
82 +
83 +
84 +def make_config(**overrides) -> INCConfig:
85 + kwargs = {
86 + "weight_bits": 4,
87 + "group_size": 128,
88 + "sym": True,
89 + "packing_format": "auto_round:auto_gptq",
90 + "block_name_to_quantize": None,
91 + "extra_config": None,
92 + "data_type": "int",
93 + "backend": "auto",
94 + }
95 + kwargs.update(overrides)
96 + return INCConfig(**kwargs)
97 +
98 +
99 +def make_layer_config(**overrides) -> INCLayerConfig:
100 + kwargs = {
101 + "bits": 4,
102 + "group_size": 128,
103 + "sym": True,
104 + "packing_format": "auto_round:auto_gptq",
105 + "backend": "auto",
106 + "data_type": "int",
107 + "quantized": True,
108 + }
109 + kwargs.update(overrides)
110 + return INCLayerConfig(**kwargs)
111 +
112 +
113 +def test_inc_config_parser_exact_match() -> None:
114 + config = make_config(
115 + extra_config={
116 + "layers.0.self_attn.q_proj": {
117 + "bits": 8,
118 + "group_size": 64,
119 + "sym": False,
120 + }
121 + }
122 + )
123 +
124 + layer_config = config.config_parser.resolve(
125 + DummyLayer(), "layers.0.self_attn.q_proj"
126 + )
127 +
128 + assert layer_config.bits == 8
129 + assert layer_config.group_size == 64
130 + assert layer_config.sym is False
131 + assert layer_config.quantized is True
132 +
133 +
134 +def test_inc_model_prefix_early_exit() -> None:
135 + """extra_config keys with model. prefix trigger early unquantized return."""
136 + config = make_config(
137 + extra_config={
138 + "model.layers.1.mlp.gate_proj": {
139 + "bits": 16,
140 + },
141 + }
142 + )
143 +
144 + # get_quant_method checks model. prefix for unquantized early-exit
145 + result = config.get_quant_method(DummyLayer(), "layers.1.mlp.gate_proj")
146 + assert isinstance(result, UnquantizedLinearMethod)
147 +
148 +
149 +def test_inc_config_parser_regex_match() -> None:
150 + config = make_config(
151 + extra_config={
152 + r"layers\.\d+\.self_attn\.(q|k|v)_proj": {
153 + "bits": 8,
154 + "group_size": 64,
155 + "sym": False,
156 + }
157 + }
158 + )
159 +
160 + layer_config = config.config_parser.resolve(
161 + DummyLayer(), "layers.3.self_attn.q_proj"
162 + )
163 +
164 + assert layer_config.bits == 8
165 + assert layer_config.group_size == 64
166 + assert layer_config.sym is False
167 +
168 +
169 +def test_inc_config_parser_invalid_regex_ignored() -> None:
170 + config = make_config(
171 + extra_config={
172 + "[invalid": {
173 + "bits": 8,
174 + "group_size": 64,
175 + "sym": False,
176 + }
177 + }
178 + )
179 +
180 + layer_config = config.config_parser.resolve(
181 + DummyLayer(), "layers.0.self_attn.q_proj"
182 + )
183 +
184 + assert layer_config.bits == 4
185 + assert layer_config.group_size == 128
186 + assert layer_config.sym is True
187 +
188 +
189 +def test_inc_config_parser_block_name_to_quantize_marks_unquantized() -> None:
190 + config = make_config(block_name_to_quantize=["layers.1"])
191 +
192 + layer_config = config.config_parser.resolve(
193 + DummyLayer(), "layers.0.self_attn.q_proj"
194 + )
195 +
196 + assert layer_config.bits == 16
197 + assert layer_config.group_size == -1
198 + assert layer_config.sym is True
199 + assert layer_config.quantized is False
200 +
201 +
202 +def test_inc_config_parser_parallel_lm_head_defaults_to_unquantized() -> None:
203 + layer = object.__new__(ParallelLMHead)
204 + config = make_config()
205 +
206 + layer_config = config.config_parser.resolve(layer, "lm_head")
207 +
208 + assert layer_config.quantized is False
209 + assert layer_config.bits == 16
210 +
211 +
212 +def test_inc_config_parser_fused_moe_requires_consistent_configs() -> None:
213 + config = make_config(
214 + extra_config={
215 + "layers.0.block_sparse_moe.experts.0.w1": {
216 + "bits": 4,
217 + "group_size": 128,
218 + "sym": True,
219 + },
220 + "layers.0.block_sparse_moe.experts.0.w2": {
221 + "bits": 8,
222 + "group_size": 128,
223 + "sym": True,
224 + },
225 + }
226 + )
227 +
228 + with pytest.raises(ValueError, match="requires consistent quant config"):
229 + config.config_parser.resolve(DummyFusedMoE(), "layers.0.block_sparse_moe")
230 +
231 +
232 +def test_inc_config_parser_fused_module_requires_consistent_configs() -> None:
233 + config = make_config(
234 + extra_config={
235 + "layers.0.self_attn.q_proj": {
236 + "bits": 4,
237 + "group_size": 128,
238 + "sym": True,
239 + },
240 + "layers.0.self_attn.k_proj": {
241 + "bits": 8,
242 + "group_size": 128,
243 + "sym": True,
244 + },
245 + "layers.0.self_attn.v_proj": {
246 + "bits": 4,
247 + "group_size": 128,
248 + "sym": True,
249 + },
250 + }
251 + )
252 + config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
253 +
254 + with pytest.raises(ValueError, match="requires consistent quant config"):
255 + config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj")
256 +
257 +
258 +def test_inc_layer_config_mx_fp_helpers() -> None:
259 + layer_config = INCLayerConfig(
260 + bits=4,
261 + group_size=32,
262 + sym=True,
263 + packing_format="",
264 + backend="",
265 + data_type="mx_fp",
266 + quantized=True,
267 + )
268 +
269 + assert layer_config.is_mxfp4 is True
270 + assert layer_config.is_mxfp8 is False
271 +
272 +
273 +def test_inc_resolve_scheme_selects_wna16() -> None:
274 + layer_config = INCLayerConfig(
275 + bits=4,
276 + group_size=128,
277 + sym=True,
278 + packing_format="auto_round:auto_gptq",
279 + backend="auto",
280 + data_type="int",
281 + quantized=True,
282 + )
283 +
284 + scheme = resolve_scheme(layer_config)
285 +
286 + assert isinstance(scheme, INCWna16Scheme)
287 +
288 +
289 +class DummyLinearScheme(INCLinearScheme):
290 + def __init__(self) -> None:
291 + self.calls: list[tuple] = []
292 +
293 + @classmethod
294 + def get_min_capability(cls) -> int:
295 + return 0
296 +
297 + def create_weights(self, *args, **kwargs) -> None:
298 + self.calls.append(("create_weights", args, kwargs))
299 +
300 + def process_weights_after_loading(self, layer) -> None:
301 + self.calls.append(("process_weights_after_loading", layer))
302 +
303 + def apply_weights(self, layer, x, bias=None):
304 + self.calls.append(("apply_weights", layer, x, bias))
305 + return "applied"
306 +
307 +
308 +def test_inc_linear_method_delegates() -> None:
309 + scheme = DummyLinearScheme()
310 + method = INCLinearMethod(scheme)
311 + layer = DummyLayer()
312 +
313 + method.create_weights(
314 + layer,
315 + input_size_per_partition=1,
316 + output_partition_sizes=[2],
317 + input_size=1,
318 + output_size=2,
319 + params_dtype=None,
320 + )
321 + method.process_weights_after_loading(layer)
322 + result = method.apply(layer, "x", "b")
323 +
324 + assert result == "applied"
325 + assert [call[0] for call in scheme.calls] == [
326 + "create_weights",
327 + "process_weights_after_loading",
328 + "apply_weights",
329 + ]
330 +
331 +
332 +def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
333 + class DummyQuantLinear:
334 + pass
335 +
336 + monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
337 + monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
338 + monkeypatch.setattr(
339 + "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
340 + lambda: (True, None, object(), DummyQuantLinear),
341 + )
342 +
343 + method = INCWna16Scheme().get_linear_method(
344 + make_config(),
345 + object(),
346 + "layer",
347 + make_layer_config(),
348 + )
349 +
350 + assert isinstance(method, INCLinearMethod)
351 + assert isinstance(method.scheme, INCARKLinearMethod)
352 +
353 +
354 +def test_wna16_xpu_falls_back_when_ark_unavailable(monkeypatch) -> None:
355 + monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
356 + monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
357 + monkeypatch.setattr(
358 + "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
359 + lambda: (False, "missing", None, None),
360 + )
361 +
362 + method = INCWna16Scheme().get_linear_method(
363 + make_config(),
364 + object(),
365 + "layer",
366 + make_layer_config(),
367 + )
368 +
369 + assert isinstance(method, INCLinearMethod)
370 + assert isinstance(method.scheme, INCXPULinearMethod)
371 +
372 +
373 +def test_wna16_cpu_gptq_prefers_ark_when_available(monkeypatch) -> None:
374 + class DummyQuantLinear:
375 + pass
376 +
377 + monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
378 + monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
379 + monkeypatch.setattr(
380 + "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
381 + lambda: (True, None, object(), DummyQuantLinear),
382 + )
383 +
384 + method = INCWna16Scheme().get_linear_method(
385 + make_config(),
386 + object(),
387 + "layer",
388 + make_layer_config(),
389 + )
390 +
391 + assert isinstance(method, INCLinearMethod)
392 + assert isinstance(method.scheme, INCARKLinearMethod)
393 +
394 +
395 +def test_wna16_cpu_gptq_raises_when_ark_and_marlin_unavailable(
396 + monkeypatch,
397 +) -> None:
398 + monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
399 + monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
400 + monkeypatch.setattr(
401 + "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
402 + lambda: (False, "missing", None, None),
403 + )
404 + monkeypatch.setattr(
405 + "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.check_marlin_supported",
406 + lambda *args, **kwargs: False,
407 + )
408 +
409 + with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
410 + INCWna16Scheme().get_linear_method(
411 + make_config(),
412 + object(),
413 + "layer",
414 + make_layer_config(),
415 + )
416 +
417 +
418 +def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None:
419 + captured = {}
420 +
421 + class DummyMethod:
422 + def __init__(self, cfg):
423 + captured["cfg"] = cfg
424 +
425 + monkeypatch.setattr(
426 + "vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear."
427 + "check_marlin_supported",
428 + lambda *args, **kwargs: True,
429 + )
430 + monkeypatch.setattr(
431 + "vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQLinearMethod",
432 + DummyMethod,
433 + )
434 +
435 + scheme = INCWNA16LinearScheme(make_layer_config())
436 +
437 + assert isinstance(scheme.inner_method, DummyMethod)
438 + assert isinstance(captured["cfg"], AutoGPTQConfig)
439 + assert captured["cfg"].weight_bits == 4
440 + assert captured["cfg"].group_size == 128
441 + assert captured["cfg"].is_sym is True
442 +
443 +
444 +def test_wna16_linear_gptq_unsupported_config_raises() -> None:
445 + with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
446 + INCWNA16LinearScheme(make_layer_config(sym=False))
447 +
448 +
449 +def test_wna16_xpu_unsupported_config_still_raises(monkeypatch) -> None:
450 + monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
451 + monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
452 +
453 + with pytest.raises(NotImplementedError, match="unsupported config"):
454 + INCWna16Scheme().get_linear_method(
455 + make_config(sym=False),
456 + object(),
457 + "layer",
458 + make_layer_config(sym=False),
459 + )
460 +
461 +
462 +def test_inc_get_quant_method_unquantized_linear_returns_unquantized() -> None:
463 + config = make_config(extra_config={"layer": {"bits": 16}})
464 + layer = object.__new__(LinearBase)
465 +
466 + method = config.get_quant_method(layer, "layer")
467 +
468 + assert isinstance(method, UnquantizedLinearMethod)
469 +
470 +
471 +def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
472 + monkeypatch,
473 +) -> None:
474 + """Early-exit returns UnquantizedFusedMoEMethod for FusedMoE layers
475 + when extra_config has bits >= 16."""
476 + config = make_config(extra_config={"layer": {"bits": 16}})
477 + layer = object.__new__(RoutedExperts)
478 + layer.moe_config = None # UnquantizedFusedMoEMethod accepts moe_config
479 +
480 + class DummyUnquantizedFusedMoEMethod:
481 + def __init__(self, moe_config) -> None:
482 + self.moe_config = moe_config
483 +
484 + monkeypatch.setattr(
485 + "vllm.model_executor.layers.quantization.inc.inc.UnquantizedFusedMoEMethod",
486 + DummyUnquantizedFusedMoEMethod,
487 + )
488 +
489 + method = config.get_quant_method(layer, "layer")
490 +
491 + assert isinstance(method, DummyUnquantizedFusedMoEMethod)
492 + assert method.moe_config is None
493 +
494 +
495 +def test_inc_get_quant_method_linear_uses_resolved_scheme(monkeypatch) -> None:
496 + config = make_config()
497 + layer = object.__new__(LinearBase)
498 + sentinel = object()
499 +
500 + class DummyScheme:
501 + def get_linear_method(self, _config, _layer, _prefix, _layer_config):
502 + return sentinel
503 +
504 + monkeypatch.setattr(
505 + "vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
506 + lambda _layer_config: DummyScheme(),
507 + )
508 +
509 + method = config.get_quant_method(layer, "layer")
510 +
511 + assert method is sentinel
512 +
513 +
514 +def test_inc_get_quant_method_moe_uses_resolved_scheme(monkeypatch) -> None:
515 + config = make_config()
516 + layer = object.__new__(RoutedExperts)
517 + sentinel = object()
518 +
519 + class DummyScheme:
520 + def get_moe_method(self, _config, _layer, _prefix, _layer_config):
521 + return sentinel
522 +
523 + monkeypatch.setattr(
524 + "vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
525 + lambda _layer_config: DummyScheme(),
526 + )
527 +
528 + method = config.get_quant_method(layer, "layer")
529 +
530 + assert method is sentinel
531 +
532 +
533 +def test_resolve_gptq_moe_falls_back_to_moe_wna16(monkeypatch) -> None:
534 + captured = {}
535 +
536 + class DummyMoeConfig:
537 + pass
538 +
539 + class DummyLayer:
540 + moe_config = DummyMoeConfig()
541 +
542 + class DummyBuiltConfig:
543 + pass
544 +
545 + built_config = DummyBuiltConfig()
546 +
547 + class DummyMethod:
548 + def __init__(self, cfg, moe):
549 + captured["cfg"] = cfg
550 + captured["moe"] = moe
551 +
552 + monkeypatch.setattr(
553 + "vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
554 + lambda *args, **kwargs: False,
555 + )
556 + monkeypatch.setattr(
557 + "vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Config.from_config",
558 + lambda cfg: captured.update({"from_config": cfg}) or built_config,
559 + )
560 + monkeypatch.setattr(
561 + "vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Method",
562 + DummyMethod,
563 + )
564 +
565 + layer_config = INCLayerConfig(
566 + bits=4,
567 + group_size=128,
568 + sym=True,
569 + packing_format="auto_round:auto_gptq",
570 + backend="auto",
571 + data_type="int",
572 + quantized=True,
573 + )
574 +
575 + _resolve_gptq_moe(DummyLayer(), layer_config)
576 +
577 + assert captured["from_config"] == {
578 + "quant_method": "gptq",
579 + "bits": 4,
580 + "group_size": 128,
581 + "sym": True,
582 + "lm_head": False,
583 + }
584 + assert captured["cfg"] is built_config
585 + assert captured["moe"] is DummyLayer.moe_config
586 +
587 +
588 +def test_resolve_gptq_moe_uses_auto_gptq_when_supported(monkeypatch) -> None:
589 + captured = {}
590 +
591 + class DummyMoeConfig:
592 + pass
593 +
594 + class DummyLayer:
595 + moe_config = DummyMoeConfig()
596 +
597 + class DummyMethod:
598 + def __init__(self, cfg, moe):
599 + captured["cfg"] = cfg
600 + captured["moe"] = moe
601 +
602 + monkeypatch.setattr(
603 + "vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
604 + lambda *args, **kwargs: True,
605 + )
606 + monkeypatch.setattr(
607 + "vllm.model_executor.layers.quantization.utils.marlin_utils."
608 + "check_moe_marlin_supports_layer",
609 + lambda *args, **kwargs: True,
610 + )
611 + monkeypatch.setattr(
612 + "vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQMoEMethod",
613 + DummyMethod,
614 + )
615 +
616 + _resolve_gptq_moe(DummyLayer(), make_layer_config())
617 +
618 + assert isinstance(captured["cfg"], AutoGPTQConfig)
619 + assert captured["cfg"].weight_bits == 4
620 + assert captured["cfg"].group_size == 128
621 + assert captured["cfg"].is_sym is True
622 + assert captured["moe"] is DummyLayer.moe_config
623 +
624 +
625 +def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None:
626 + captured = {}
627 +
628 + class DummyMoeConfig:
629 + pass
630 +
631 + class DummyLayer:
632 + moe_config = DummyMoeConfig()
633 +
634 + class DummyMethod:
635 + def __init__(self, cfg, moe):
636 + captured["cfg"] = cfg
637 + captured["moe"] = moe
638 +
639 + monkeypatch.setattr(
640 + "vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
641 + lambda *args, **kwargs: True,
642 + )
643 + monkeypatch.setattr(
644 + "vllm.model_executor.layers.quantization.utils.marlin_utils.check_moe_marlin_supports_layer",
645 + lambda *args, **kwargs: True,
646 + )
647 + monkeypatch.setattr(
648 + "vllm.model_executor.layers.quantization.awq_marlin.verify_marlin_supported",
649 + lambda *args, **kwargs: None,
650 + )
651 + monkeypatch.setattr(
652 + "vllm.model_executor.layers.quantization.awq_marlin.AWQMarlinMoEMethod",
653 + DummyMethod,
654 + )
655 +
656 + layer_config = INCLayerConfig(
657 + bits=4,
658 + group_size=128,
659 + sym=False,
660 + packing_format="auto_round:auto_awq",
661 + backend="auto",
662 + data_type="int",
663 + quantized=True,
664 + )
665 +
666 + _resolve_awq_moe(DummyLayer(), layer_config)
667 +
668 + assert captured["cfg"].weight_bits == 4
669 + assert captured["cfg"].zero_point is True
670 + assert captured["moe"] is DummyLayer.moe_config
671 +
672 +
673 +# ---------------------------------------------------------------------------
674 +# Tests for get_layer_config step 4 (fused QKV / packed_modules_mapping)
675 +# ---------------------------------------------------------------------------
676 +
677 +
678 +class TestGetLayerConfigFusedQKV:
679 + """Tests for step-4 (fused QKV / packed_modules_mapping) logic.
680 +
681 + Focused on preventing false-positive substring matches.
682 + """
683 +
684 + def test_exact_fusion_key_match(self):
685 + """A layer whose name contains 'qkv' maps to its extra_config entry."""
686 + config = make_config(
687 + extra_config={
688 + "model.layers.0.self_attn.qkv_proj": {"bits": 8},
689 + }
690 + )
691 + config.packed_modules_mapping = {
692 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
693 + }
694 + bits, _, _ = config.get_layer_config(
695 + DummyLayer(), "model.layers.0.self_attn.qkv_proj"
696 + )
697 + assert bits == 8
698 +
699 + def test_false_substring_match_does_not_override(self):
700 + """Regression test for the false-substring-match bug.
701 +
702 + Scenario (Qwen3.6-35B-A3B VLM):
703 + - packed_modules_mapping has "qkv" → ["qkv"] (from vision encoder).
704 + - The GDN text-attention layer is named "in_proj_qkvz".
705 + - "qkv" is a substring of "in_proj_qkvz", so old code would enter
706 + step 4 and generate sub_name "in_proj_qkvz" (replacing "qkv" with
707 + "qkv"). That name is NOT in extra_config, so get_config() falls
708 + back to the global default (bits=4), even though correct is 16.
709 + - Fix: skip the fusion key when none of the generated sub_names
710 + actually exist in extra_config.
711 + """
712 + config = make_config(
713 + extra_config={
714 + "model.layers.0.in_proj_qkv": {"bits": 16},
715 + "model.layers.0.in_proj_z": {"bits": 16},
716 + }
717 + )
718 + config.packed_modules_mapping = {
719 + "qkv": ["qkv"],
720 + }
721 + bits, _, _ = config.get_layer_config(
722 + DummyLayer(), "model.layers.0.in_proj_qkvz"
723 + )
724 + # bits should be the global default (4) – no erroneous fusion match
725 + assert bits == 4
726 +
727 + def test_real_qkv_fusion_key_still_resolves(self):
728 + """The true "qkv" fusion (vision encoder) still resolves correctly."""
729 + config = make_config(
730 + extra_config={
731 + "vision_model.encoder.layers.0.self_attn.qkv": {"bits": 8},
732 + }
733 + )
734 + config.packed_modules_mapping = {
735 + "qkv": ["qkv"],
736 + }
737 + bits, _, _ = config.get_layer_config(
738 + DummyLayer(), "vision_model.encoder.layers.0.self_attn.qkv"
739 + )
740 + assert bits == 8
741 +
742 + def test_mixed_fp16_and_int4_fused_layer(self):
743 + """All sub-keys must agree; inconsistent configs raise ValueError."""
744 + config = make_config(
745 + extra_config={
746 + "model.layers.0.self_attn.q_proj": {"bits": 16},
747 + "model.layers.0.self_attn.k_proj": {"bits": 4},
748 + "model.layers.0.self_attn.v_proj": {"bits": 4},
749 + }
750 + )
751 + config.packed_modules_mapping = {
752 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
753 + }
754 + with pytest.raises(ValueError, match="consistent quant config"):
755 + config.get_layer_config(DummyLayer(), "model.layers.0.self_attn.qkv_proj")
756 +
757 + def test_fusion_triggered_by_regex_configured_sub_name(self):
758 + """Fusion step 4 is still triggered when sub_names match via regex.
759 +
760 + Ensures the guard does not regress when extra_config uses regex
761 + patterns instead of exact keys to configure sub-modules.
762 + """
763 + config = make_config(
764 + extra_config={
765 + r"model\.layers\.\d+\.self_attn\.(q|k|v)_proj": {"bits": 8},
766 + }
767 + )
768 + config.packed_modules_mapping = {
769 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
770 + }
771 + bits, _, _ = config.get_layer_config(
772 + DummyLayer(), "model.layers.0.self_attn.qkv_proj"
773 + )
774 + assert bits == 8
vllm/model_executor/layers/quantization/inc.py deleted
-794
@@ -1,794 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -from fractions import Fraction
5 -from functools import lru_cache
6 -from typing import TYPE_CHECKING, Any
7 -
8 -import regex as re
9 -import torch
10 -from torch.nn.parameter import Parameter
11 -
12 -from vllm.logger import init_logger
13 -from vllm.model_executor.layers.fused_moe import RoutedExperts
14 -from vllm.model_executor.layers.linear import (
15 - LinearBase,
16 - LinearMethodBase,
17 - UnquantizedLinearMethod,
18 -)
19 -from vllm.model_executor.layers.quantization import (
20 - QuantizationConfig,
21 - QuantizationMethods,
22 -)
23 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
24 -from vllm.model_executor.parameter import (
25 - GroupQuantScaleParameter,
26 - PackedvLLMParameter,
27 - RowvLLMParameter,
28 -)
29 -from vllm.platforms import current_platform
30 -from vllm.scalar_type import scalar_types
31 -
32 -if TYPE_CHECKING:
33 - from vllm.model_executor.models.utils import WeightsMapper
34 -
35 -logger = init_logger(__name__)
36 -
37 -
38 -class INCConfig(QuantizationConfig):
39 - """Config class for Intel Neural Compressor (INC).
40 - Repo: https://github.com/intel/neural-compressor
41 - """
42 -
43 - SUPPORTED_BITS = {2, 3, 4, 8}
44 - SUPPORTED_DTYPES = {"int"}
45 - SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
46 - SUPPORTED_BACKENDS = {
47 - "auto",
48 - "gptq",
49 - "gptq:marlin",
50 - "awq",
51 - "awq:marlin",
52 - "marlin",
53 - }
54 -
55 - def __init__(
56 - self,
57 - weight_bits: int,
58 - group_size: int,
59 - sym: bool = True,
60 - packing_format: str = "auto_round:auto_gptq",
61 - block_name_to_quantize: str | list[str] | None = None,
62 - extra_config: dict[str, Any] | None = None,
63 - data_type: str = "int",
64 - backend: str = "auto",
65 - ) -> None:
66 - super().__init__()
67 - if weight_bits not in self.SUPPORTED_BITS:
68 - raise ValueError(
69 - f"Unsupported weight_bits: {weight_bits}, "
70 - f"currently only support {self.SUPPORTED_BITS}."
71 - )
72 - if data_type not in self.SUPPORTED_DTYPES:
73 - raise ValueError(
74 - f"Unsupported data_type: {data_type},"
75 - f" currently only support {self.SUPPORTED_DTYPES}."
76 - )
77 - if packing_format not in self.SUPPORTED_FORMATS:
78 - raise ValueError(
79 - f"Unsupported packing_format: {packing_format}, "
80 - f"currently only support {self.SUPPORTED_FORMATS}."
81 - )
82 - if backend not in self.SUPPORTED_BACKENDS:
83 - raise ValueError(
84 - f"Unsupported backend: {backend}, "
85 - f"currently only support {self.SUPPORTED_BACKENDS}."
86 - )
87 -
88 - self.weight_bits = weight_bits
89 - self.group_size = group_size
90 - self.sym = sym
91 - self.packing_format = packing_format
92 - self.block_name_to_quantize = (
93 - block_name_to_quantize.split(",")
94 - if isinstance(block_name_to_quantize, str)
95 - else block_name_to_quantize
96 - )
97 - self.extra_config = extra_config
98 - self.data_type = data_type
99 - self.backend = backend
100 - self.pack_factor = Fraction(32, weight_bits)
101 -
102 - def __repr__(self) -> str:
103 - return (
104 - f"INCConfig(weight_bits={self.weight_bits}, "
105 - f"group_size={self.group_size}, sym={self.sym})"
106 - )
107 -
108 - @classmethod
109 - def get_name(cls) -> QuantizationMethods:
110 - return "inc"
111 -
112 - @classmethod
113 - def get_supported_act_dtypes(cls) -> list[torch.dtype]:
114 - return [torch.half, torch.bfloat16]
115 -
116 - @classmethod
117 - def get_min_capability(cls) -> int:
118 - return 60
119 -
120 - @classmethod
121 - def get_config_filenames(cls) -> list[str]:
122 - return ["quantization_config.json"]
123 -
124 - @classmethod
125 - def from_config(cls, config: dict[str, Any]) -> "INCConfig":
126 - return cls(
127 - weight_bits=cls.get_from_keys(config, ["bits"]),
128 - group_size=cls.get_from_keys(config, ["group_size"]),
129 - sym=cls.get_from_keys(config, ["sym"]),
130 - packing_format=cls.get_from_keys_or(
131 - config, ["packing_format"], "auto_round:auto_gptq"
132 - ),
133 - block_name_to_quantize=cls.get_from_keys_or(
134 - config, ["block_name_to_quantize", "to_quant_block_names"], None
135 - ),
136 - extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
137 - data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
138 - backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
139 - )
140 -
141 - def get_layer_config(self, layer, layer_name: str):
142 - def get_config(name: str, quantized: bool = True):
143 - if not self.extra_config:
144 - return (
145 - self.weight_bits if quantized else 16,
146 - self.group_size if quantized else -1,
147 - self.sym if quantized else True,
148 - )
149 -
150 - # exact match first
151 - if name in self.extra_config:
152 - cfg = self.extra_config[name]
153 - return (
154 - cfg.get("bits", self.weight_bits if quantized else 16),
155 - cfg.get("group_size", self.group_size if quantized else -1),
156 - cfg.get("sym", self.sym if quantized else True),
157 - )
158 -
159 - REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
160 - for pattern, cfg in self.extra_config.items():
161 - if not isinstance(pattern, str) or not any(
162 - c in REGEX_SPECIAL_CHARS for c in pattern
163 - ):
164 - continue
165 -
166 - try:
167 - if re.search(re.compile(pattern), name) is not None:
168 - return (
169 - cfg.get("bits", self.weight_bits if quantized else 16),
170 - cfg.get("group_size", self.group_size if quantized else -1),
171 - cfg.get("sym", self.sym if quantized else True),
172 - )
173 - except re.error:
174 - # Invalid regex, ignore.
175 - continue
176 -
177 - return (
178 - self.weight_bits if quantized else 16,
179 - self.group_size if quantized else -1,
180 - self.sym if quantized else True,
181 - )
182 -
183 - # 1. Exact match from config
184 - if self.extra_config and layer_name in self.extra_config:
185 - return get_config(layer_name)
186 -
187 - # 2. Determine whether layer should be quantized
188 - quantized = not isinstance(layer, ParallelLMHead)
189 - if self.block_name_to_quantize:
190 - quantized = any(
191 - layer_name.startswith(name) for name in self.block_name_to_quantize
192 - )
193 -
194 - # 3. Handle fused MoE
195 - if self.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
196 - moe_configs = [
197 - get_config(name, quantized)
198 - for name in self.extra_config
199 - if name.startswith(layer_name)
200 - ]
201 - if moe_configs:
202 - if len(set(moe_configs)) == 1:
203 - return moe_configs[0]
204 - raise ValueError(
205 - f"Fused MoE layer '{layer_name}' requires "
206 - f"consistent quant config for all sub-layers"
207 - )
208 -
209 - # 4. Handle fused QKV or other patterns
210 - if self.extra_config:
211 - for fusion_key, sub_keys in self.packed_modules_mapping.items():
212 - if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
213 - sub_names = [
214 - layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
215 - ]
216 - sub_configs = [get_config(name, quantized) for name in sub_names]
217 - if len(set(sub_configs)) == 1:
218 - return sub_configs[0]
219 - raise ValueError(
220 - f"Fused module '{layer_name}' requires "
221 - f"consistent quant config for {sub_names}"
222 - )
223 -
224 - # 5. Fallback or try a regular expression match
225 - return get_config(layer_name, quantized)
226 -
227 - def check_quantized(self, weight_bits: int) -> bool:
228 - return weight_bits < 16
229 -
230 - def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
231 - if self.block_name_to_quantize is not None:
232 - self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
233 - self.block_name_to_quantize
234 - )
235 - if self.extra_config is not None:
236 - self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
237 -
238 - def apply_awq_quant_layer(self, layer, prefix: str, backend: str = "auto"):
239 - from vllm.model_executor.layers.quantization.utils.marlin_utils import (
240 - check_marlin_supported,
241 - check_moe_marlin_supports_layer,
242 - )
243 -
244 - weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
245 - if not self.check_quantized(weight_bits):
246 - if isinstance(layer, (LinearBase, ParallelLMHead)):
247 - return UnquantizedLinearMethod()
248 - else:
249 - return None
250 -
251 - logger.debug(
252 - "[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
253 - prefix,
254 - layer.__class__.__name__,
255 - weight_bits,
256 - group_size,
257 - sym,
258 - )
259 - if backend == "auto" or "marlin" in backend:
260 - AWQ_TYPE_MAP = {
261 - 4: scalar_types.uint4,
262 - 8: scalar_types.uint8,
263 - }
264 - use_marlin = (weight_bits in AWQ_TYPE_MAP) and check_marlin_supported(
265 - AWQ_TYPE_MAP[weight_bits], group_size, not sym
266 - )
267 -
268 - if isinstance(layer, RoutedExperts):
269 - use_marlin = use_marlin and check_moe_marlin_supports_layer(
270 - layer, group_size
271 - )
272 -
273 - else:
274 - use_marlin = False
275 - if use_marlin:
276 - from vllm.model_executor.layers.quantization.awq_marlin import (
277 - AWQMarlinConfig,
278 - AWQMarlinLinearMethod,
279 - AWQMarlinMoEMethod,
280 - )
281 -
282 - quant_args_marlin = AWQMarlinConfig(
283 - weight_bits=weight_bits,
284 - group_size=group_size,
285 - zero_point=not sym,
286 - lm_head_quantized=False,
287 - full_config={},
288 - modules_to_not_convert=[],
289 - )
290 - else:
291 - from vllm.model_executor.layers.quantization.awq import (
292 - AWQConfig,
293 - AWQLinearMethod,
294 - )
295 -
296 - quant_args = AWQConfig(
297 - weight_bits=weight_bits,
298 - group_size=group_size,
299 - zero_point=not sym,
300 - )
301 -
302 - if isinstance(layer, RoutedExperts):
303 - if use_marlin:
304 - return AWQMarlinMoEMethod(quant_args_marlin, layer.moe_config)
305 - from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
306 -
307 - config = {
308 - "quant_method": "awq",
309 - "bits": weight_bits,
310 - "group_size": group_size,
311 - "zero_point": not sym,
312 - "lm_head": False,
313 - }
314 - return MoeWNA16Config.from_config(config).get_quant_method(layer, prefix)
315 -
316 - if isinstance(layer, (LinearBase, ParallelLMHead)):
317 - if use_marlin:
318 - return AWQMarlinLinearMethod(quant_args_marlin)
319 - else:
320 - return AWQLinearMethod(quant_args)
321 - return None
322 -
323 - def apply_gptq_quant_layer(self, layer, prefix: str, backend: str = "auto"):
324 - from vllm.model_executor.layers.quantization.utils.marlin_utils import (
325 - check_marlin_supported,
326 - check_moe_marlin_supports_layer,
327 - )
328 -
329 - weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
330 - if not self.check_quantized(weight_bits):
331 - if isinstance(layer, (LinearBase, ParallelLMHead)):
332 - return UnquantizedLinearMethod()
333 - else:
334 - return None
335 -
336 - logger.debug(
337 - "[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
338 - prefix,
339 - layer.__class__.__name__,
340 - weight_bits,
341 - group_size,
342 - sym,
343 - )
344 - if backend == "auto" or "marlin" in backend:
345 - GPTQ_TYPE_MAP = {
346 - (4, True): scalar_types.uint4b8,
347 - (8, True): scalar_types.uint8b128,
348 - }
349 - use_marlin = (weight_bits, sym) in GPTQ_TYPE_MAP and check_marlin_supported(
350 - GPTQ_TYPE_MAP[(weight_bits, sym)], group_size, has_zp=not sym
351 - )
352 - if isinstance(layer, RoutedExperts):
353 - use_marlin = use_marlin and check_moe_marlin_supports_layer(
354 - layer, group_size
355 - )
356 - else:
357 - use_marlin = False
358 - if use_marlin:
359 - from vllm.model_executor.layers.quantization.auto_gptq import (
360 - AutoGPTQConfig,
361 - AutoGPTQLinearMethod,
362 - AutoGPTQMoEMethod,
363 - )
364 -
365 - quant_args_marlin = AutoGPTQConfig(
366 - weight_bits=weight_bits,
367 - group_size=group_size,
368 - is_sym=sym,
369 - lm_head_quantized=False,
370 - desc_act=False,
371 - dynamic={},
372 - full_config={},
373 - )
374 -
375 - if isinstance(layer, RoutedExperts):
376 - if use_marlin:
377 - return AutoGPTQMoEMethod(quant_args_marlin, layer.moe_config)
378 - else:
379 - from vllm.model_executor.layers.quantization.moe_wna16 import (
380 - MoeWNA16Config,
381 - )
382 -
383 - config = {
384 - "quant_method": "gptq",
385 - "bits": weight_bits,
386 - "group_size": group_size,
387 - "sym": sym,
388 - "lm_head": False,
389 - }
390 - return MoeWNA16Config.from_config(config).get_quant_method(
391 - layer, prefix
392 - )
393 -
394 - if isinstance(layer, (LinearBase, ParallelLMHead)):
395 - if use_marlin:
396 - return AutoGPTQLinearMethod(quant_args_marlin)
397 - else:
398 - raise NotImplementedError(
399 - f"INC quantization with bits={weight_bits}, sym={sym} "
400 - "is not supported. Only 4-bit and 8-bit symmetric "
401 - "quantization is supported with Marlin kernels."
402 - )
403 -
404 - return None
405 -
406 - def apply_xpu_w4a16_quant_layer(self, layer, prefix: str):
407 - weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
408 -
409 - if not self.check_quantized(weight_bits):
410 - if isinstance(layer, (LinearBase, ParallelLMHead)):
411 - return UnquantizedLinearMethod()
412 - else:
413 - return None
414 -
415 - if weight_bits != 4:
416 - raise NotImplementedError(
417 - f"INC on XPU only supports 4-bit quantization, "
418 - f"got weight_bits={weight_bits}."
419 - )
420 - if not sym:
421 - raise NotImplementedError(
422 - "INC W4A16 on XPU only supports symmetric quantization for now."
423 - )
424 -
425 - if isinstance(layer, (LinearBase, ParallelLMHead)):
426 - is_ark_available, ark_error, _, _ = get_ark_state()
427 - if is_ark_available:
428 - return INCARKLinearMethod(
429 - weight_bits=weight_bits,
430 - group_size=group_size,
431 - sym=sym,
432 - )
433 -
434 - logger.debug(
435 - "ARK backend is unavailable for layer %s; "
436 - "falling back to the default XPU INC path. Error: %s",
437 - prefix,
438 - ark_error or "unknown error",
439 - )
440 -
441 - return INCXPULinearMethod(
442 - weight_bits=weight_bits,
443 - group_size=group_size,
444 - sym=sym,
445 - )
446 - return None
447 -
448 - def apply_cpu_w4a16_quant_layer(self, layer, prefix: str):
449 - weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
450 - if not self.check_quantized(weight_bits):
451 - if isinstance(layer, (LinearBase, ParallelLMHead)):
452 - return UnquantizedLinearMethod()
453 - else:
454 - return None
455 -
456 - if weight_bits != 4:
457 - raise NotImplementedError(
458 - f"INC on CPU only supports 4-bit quantization, "
459 - f"got weight_bits={weight_bits}."
460 - )
461 - if not sym:
462 - raise NotImplementedError(
463 - "INC W4A16 on CPU only supports symmetric quantization for now."
464 - )
465 - if isinstance(layer, (LinearBase, ParallelLMHead)):
466 - is_ark_available, ark_error, _, _ = get_ark_state()
467 - if is_ark_available:
468 - return INCARKLinearMethod(
469 - weight_bits=weight_bits,
470 - group_size=group_size,
471 - sym=sym,
472 - )
473 -
474 - logger.debug(
475 - "ARK backend is unavailable for layer %s; "
476 - "falling back to the default CPU INC path. Error: %s",
477 - prefix,
478 - ark_error or "unknown error",
479 - )
480 -
481 - return self.apply_gptq_quant_layer(layer, prefix)
482 - return None
483 -
484 - def get_quant_method(self, layer: torch.nn.Module, prefix: str):
485 - if prefix and self.extra_config:
486 - for layer_name in self.extra_config:
487 - if (
488 - layer_name == prefix or layer_name == f"model.{prefix}"
489 - ) and self.extra_config[layer_name].get("bits", 16) >= 16:
490 - return UnquantizedLinearMethod()
491 -
492 - if current_platform.is_xpu():
493 - return self.apply_xpu_w4a16_quant_layer(layer, prefix)
494 - is_gptq = "gptq" in self.packing_format or "gptq" in self.backend
495 - if current_platform.is_cpu() and is_gptq:
496 - return self.apply_cpu_w4a16_quant_layer(layer, prefix)
497 - if is_gptq:
498 - return self.apply_gptq_quant_layer(layer, prefix)
499 - if "awq" in self.packing_format or "awq" in self.backend:
500 - return self.apply_awq_quant_layer(layer, prefix)
501 -
502 - raise NotImplementedError(
503 - f"Unsupported quantization configuration for layer '{prefix}'. "
504 - f"Platform: CPU={current_platform.is_cpu()}. "
505 - f"Platform: XPU={current_platform.is_xpu()}. "
506 - f"Format: {self.packing_format}, Backend: {self.backend}."
507 - )
508 -
509 - @classmethod
510 - def override_quantization_method(
511 - cls, hf_quant_cfg, user_quant, hf_config=None
512 - ) -> "QuantizationMethods | None":
513 - """Override the `auto-round` method to `inc`."""
514 - is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
515 - if is_auto_round_format:
516 - return cls.get_name()
517 - return None
518 -
519 -
520 -class INCXPULinearBase(LinearMethodBase):
521 - def __init__(self, weight_bits: int, group_size: int, sym: bool):
522 - self.weight_bits = weight_bits
523 - self.group_size = group_size
524 - self.sym = sym
525 - self.pack_factor = 32 // weight_bits
526 -
527 - def _create_inc_weights(
528 - self,
529 - layer: torch.nn.Module,
530 - input_size_per_partition: int,
531 - output_partition_sizes: list[int],
532 - params_dtype: torch.dtype,
533 - weight_loader: Any,
534 - group_size: int,
535 - pack_factor: int,
536 - ) -> None:
537 - output_size_per_partition = sum(output_partition_sizes)
538 - scales_and_zp_size = input_size_per_partition // group_size
539 -
540 - qweight = PackedvLLMParameter(
541 - data=torch.empty(
542 - input_size_per_partition // pack_factor,
543 - output_size_per_partition,
544 - dtype=torch.int32,
545 - ),
546 - input_dim=0,
547 - output_dim=1,
548 - packed_dim=0,
549 - packed_factor=pack_factor,
550 - weight_loader=weight_loader,
551 - )
552 -
553 - scales = GroupQuantScaleParameter(
554 - data=torch.empty(
555 - scales_and_zp_size,
556 - output_size_per_partition,
557 - dtype=params_dtype,
558 - ),
559 - input_dim=0,
560 - output_dim=1,
561 - weight_loader=weight_loader,
562 - )
563 -
564 - qzeros = PackedvLLMParameter(
565 - data=torch.empty(
566 - scales_and_zp_size,
567 - output_size_per_partition // pack_factor,
568 - dtype=torch.int32,
569 - ),
570 - input_dim=0,
571 - output_dim=1,
572 - packed_dim=1,
573 - packed_factor=pack_factor,
574 - weight_loader=weight_loader,
575 - )
576 -
577 - layer.register_parameter("qweight", qweight)
578 - layer.register_parameter("scales", scales)
579 - layer.register_parameter("qzeros", qzeros)
580 -
581 - g_idx = RowvLLMParameter(
582 - data=torch.tensor(
583 - [i // group_size for i in range(input_size_per_partition)],
584 - dtype=torch.int32,
585 - ),
586 - input_dim=0,
587 - weight_loader=weight_loader,
588 - )
589 - layer.register_parameter("g_idx", g_idx)
590 -
591 - def create_weights(
592 - self,
593 - layer: torch.nn.Module,
594 - input_size_per_partition: int,
595 - output_partition_sizes: list[int],
596 - input_size: int,
597 - output_size: int,
598 - params_dtype: torch.dtype,
599 - **extra_weight_attrs,
600 - ):
601 - self._create_inc_weights(
602 - layer=layer,
603 - input_size_per_partition=input_size_per_partition,
604 - output_partition_sizes=output_partition_sizes,
605 - params_dtype=params_dtype,
606 - weight_loader=extra_weight_attrs.get("weight_loader"),
607 - group_size=self.group_size,
608 - pack_factor=self.pack_factor,
609 - )
610 -
611 -
612 -@lru_cache(maxsize=1)
613 -def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
614 - """Return ARK availability, error details, cached instance, and QuantLinear."""
615 - try:
616 - import auto_round_kernel
617 - from auto_round_kernel.qlinear import QuantLinear
618 -
619 - logger.info("Successfully imported auto_round_kernel.")
620 - except ImportError as error:
621 - return False, str(error), None, None
622 -
623 - ark_loader = getattr(auto_round_kernel, "_ark_instance", None)
624 - if not callable(ark_loader):
625 - return False, "auto_round_kernel does not expose _ark_instance().", None, None
626 -
627 - try:
628 - ark_instance = ark_loader()
629 - except Exception as error:
630 - return False, str(error), None, None
631 -
632 - if ark_instance is None:
633 - return False, "auto_round_kernel._ark_instance() returned None.", None, None
634 -
635 - return True, None, ark_instance, QuantLinear
636 -
637 -
638 -class INCXPULinearMethod(INCXPULinearBase):
639 - """XPU linear method for INC w4a16 GPTQ quantization (symmetric only).
640 -
641 - Repacks GPTQ weights from [in_packed, out] to oneDNN [out, in_packed]
642 - layout and calls torch.ops._xpu_C.int4_gemm_w4a16.
643 -
644 - GPTQ format: qweight [in_packed, out] with sequential nibble order.
645 -
646 - Note: Asymmetric quantization (sym=false) is not for now.
647 -
648 - FIXME(yiliu30): Refine the implementation to reuse XPUwNa16LinearKernel.
649 - """
650 -
651 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
652 - """Repack GPTQ weights into kernel-ready NT layout."""
653 - device = layer.qweight.data.device
654 -
655 - # oneDNN int4 kernel requires strides[0]==1 ("NT format"), but GPTQ
656 - # checkpoint is [K_packed, N] contiguous with strides (N, 1).
657 - # Two transposes are needed — neither alone can achieve this:
658 - # 1. .t().contiguous() → [N, K_packed] contiguous in memory
659 - # 2. .t() → [K_packed, N] view with strides (1, K_packed)
660 - # The result has the same logical shape but strides[0]==1 as required.
661 - qweight_ct = layer.qweight.data.t().contiguous()
662 - layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
663 -
664 - # Scales: [num_groups, out] — no change needed
665 - layer.scales = Parameter(layer.scales.data, requires_grad=False)
666 -
667 - # Symmetric: GPTQ v1 stores qzeros=7, effective zp = 7+1 = 8
668 - # Kernel expects int8 scalar = 8
669 - layer.qzeros = Parameter(
670 - torch.tensor([8], dtype=torch.int8, device=device),
671 - requires_grad=False,
672 - )
673 -
674 - def apply(
675 - self,
676 - layer: torch.nn.Module,
677 - x: torch.Tensor,
678 - bias: torch.Tensor | None = None,
679 - ) -> torch.Tensor:
680 - # qweight is already in NT layout [K_packed, N] (strides (1, K_packed))
681 - # from process_weights_after_loading — pass directly to kernel.
682 - out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
683 - reshaped_x = x.reshape(-1, x.shape[-1])
684 - out = torch.ops._xpu_C.int4_gemm_w4a16(
685 - reshaped_x,
686 - layer.qweight,
687 - bias,
688 - layer.scales,
689 - layer.qzeros,
690 - self.group_size,
691 - None, # g_idx not needed: desc_act is always False for INC models
692 - )
693 - return out.reshape(out_shape)
694 -
695 -
696 -class INCARKLinearMethod(INCXPULinearBase):
697 - """XPU & CPU w4a16 linear method for INC quantization utilizing the ARK backend.
698 -
699 - See: https://github.com/intel/auto-round/blob/main/auto_round_extension/ark/README.md
700 -
701 - Repacks GPTQ/INC weights into ARK's layout.
702 - """
703 -
704 - def __init__(self, weight_bits: int, group_size: int, sym: bool):
705 - super().__init__(weight_bits=weight_bits, group_size=group_size, sym=sym)
706 -
707 - is_available, error_str, _, quant_linear_cls = get_ark_state()
708 - if not is_available or quant_linear_cls is None:
709 - reason = error_str or "unknown error"
710 - raise ImportError(f"Failed to import auto_round_kernel. {reason}")
711 -
712 - self.QuantLinear = quant_linear_cls
713 -
714 - def create_weights(
715 - self,
716 - layer: torch.nn.Module,
717 - input_size_per_partition: int,
718 - output_partition_sizes: list[int],
719 - input_size: int,
720 - output_size: int,
721 - params_dtype: torch.dtype,
722 - **extra_weight_attrs,
723 - ):
724 - super().create_weights(
725 - layer=layer,
726 - input_size_per_partition=input_size_per_partition,
727 - output_partition_sizes=output_partition_sizes,
728 - input_size=input_size,
729 - output_size=output_size,
730 - params_dtype=params_dtype,
731 - **extra_weight_attrs,
732 - )
733 - layer.in_features = input_size_per_partition
734 - layer.out_features = sum(output_partition_sizes)
735 - layer.params_dtype = params_dtype
736 -
737 - def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
738 - if hasattr(layer, "input_size_per_partition"):
739 - in_features = layer.input_size_per_partition
740 - elif hasattr(layer, "input_size"):
741 - in_features = layer.input_size
742 - else:
743 - raise AttributeError("Cannot determine in_features for layer.")
744 -
745 - if hasattr(layer, "output_partition_sizes"):
746 - out_features = sum(layer.output_partition_sizes)
747 - elif hasattr(layer, "output_size_per_partition"):
748 - out_features = layer.output_size_per_partition
749 - elif hasattr(layer, "output_size"):
750 - out_features = layer.output_size
751 - else:
752 - out_features = layer.scales.shape[-1]
753 -
754 - ark_linear = self.QuantLinear(
755 - bits=self.weight_bits,
756 - group_size=self.group_size,
757 - sym=self.sym,
758 - in_features=in_features,
759 - out_features=out_features,
760 - bias=layer.bias is not None,
761 - weight_dtype=layer.params_dtype,
762 - )
763 -
764 - ark_linear.to(layer.qweight.device)
765 -
766 - with torch.no_grad():
767 - ark_linear.qweight.copy_(layer.qweight.detach())
768 -
769 - if hasattr(layer, "qzeros") and layer.qzeros is not None:
770 - ark_linear.qzeros.copy_(layer.qzeros.detach())
771 - else:
772 - ark_linear.qzeros = None
773 -
774 - ark_linear.scales.copy_(layer.scales.detach())
775 -
776 - if hasattr(layer, "bias") and layer.bias is not None:
777 - ark_linear.bias.copy_(layer.bias.detach())
778 -
779 - ark_linear.post_init()
780 -
781 - layer.ark_linear = ark_linear
782 -
783 - del layer.qweight
784 - if hasattr(layer, "qzeros"):
785 - del layer.qzeros
786 - del layer.scales
787 -
788 - def apply(
789 - self,
790 - layer: torch.nn.Module,
791 - x: torch.Tensor,
792 - bias: torch.Tensor | None = None,
793 - ) -> torch.Tensor:
794 - return layer.ark_linear.forward(x)
vllm/model_executor/layers/quantization/inc/__init__.py new
+6
@@ -0,0 +1,6 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from .inc import INCConfig
5 +
6 +__all__ = ["INCConfig"]
vllm/model_executor/layers/quantization/inc/config_parser.py new
+188
@@ -0,0 +1,188 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from dataclasses import dataclass
5 +from typing import TYPE_CHECKING
6 +
7 +import regex as re
8 +
9 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
10 +
11 +if TYPE_CHECKING:
12 + import torch
13 +
14 + from .inc import INCConfig
15 +
16 +
17 +@dataclass(frozen=True)
18 +class INCLayerConfig:
19 + bits: int
20 + group_size: int
21 + sym: bool
22 + packing_format: str
23 + backend: str
24 + data_type: str
25 + quantized: bool
26 +
27 + @property
28 + def is_gptq(self) -> bool:
29 + return "gptq" in self.packing_format or "gptq" in self.backend
30 +
31 + @property
32 + def is_awq(self) -> bool:
33 + return "awq" in self.packing_format or "awq" in self.backend
34 +
35 + @property
36 + def is_wna16_int(self) -> bool:
37 + return self.data_type == "int" and self.quantized
38 +
39 + @property
40 + def is_mxfp4(self) -> bool:
41 + return self.data_type == "mx_fp" and self.bits == 4
42 +
43 + @property
44 + def is_mxfp8(self) -> bool:
45 + return self.data_type == "mx_fp" and self.bits == 8
46 +
47 +
48 +class INCConfigParser:
49 + def __init__(self, config: "INCConfig") -> None:
50 + self._config = config
51 +
52 + def resolve(self, layer: "torch.nn.Module", layer_name: str) -> INCLayerConfig:
53 + bits, group_size, sym = self._resolve_raw(layer, layer_name)
54 + return INCLayerConfig(
55 + bits=bits,
56 + group_size=group_size,
57 + sym=sym,
58 + packing_format=self._config.packing_format,
59 + backend=self._config.backend,
60 + data_type=self._config.data_type,
61 + quantized=bits < 16,
62 + )
63 +
64 + def get_layer_config(
65 + self, layer: "torch.nn.Module", layer_name: str
66 + ) -> tuple[int, int, bool]:
67 + layer_config = self.resolve(layer, layer_name)
68 + return layer_config.bits, layer_config.group_size, layer_config.sym
69 +
70 + def _resolve_raw(
71 + self, layer: "torch.nn.Module", layer_name: str
72 + ) -> tuple[int, int, bool]:
73 + REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
74 +
75 + def is_explicitly_configured(name: str) -> bool:
76 + """Return True if *name* has an explicit entry in extra_config,
77 + either via exact key match or via a regex pattern key."""
78 + if not self._config.extra_config:
79 + return False
80 + if name in self._config.extra_config:
81 + return True
82 + for pattern in self._config.extra_config:
83 + if not isinstance(pattern, str) or not any(
84 + c in REGEX_SPECIAL_CHARS for c in pattern
85 + ):
86 + continue
87 + try:
88 + if re.search(re.compile(pattern), name) is not None:
89 + return True
90 + except re.error:
91 + continue
92 + return False
93 +
94 + def get_config(name: str, quantized: bool = True) -> tuple[int, int, bool]:
95 + if not self._config.extra_config:
96 + return (
97 + self._config.weight_bits if quantized else 16,
98 + self._config.group_size if quantized else -1,
99 + self._config.sym if quantized else True,
100 + )
101 +
102 + if name in self._config.extra_config:
103 + cfg = self._config.extra_config[name]
104 + return (
105 + cfg.get("bits", self._config.weight_bits if quantized else 16),
106 + cfg.get(
107 + "group_size",
108 + self._config.group_size if quantized else -1,
109 + ),
110 + cfg.get("sym", self._config.sym if quantized else True),
111 + )
112 +
113 + regex_special_chars = set(r"*+?^$()[]{}|\\")
114 + for pattern, cfg in self._config.extra_config.items():
115 + if not isinstance(pattern, str) or not any(
116 + c in regex_special_chars for c in pattern
117 + ):
118 + continue
119 +
120 + try:
121 + if re.search(re.compile(pattern), name) is not None:
122 + return (
123 + cfg.get(
124 + "bits",
125 + self._config.weight_bits if quantized else 16,
126 + ),
127 + cfg.get(
128 + "group_size",
129 + self._config.group_size if quantized else -1,
130 + ),
131 + cfg.get("sym", self._config.sym if quantized else True),
132 + )
133 + except re.error:
134 + continue
135 +
136 + return (
137 + self._config.weight_bits if quantized else 16,
138 + self._config.group_size if quantized else -1,
139 + self._config.sym if quantized else True,
140 + )
141 +
142 + if self._config.extra_config and layer_name in self._config.extra_config:
143 + return get_config(layer_name)
144 +
145 + quantized = not isinstance(layer, ParallelLMHead)
146 + if self._config.block_name_to_quantize:
147 + quantized = any(
148 + layer_name.startswith(name)
149 + for name in self._config.block_name_to_quantize
150 + )
151 +
152 + if self._config.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
153 + moe_configs = [
154 + get_config(name, quantized)
155 + for name in self._config.extra_config
156 + if name.startswith(layer_name)
157 + ]
158 + if moe_configs:
159 + if len(set(moe_configs)) == 1:
160 + return moe_configs[0]
161 + raise ValueError(
162 + f"Fused MoE layer '{layer_name}' requires "
163 + f"consistent quant config for all sub-layers"
164 + )
165 +
166 + if self._config.extra_config:
167 + for fusion_key, sub_keys in self._config.packed_modules_mapping.items():
168 + if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
169 + sub_names = [
170 + layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
171 + ]
172 + # Only trigger if at least one sub_name is explicitly
173 + # configured in extra_config (via exact match or regex).
174 + # This prevents false matches when a short fusion_key
175 + # (e.g. "qkv") is merely a substring of a longer layer
176 + # name (e.g. "in_proj_qkvz") and none of the generated
177 + # sub_names are actually configured.
178 + if not any(is_explicitly_configured(n) for n in sub_names):
179 + continue
180 + sub_configs = [get_config(name, quantized) for name in sub_names]
181 + if len(set(sub_configs)) == 1:
182 + return sub_configs[0]
183 + raise ValueError(
184 + f"Fused module '{layer_name}' requires "
185 + f"consistent quant config for {sub_names}"
186 + )
187 +
188 + return get_config(layer_name, quantized)
vllm/model_executor/layers/quantization/inc/inc.py new
+192
@@ -0,0 +1,192 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from fractions import Fraction
5 +from typing import TYPE_CHECKING, Any
6 +
7 +import torch
8 +
9 +from vllm.logger import init_logger
10 +from vllm.model_executor.layers.fused_moe import (
11 + RoutedExperts,
12 + UnquantizedFusedMoEMethod,
13 +)
14 +from vllm.model_executor.layers.linear import (
15 + LinearBase,
16 + UnquantizedLinearMethod,
17 +)
18 +from vllm.model_executor.layers.quantization import (
19 + QuantizationConfig,
20 + QuantizationMethods,
21 +)
22 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23 +
24 +from .config_parser import INCConfigParser
25 +
26 +if TYPE_CHECKING:
27 + from vllm.model_executor.models.utils import WeightsMapper
28 +
29 +logger = init_logger(__name__)
30 +
31 +
32 +class INCConfig(QuantizationConfig):
33 + """Config class for Intel Neural Compressor (INC).
34 + Repo: https://github.com/intel/neural-compressor
35 + """
36 +
37 + SUPPORTED_BITS = {2, 3, 4, 8}
38 + SUPPORTED_DTYPES = {"int"}
39 + SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
40 + SUPPORTED_BACKENDS = {
41 + "auto",
42 + "gptq",
43 + "gptq:marlin",
44 + "awq",
45 + "awq:marlin",
46 + "marlin",
47 + }
48 +
49 + def __init__(
50 + self,
51 + weight_bits: int,
52 + group_size: int,
53 + sym: bool = True,
54 + packing_format: str = "auto_round:auto_gptq",
55 + block_name_to_quantize: str | list[str] | None = None,
56 + extra_config: dict[str, Any] | None = None,
57 + data_type: str = "int",
58 + backend: str = "auto",
59 + ) -> None:
60 + super().__init__()
61 + if weight_bits not in self.SUPPORTED_BITS:
62 + raise ValueError(
63 + f"Unsupported weight_bits: {weight_bits}, "
64 + f"currently only support {self.SUPPORTED_BITS}."
65 + )
66 + if data_type not in self.SUPPORTED_DTYPES:
67 + raise ValueError(
68 + f"Unsupported data_type: {data_type},"
69 + f" currently only support {self.SUPPORTED_DTYPES}."
70 + )
71 + if packing_format not in self.SUPPORTED_FORMATS:
72 + raise ValueError(
73 + f"Unsupported packing_format: {packing_format}, "
74 + f"currently only support {self.SUPPORTED_FORMATS}."
75 + )
76 + if backend not in self.SUPPORTED_BACKENDS:
77 + raise ValueError(
78 + f"Unsupported backend: {backend}, "
79 + f"currently only support {self.SUPPORTED_BACKENDS}."
80 + )
81 +
82 + self.weight_bits = weight_bits
83 + self.group_size = group_size
84 + self.sym = sym
85 + self.packing_format = packing_format
86 + self.block_name_to_quantize = (
87 + block_name_to_quantize.split(",")
88 + if isinstance(block_name_to_quantize, str)
89 + else block_name_to_quantize
90 + )
91 + self.extra_config = extra_config
92 + self.data_type = data_type
93 + self.backend = backend
94 + self.pack_factor = Fraction(32, weight_bits)
95 + self.config_parser = INCConfigParser(self)
96 +
97 + def __repr__(self) -> str:
98 + return (
99 + f"INCConfig(weight_bits={self.weight_bits}, "
100 + f"group_size={self.group_size}, sym={self.sym})"
101 + )
102 +
103 + @classmethod
104 + def get_name(cls) -> QuantizationMethods:
105 + return "inc"
106 +
107 + @classmethod
108 + def get_supported_act_dtypes(cls) -> list[torch.dtype]:
109 + return [torch.half, torch.bfloat16]
110 +
111 + @classmethod
112 + def get_min_capability(cls) -> int:
113 + return 60
114 +
115 + @classmethod
116 + def get_config_filenames(cls) -> list[str]:
117 + return ["quantization_config.json"]
118 +
119 + @classmethod
120 + def from_config(cls, config: dict[str, Any]) -> "INCConfig":
121 + return cls(
122 + weight_bits=cls.get_from_keys(config, ["bits"]),
123 + group_size=cls.get_from_keys(config, ["group_size"]),
124 + sym=cls.get_from_keys(config, ["sym"]),
125 + packing_format=cls.get_from_keys_or(
126 + config, ["packing_format"], "auto_round:auto_gptq"
127 + ),
128 + block_name_to_quantize=cls.get_from_keys_or(
129 + config, ["block_name_to_quantize", "to_quant_block_names"], None
130 + ),
131 + extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
132 + data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
133 + backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
134 + )
135 +
136 + def get_layer_config(self, layer, layer_name: str):
137 + return self.config_parser.get_layer_config(layer, layer_name)
138 +
139 + def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
140 + if self.block_name_to_quantize is not None:
141 + self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
142 + self.block_name_to_quantize
143 + )
144 + if self.extra_config is not None:
145 + self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
146 +
147 + def get_quant_method(self, layer: torch.nn.Module, prefix: str):
148 + from .schemes.factory import resolve_scheme
149 +
150 + # Match original: check model.-prefixed names for unquantized layers
151 + if prefix and self.extra_config:
152 + for layer_name in self.extra_config:
153 + if (
154 + layer_name == prefix or layer_name == f"model.{prefix}"
155 + ) and self.extra_config[layer_name].get("bits", 16) >= 16:
156 + if isinstance(layer, RoutedExperts):
157 + return UnquantizedFusedMoEMethod(layer.moe_config)
158 + return UnquantizedLinearMethod()
159 +
160 + layer_config = self.config_parser.resolve(layer, prefix)
161 + if not layer_config.quantized:
162 + if isinstance(layer, (LinearBase, ParallelLMHead)):
163 + return UnquantizedLinearMethod()
164 + if isinstance(layer, RoutedExperts):
165 + return UnquantizedFusedMoEMethod(layer.moe_config)
166 + return None
167 +
168 + logger.debug(
169 + "[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
170 + prefix,
171 + layer.__class__.__name__,
172 + layer_config.bits,
173 + layer_config.group_size,
174 + layer_config.sym,
175 + )
176 +
177 + scheme = resolve_scheme(layer_config)
178 + if isinstance(layer, (LinearBase, ParallelLMHead)):
179 + return scheme.get_linear_method(self, layer, prefix, layer_config)
180 + if isinstance(layer, RoutedExperts):
181 + return scheme.get_moe_method(self, layer, prefix, layer_config)
182 + return None
183 +
184 + @classmethod
185 + def override_quantization_method(
186 + cls, hf_quant_cfg, user_quant, hf_config=None
187 + ) -> "QuantizationMethods | None":
188 + """Override the `auto-round` method to `inc`."""
189 + is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
190 + if is_auto_round_format:
191 + return cls.get_name()
192 + return None
vllm/model_executor/layers/quantization/inc/inc_linear.py new
+47
@@ -0,0 +1,47 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from typing import TYPE_CHECKING
5 +
6 +import torch
7 +
8 +from vllm.model_executor.layers.linear import LinearMethodBase
9 +
10 +if TYPE_CHECKING:
11 + from .schemes.inc_scheme import INCLinearScheme
12 +
13 +
14 +class INCLinearMethod(LinearMethodBase):
15 + def __init__(self, scheme: "INCLinearScheme") -> None:
16 + self.scheme = scheme
17 +
18 + def create_weights(
19 + self,
20 + layer: torch.nn.Module,
21 + input_size_per_partition: int,
22 + output_partition_sizes: list[int],
23 + input_size: int,
24 + output_size: int,
25 + params_dtype: torch.dtype,
26 + **extra_weight_attrs,
27 + ):
28 + return self.scheme.create_weights(
29 + layer=layer,
30 + input_size_per_partition=input_size_per_partition,
31 + output_partition_sizes=output_partition_sizes,
32 + input_size=input_size,
33 + output_size=output_size,
34 + params_dtype=params_dtype,
35 + **extra_weight_attrs,
36 + )
37 +
38 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
39 + return self.scheme.process_weights_after_loading(layer)
40 +
41 + def apply(
42 + self,
43 + layer: torch.nn.Module,
44 + x: torch.Tensor,
45 + bias: torch.Tensor | None = None,
46 + ) -> torch.Tensor:
47 + return self.scheme.apply_weights(layer, x, bias)
vllm/model_executor/layers/quantization/inc/schemes/__init__.py new
+13
@@ -0,0 +1,13 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from .factory import resolve_scheme
5 +from .inc_scheme import INCLinearScheme, INCScheme
6 +from .inc_wna16_scheme import INCWna16Scheme
7 +
8 +__all__ = [
9 + "INCScheme",
10 + "INCLinearScheme",
11 + "INCWna16Scheme",
12 + "resolve_scheme",
13 +]
vllm/model_executor/layers/quantization/inc/schemes/factory.py new
+22
@@ -0,0 +1,22 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from typing import TYPE_CHECKING
5 +
6 +if TYPE_CHECKING:
7 + from ..config_parser import INCLayerConfig
8 + from .inc_scheme import INCScheme
9 +
10 +
11 +def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
12 + from .inc_wna16_scheme import INCWna16Scheme
13 +
14 + scheme_list: list[type[INCScheme]] = [
15 + INCWna16Scheme,
16 + ]
17 +
18 + for scheme_cls in scheme_list:
19 + if scheme_cls.can_handle(layer_config):
20 + return scheme_cls()
21 +
22 + raise NotImplementedError(f"No INC scheme found for layer config: {layer_config}")
vllm/model_executor/layers/quantization/inc/schemes/inc_scheme.py new
+104
@@ -0,0 +1,104 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from abc import ABC, abstractmethod
5 +from typing import TYPE_CHECKING
6 +
7 +if TYPE_CHECKING:
8 + import torch
9 +
10 + from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
11 + from vllm.model_executor.layers.linear import LinearMethodBase
12 + from vllm.model_executor.layers.quantization import QuantizationMethods
13 +
14 + from ..config_parser import INCLayerConfig
15 + from ..inc import INCConfig
16 +
17 +
18 +class INCScheme(ABC):
19 + """One class per quant type. Single registration point for the factory.
20 +
21 + Each subclass defines:
22 + - can_handle(): when does this scheme apply?
23 + - get_linear_method(): required — how to quantize Linear layers
24 + - get_moe_method(): optional — how to quantize MoE layers
25 + - get_kvcache_method(): optional — how to quantize KV cache
26 +
27 + Schemes that don't support MoE/KVCache inherit the default raise.
28 + """
29 +
30 + @staticmethod
31 + @abstractmethod
32 + def can_handle(layer_config: "INCLayerConfig") -> bool:
33 + raise NotImplementedError
34 +
35 + @abstractmethod
36 + def get_linear_method(
37 + self,
38 + config: "INCConfig",
39 + layer: "torch.nn.Module",
40 + prefix: str,
41 + layer_config: "INCLayerConfig",
42 + ) -> "LinearMethodBase":
43 + raise NotImplementedError
44 +
45 + def get_moe_method(
46 + self,
47 + config: "INCConfig",
48 + layer: "torch.nn.Module",
49 + prefix: str,
50 + layer_config: "INCLayerConfig",
51 + ) -> "FusedMoEMethodBase | None":
52 + """Optional. Override if this scheme supports MoE.
53 + Default raises NotImplementedError."""
54 + raise NotImplementedError(
55 + f"{type(self).__name__} does not support MoE layers. "
56 + f"Layer config: {layer_config}"
57 + )
58 +
59 + def get_kvcache_method(
60 + self,
61 + config: "INCConfig",
62 + layer: "torch.nn.Module",
63 + prefix: str,
64 + layer_config: "INCLayerConfig",
65 + ) -> "QuantizationMethods":
66 + """Optional. Override if this scheme supports KV cache quantization.
67 + Default raises NotImplementedError."""
68 + raise NotImplementedError(
69 + f"{type(self).__name__} does not support KV cache quantization. "
70 + f"Layer config: {layer_config}"
71 + )
72 +
73 +
74 +class INCLinearScheme(ABC):
75 + @classmethod
76 + @abstractmethod
77 + def get_min_capability(cls) -> int:
78 + raise NotImplementedError
79 +
80 + @abstractmethod
81 + def create_weights(
82 + self,
83 + layer: "torch.nn.Module",
84 + input_size_per_partition: int,
85 + output_partition_sizes: list[int],
86 + input_size: int,
87 + output_size: int,
88 + params_dtype: "torch.dtype",
89 + **extra_weight_attrs,
90 + ) -> None:
91 + raise NotImplementedError
92 +
93 + @abstractmethod
94 + def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
95 + raise NotImplementedError
96 +
97 + @abstractmethod
98 + def apply_weights(
99 + self,
100 + layer: "torch.nn.Module",
101 + x: "torch.Tensor",
102 + bias: "torch.Tensor | None" = None,
103 + ) -> "torch.Tensor":
104 + raise NotImplementedError
vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py new
+399
@@ -0,0 +1,399 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from functools import lru_cache
5 +from typing import TYPE_CHECKING, Any
6 +
7 +import torch
8 +from torch.nn.parameter import Parameter
9 +
10 +from vllm.logger import init_logger
11 +from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
12 +from vllm.model_executor.layers.quantization.awq import AWQConfig
13 +from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
14 +from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 + check_marlin_supported,
16 +)
17 +from vllm.model_executor.parameter import (
18 + GroupQuantScaleParameter,
19 + PackedvLLMParameter,
20 + RowvLLMParameter,
21 +)
22 +from vllm.scalar_type import scalar_types
23 +
24 +from .inc_scheme import INCLinearScheme
25 +
26 +logger = init_logger(__name__)
27 +
28 +if TYPE_CHECKING:
29 + from ..config_parser import INCLayerConfig
30 +
31 +
32 +@lru_cache(maxsize=1)
33 +def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
34 + """Return ARK availability, error details, cached module, and QuantLinear."""
35 + try:
36 + import auto_round_kernel as ark
37 + from auto_round_kernel.qlinear import QuantLinear
38 +
39 + logger.info("Successfully imported auto_round_kernel.")
40 + except ImportError as error:
41 + return False, str(error), None, None
42 +
43 + if getattr(ark, "cpu_lib", None) is None and getattr(ark, "xpu_lib", None) is None:
44 + return (
45 + False,
46 + "No ARK backend library is available.",
47 + None,
48 + None,
49 + )
50 + logger.info("Successfully loaded auto_round_kernel backend library.")
51 +
52 + return True, None, ark, QuantLinear
53 +
54 +
55 +class INCWNA16LinearScheme(INCLinearScheme):
56 + def __init__(self, layer_config: "INCLayerConfig") -> None:
57 + self.layer_config = layer_config
58 + self.inner_method = self._build_inner_method()
59 +
60 + @classmethod
61 + def get_min_capability(cls) -> int:
62 + return 60
63 +
64 + def _build_inner_method(self):
65 + if self.layer_config.is_gptq:
66 + return self._build_gptq_method()
67 + if self.layer_config.is_awq:
68 + return self._build_awq_method()
69 + raise NotImplementedError(
70 + f"WNA16 linear scheme does not support {self.layer_config}"
71 + )
72 +
73 + def _build_gptq_method(self):
74 + gptq_type_map = {
75 + (4, True): scalar_types.uint4b8,
76 + (8, True): scalar_types.uint8b128,
77 + }
78 + use_marlin = (
79 + self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
80 + ) and (self.layer_config.bits, self.layer_config.sym) in gptq_type_map
81 + if use_marlin:
82 + use_marlin = check_marlin_supported(
83 + gptq_type_map[(self.layer_config.bits, self.layer_config.sym)],
84 + self.layer_config.group_size,
85 + has_zp=not self.layer_config.sym,
86 + )
87 +
88 + if use_marlin:
89 + from vllm.model_executor.layers.quantization.auto_gptq import (
90 + AutoGPTQLinearMethod,
91 + )
92 +
93 + return AutoGPTQLinearMethod(
94 + AutoGPTQConfig(
95 + weight_bits=self.layer_config.bits,
96 + group_size=self.layer_config.group_size,
97 + desc_act=False,
98 + is_sym=self.layer_config.sym,
99 + lm_head_quantized=False,
100 + dynamic={},
101 + full_config={},
102 + )
103 + )
104 +
105 + raise NotImplementedError(
106 + f"INC quantization with bits={self.layer_config.bits}, "
107 + f"sym={self.layer_config.sym} is not supported. "
108 + "Only 4-bit and 8-bit symmetric quantization is supported "
109 + "with Marlin kernels."
110 + )
111 +
112 + def _build_awq_method(self):
113 + awq_type_map = {
114 + 4: scalar_types.uint4,
115 + 8: scalar_types.uint8,
116 + }
117 + use_marlin = (
118 + self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
119 + ) and self.layer_config.bits in awq_type_map
120 + if use_marlin:
121 + use_marlin = check_marlin_supported(
122 + awq_type_map[self.layer_config.bits],
123 + self.layer_config.group_size,
124 + not self.layer_config.sym,
125 + )
126 +
127 + if use_marlin:
128 + from vllm.model_executor.layers.quantization.awq_marlin import (
129 + AWQMarlinLinearMethod,
130 + )
131 +
132 + return AWQMarlinLinearMethod(
133 + AWQMarlinConfig(
134 + weight_bits=self.layer_config.bits,
135 + group_size=self.layer_config.group_size,
136 + zero_point=not self.layer_config.sym,
137 + lm_head_quantized=False,
138 + modules_to_not_convert=[],
139 + full_config={},
140 + )
141 + )
142 +
143 + from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
144 +
145 + return AWQLinearMethod(
146 + AWQConfig(
147 + weight_bits=self.layer_config.bits,
148 + group_size=self.layer_config.group_size,
149 + zero_point=not self.layer_config.sym,
150 + )
151 + )
152 +
153 + def create_weights(
154 + self,
155 + layer: "torch.nn.Module",
156 + input_size_per_partition: int,
157 + output_partition_sizes: list[int],
158 + input_size: int,
159 + output_size: int,
160 + params_dtype: "torch.dtype",
161 + **extra_weight_attrs,
162 + ) -> None:
163 + return self.inner_method.create_weights(
164 + layer=layer,
165 + input_size_per_partition=input_size_per_partition,
166 + output_partition_sizes=output_partition_sizes,
167 + input_size=input_size,
168 + output_size=output_size,
169 + params_dtype=params_dtype,
170 + **extra_weight_attrs,
171 + )
172 +
173 + def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
174 + return self.inner_method.process_weights_after_loading(layer)
175 +
176 + def apply_weights(
177 + self,
178 + layer: "torch.nn.Module",
179 + x: "torch.Tensor",
180 + bias: "torch.Tensor | None" = None,
181 + ) -> "torch.Tensor":
182 + return self.inner_method.apply(layer, x, bias)
183 +
184 +
185 +class INCXPULinearBase(INCLinearScheme):
186 + def __init__(self, layer_config: "INCLayerConfig") -> None:
187 + self.weight_bits = layer_config.bits
188 + self.group_size = layer_config.group_size
189 + self.sym = layer_config.sym
190 + self.pack_factor = 32 // self.weight_bits
191 +
192 + @classmethod
193 + def get_min_capability(cls) -> int:
194 + return 0
195 +
196 + def _create_inc_weights(
197 + self,
198 + layer: torch.nn.Module,
199 + input_size_per_partition: int,
200 + output_partition_sizes: list[int],
201 + params_dtype: torch.dtype,
202 + weight_loader: Any,
203 + ) -> None:
204 + output_size_per_partition = sum(output_partition_sizes)
205 + scales_and_zp_size = input_size_per_partition // self.group_size
206 +
207 + qweight = PackedvLLMParameter(
208 + data=torch.empty(
209 + input_size_per_partition // self.pack_factor,
210 + output_size_per_partition,
211 + dtype=torch.int32,
212 + ),
213 + input_dim=0,
214 + output_dim=1,
215 + packed_dim=0,
216 + packed_factor=self.pack_factor,
217 + weight_loader=weight_loader,
218 + )
219 + scales = GroupQuantScaleParameter(
220 + data=torch.empty(
221 + scales_and_zp_size,
222 + output_size_per_partition,
223 + dtype=params_dtype,
224 + ),
225 + input_dim=0,
226 + output_dim=1,
227 + weight_loader=weight_loader,
228 + )
229 + qzeros = PackedvLLMParameter(
230 + data=torch.empty(
231 + scales_and_zp_size,
232 + output_size_per_partition // self.pack_factor,
233 + dtype=torch.int32,
234 + ),
235 + input_dim=0,
236 + output_dim=1,
237 + packed_dim=1,
238 + packed_factor=self.pack_factor,
239 + weight_loader=weight_loader,
240 + )
241 +
242 + layer.register_parameter("qweight", qweight)
243 + layer.register_parameter("scales", scales)
244 + layer.register_parameter("qzeros", qzeros)
245 +
246 + g_idx = RowvLLMParameter(
247 + data=torch.tensor(
248 + [i // self.group_size for i in range(input_size_per_partition)],
249 + dtype=torch.int32,
250 + ),
251 + input_dim=0,
252 + weight_loader=weight_loader,
253 + )
254 + layer.register_parameter("g_idx", g_idx)
255 +
256 + def create_weights(
257 + self,
258 + layer: torch.nn.Module,
259 + input_size_per_partition: int,
260 + output_partition_sizes: list[int],
261 + input_size: int,
262 + output_size: int,
263 + params_dtype: torch.dtype,
264 + **extra_weight_attrs,
265 + ) -> None:
266 + del input_size, output_size
267 + self._create_inc_weights(
268 + layer=layer,
269 + input_size_per_partition=input_size_per_partition,
270 + output_partition_sizes=output_partition_sizes,
271 + params_dtype=params_dtype,
272 + weight_loader=extra_weight_attrs.get("weight_loader"),
273 + )
274 +
275 +
276 +class INCXPULinearMethod(INCXPULinearBase):
277 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
278 + device = layer.qweight.data.device
279 +
280 + qweight_ct = layer.qweight.data.t().contiguous()
281 + layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
282 + layer.scales = Parameter(layer.scales.data, requires_grad=False)
283 + layer.qzeros = Parameter(
284 + torch.tensor([8], dtype=torch.int8, device=device),
285 + requires_grad=False,
286 + )
287 +
288 + def apply_weights(
289 + self,
290 + layer: torch.nn.Module,
291 + x: torch.Tensor,
292 + bias: torch.Tensor | None = None,
293 + ) -> torch.Tensor:
294 + out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
295 + reshaped_x = x.reshape(-1, x.shape[-1])
296 + out = torch.ops._xpu_C.int4_gemm_w4a16(
297 + reshaped_x,
298 + layer.qweight,
299 + bias,
300 + layer.scales,
301 + layer.qzeros,
302 + self.group_size,
303 + None,
304 + )
305 + return out.reshape(out_shape)
306 +
307 +
308 +class INCARKLinearMethod(INCXPULinearBase):
309 + def __init__(self, layer_config: "INCLayerConfig") -> None:
310 + super().__init__(layer_config)
311 +
312 + is_available, error_str, _, quant_linear_cls = get_ark_state()
313 + if not is_available or quant_linear_cls is None:
314 + reason = error_str or "unknown error"
315 + raise ImportError(f"Failed to import auto_round_kernel. {reason}")
316 +
317 + self.quant_linear_cls = quant_linear_cls
318 +
319 + def create_weights(
320 + self,
321 + layer: torch.nn.Module,
322 + input_size_per_partition: int,
323 + output_partition_sizes: list[int],
324 + input_size: int,
325 + output_size: int,
326 + params_dtype: torch.dtype,
327 + **extra_weight_attrs,
328 + ) -> None:
329 + super().create_weights(
330 + layer=layer,
331 + input_size_per_partition=input_size_per_partition,
332 + output_partition_sizes=output_partition_sizes,
333 + input_size=input_size,
334 + output_size=output_size,
335 + params_dtype=params_dtype,
336 + **extra_weight_attrs,
337 + )
338 + layer.in_features = input_size_per_partition
339 + layer.out_features = sum(output_partition_sizes)
340 + layer.params_dtype = params_dtype
341 +
342 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
343 + if hasattr(layer, "input_size_per_partition"):
344 + in_features = layer.input_size_per_partition
345 + elif hasattr(layer, "input_size"):
346 + in_features = layer.input_size
347 + else:
348 + raise AttributeError("Cannot determine in_features for layer.")
349 +
350 + if hasattr(layer, "output_partition_sizes"):
351 + out_features = sum(layer.output_partition_sizes)
352 + elif hasattr(layer, "output_size_per_partition"):
353 + out_features = layer.output_size_per_partition
354 + elif hasattr(layer, "output_size"):
355 + out_features = layer.output_size
356 + else:
357 + out_features = layer.scales.shape[-1]
358 +
359 + ark_linear = self.quant_linear_cls(
360 + bits=self.weight_bits,
361 + group_size=self.group_size,
362 + sym=self.sym,
363 + in_features=in_features,
364 + out_features=out_features,
365 + bias=layer.bias is not None,
366 + weight_dtype=layer.params_dtype,
367 + )
368 + ark_linear.to(layer.qweight.device)
369 +
370 + with torch.no_grad():
371 + ark_linear.qweight.copy_(layer.qweight.detach())
372 + if hasattr(layer, "qzeros") and layer.qzeros is not None:
373 + ark_linear.qzeros.copy_(layer.qzeros.detach())
374 + else:
375 + ark_linear.qzeros = None
376 + ark_linear.scales.copy_(layer.scales.detach())
377 + if hasattr(layer, "bias") and layer.bias is not None:
378 + ark_linear.bias.copy_(layer.bias.detach())
379 +
380 + ark_linear.post_init()
381 + layer.ark_linear = ark_linear
382 +
383 + del layer.qweight
384 + if hasattr(layer, "qzeros"):
385 + del layer.qzeros
386 + del layer.scales
387 +
388 + def apply_weights(
389 + self,
390 + layer: torch.nn.Module,
391 + x: torch.Tensor,
392 + bias: torch.Tensor | None = None,
393 + ) -> torch.Tensor:
394 + del bias
395 + return layer.ark_linear.forward(x)
396 +
397 +
398 +class INCXPUW4A16LinearScheme(INCXPULinearMethod):
399 + pass
vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py new
+201
@@ -0,0 +1,201 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from typing import TYPE_CHECKING
5 +
6 +from vllm.logger import init_logger
7 +from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
8 +from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
9 +from vllm.platforms import current_platform
10 +from vllm.scalar_type import scalar_types
11 +
12 +from ..inc_linear import INCLinearMethod
13 +from .inc_scheme import INCScheme
14 +
15 +if TYPE_CHECKING:
16 + import torch
17 +
18 + from ..config_parser import INCLayerConfig
19 + from ..inc import INCConfig
20 +
21 +logger = init_logger(__name__)
22 +
23 +
24 +class INCWna16Scheme(INCScheme):
25 + @staticmethod
26 + def can_handle(layer_config: "INCLayerConfig") -> bool:
27 + return layer_config.is_wna16_int
28 +
29 + def get_linear_method(
30 + self,
31 + config: "INCConfig",
32 + layer: "torch.nn.Module",
33 + prefix: str,
34 + layer_config: "INCLayerConfig",
35 + ):
36 + del config, layer
37 + if current_platform.is_xpu():
38 + if layer_config.bits == 4 and layer_config.sym:
39 + from .inc_wna16_linear import (
40 + INCARKLinearMethod,
41 + INCXPULinearMethod,
42 + get_ark_state,
43 + )
44 +
45 + is_ark_available, ark_error, _, _ = get_ark_state()
46 + if is_ark_available:
47 + return INCLinearMethod(INCARKLinearMethod(layer_config))
48 +
49 + logger.debug(
50 + "ARK backend is unavailable for layer %s; "
51 + "falling back to the default XPU INC path. Error: %s",
52 + prefix,
53 + ark_error or "unknown error",
54 + )
55 + return INCLinearMethod(INCXPULinearMethod(layer_config))
56 + raise NotImplementedError(f"INC on XPU: unsupported config {layer_config}")
57 +
58 + if current_platform.is_cpu() and layer_config.is_gptq:
59 + if layer_config.bits == 4 and layer_config.sym:
60 + from .inc_wna16_linear import (
61 + INCARKLinearMethod,
62 + INCWNA16LinearScheme,
63 + get_ark_state,
64 + )
65 +
66 + is_ark_available, ark_error, _, _ = get_ark_state()
67 + if is_ark_available:
68 + return INCLinearMethod(INCARKLinearMethod(layer_config))
69 +
70 + logger.debug(
71 + "ARK backend is unavailable for layer %s; "
72 + "falling back to the default CPU INC path. Error: %s",
73 + prefix,
74 + ark_error or "unknown error",
75 + )
76 + return INCLinearMethod(INCWNA16LinearScheme(layer_config))
77 + raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
78 +
79 + from .inc_wna16_linear import INCWNA16LinearScheme
80 +
81 + return INCLinearMethod(INCWNA16LinearScheme(layer_config))
82 +
83 + def get_moe_method(
84 + self,
85 + config: "INCConfig",
86 + layer: "torch.nn.Module",
87 + prefix: str,
88 + layer_config: "INCLayerConfig",
89 + ):
90 + del config, prefix
91 + # XPU and CPU do not support MoE quantization yet
92 + if current_platform.is_xpu() or current_platform.is_cpu():
93 + from vllm.model_executor.layers.fused_moe import (
94 + UnquantizedFusedMoEMethod,
95 + )
96 +
97 + return UnquantizedFusedMoEMethod(layer.moe_config)
98 + if layer_config.is_gptq:
99 + return _resolve_gptq_moe(layer, layer_config)
100 + if layer_config.is_awq:
101 + return _resolve_awq_moe(layer, layer_config)
102 + raise NotImplementedError(f"WNA16 MoE does not support config {layer_config}")
103 +
104 +
105 +def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
106 + from vllm.model_executor.layers.quantization.auto_gptq import (
107 + AutoGPTQMoEMethod,
108 + )
109 + from vllm.model_executor.layers.quantization.moe_wna16 import (
110 + MoeWNA16Config,
111 + MoeWNA16Method,
112 + )
113 + from vllm.model_executor.layers.quantization.utils.marlin_utils import (
114 + check_marlin_supported,
115 + check_moe_marlin_supports_layer,
116 + )
117 +
118 + gptq_type_map = {
119 + (4, True): scalar_types.uint4b8,
120 + (8, True): scalar_types.uint8b128,
121 + }
122 + use_marlin = (layer_config.bits, layer_config.sym) in gptq_type_map
123 + if use_marlin:
124 + use_marlin = check_marlin_supported(
125 + gptq_type_map[(layer_config.bits, layer_config.sym)],
126 + layer_config.group_size,
127 + has_zp=not layer_config.sym,
128 + ) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
129 +
130 + if use_marlin:
131 + return AutoGPTQMoEMethod(
132 + AutoGPTQConfig(
133 + weight_bits=layer_config.bits,
134 + group_size=layer_config.group_size,
135 + desc_act=False,
136 + is_sym=layer_config.sym,
137 + lm_head_quantized=False,
138 + dynamic={},
139 + full_config={},
140 + ),
141 + layer.moe_config,
142 + )
143 +
144 + moe_config = MoeWNA16Config.from_config(
145 + {
146 + "quant_method": "gptq",
147 + "bits": layer_config.bits,
148 + "group_size": layer_config.group_size,
149 + "sym": layer_config.sym,
150 + "lm_head": False,
151 + }
152 + )
153 + return MoeWNA16Method(moe_config, layer.moe_config)
154 +
155 +
156 +def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
157 + from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinMoEMethod
158 + from vllm.model_executor.layers.quantization.moe_wna16 import (
159 + MoeWNA16Config,
160 + MoeWNA16Method,
161 + )
162 + from vllm.model_executor.layers.quantization.utils.marlin_utils import (
163 + check_marlin_supported,
164 + check_moe_marlin_supports_layer,
165 + )
166 +
167 + awq_type_map = {
168 + 4: scalar_types.uint4,
169 + 8: scalar_types.uint8,
170 + }
171 + use_marlin = layer_config.bits in awq_type_map
172 + if use_marlin:
173 + use_marlin = check_marlin_supported(
174 + awq_type_map[layer_config.bits],
175 + layer_config.group_size,
176 + not layer_config.sym,
177 + ) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
178 +
179 + if use_marlin:
180 + return AWQMarlinMoEMethod(
181 + AWQMarlinConfig(
182 + weight_bits=layer_config.bits,
183 + group_size=layer_config.group_size,
184 + zero_point=not layer_config.sym,
185 + lm_head_quantized=False,
186 + modules_to_not_convert=[],
187 + full_config={},
188 + ),
189 + layer.moe_config,
190 + )
191 +
192 + moe_config = MoeWNA16Config.from_config(
193 + {
194 + "quant_method": "awq",
195 + "bits": layer_config.bits,
196 + "group_size": layer_config.group_size,
197 + "zero_point": not layer_config.sym,
198 + "lm_head": False,
199 + }
200 + )
201 + return MoeWNA16Method(moe_config, layer.moe_config)