[Quantization][INC][ARK] Support INT2 XPU WOQ Linear (#47521)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Zhenzhong Xu committed
Jul 14, 2026 at 08:29 UTC
0b0ef8d7eb33cd1b9bbcf4fe3af348e6b3e44d57
3 files changed
+158
-5
requirements/xpu.txt
+1
-1
@@ -17,5 +17,5 @@ torchaudio
17
torchvision
18
torchcodec >= 0.14 # Required for the torchcodec video decoding backend
19
20
-auto_round_lib>=0.14.0
20
+auto_round_lib==0.14.1
21
vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.11/vllm_xpu_kernels-0.1.11-cp38-abi3-manylinux_2_28_x86_64.whl
tests/quantization/test_auto_round.py
+147
-3
@@ -9,6 +9,7 @@ Run `pytest tests/quantization/test_auto_round.py`.
9
"""
10
11
import pytest
12
+import torch
13
14
from vllm.model_executor.layers.fused_moe import RoutedExperts
15
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
@@ -48,8 +49,25 @@ MODELS = [
49
),
50
id="auto_round:auto_awq",
51
),
52
+ pytest.param(
53
+ "Intel/Qwen3-8B-w2g64-for-ut",
54
+ marks=pytest.mark.skipif(
55
+ not (current_platform.is_cuda() or current_platform.is_xpu())
56
+ or current_platform.device_count() < 2,
57
+ reason="72B INT2 AutoRound model requires XPU with at least 2 devices.",
58
+ ),
59
+ id="auto_round:auto_gptq_int2_tp2",
60
+ ),
61
]
62
63
+MODEL_RUNNER_KWARGS = {
64
+ "Intel/Qwen3-8B-w2g64-for-ut": {
65
+ "block_size": 64,
66
+ "gpu_memory_utilization": 0.8,
67
+ "max_model_len": 512,
68
+ },
69
+}
70
+
71
72
@pytest.mark.skipif(
73
not (
@@ -61,7 +79,7 @@ MODELS = [
79
)
80
@pytest.mark.parametrize("model", MODELS)
81
def test_auto_round_model(vllm_runner, model):
64
- with vllm_runner(model) as llm:
82
+ with vllm_runner(model, **MODEL_RUNNER_KWARGS.get(model, {})) as llm:
83
output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
84
85
assert output
@@ -351,6 +369,132 @@ def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
369
assert isinstance(method.scheme, INCARKLinearMethod)
370
371
372
+def test_inc_config_from_config_accepts_xpu_int2() -> None:
373
+ def _make_int2_raw_config(**overrides) -> dict[str, object]:
374
+ kwargs = {
375
+ "bits": 2,
376
+ "group_size": 64,
377
+ "sym": True,
378
+ "data_type": "int",
379
+ "quant_method": "auto-round",
380
+ }
381
+ kwargs.update(overrides)
382
+
383
+ return kwargs
384
+
385
+ config = INCConfig.from_config(_make_int2_raw_config())
386
+
387
+ assert config.weight_bits == 2
388
+ assert config.group_size == 64
389
+ assert config.sym is True
390
+ assert config.data_type == "int"
391
+ assert config.packing_format == "auto_round:auto_gptq"
392
+ assert config.backend == "auto"
393
+
394
+
395
+def test_wna16_xpu_int2_prefers_ark_when_available(monkeypatch) -> None:
396
+ class DummyQuantLinear:
397
+ pass
398
+
399
+ monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
400
+ monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
401
+ monkeypatch.setattr(
402
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
403
+ lambda: (True, None, object(), DummyQuantLinear),
404
+ )
405
+
406
+ method = INCWna16Scheme().get_linear_method(
407
+ make_config(weight_bits=2, group_size=64),
408
+ object(),
409
+ "layer",
410
+ make_layer_config(bits=2, group_size=64),
411
+ )
412
+
413
+ assert isinstance(method, INCLinearMethod)
414
+ assert isinstance(method.scheme, INCARKLinearMethod)
415
+
416
+
417
+def test_wna16_xpu_int2_requires_ark_when_unavailable(monkeypatch) -> None:
418
+ monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
419
+ monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
420
+ monkeypatch.setattr(
421
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
422
+ lambda: (False, "missing", None, None),
423
+ )
424
+
425
+ with pytest.raises(
426
+ NotImplementedError,
427
+ match="INC int2 on XPU requires the ARK backend",
428
+ ):
429
+ INCWna16Scheme().get_linear_method(
430
+ make_config(weight_bits=2, group_size=64),
431
+ object(),
432
+ "layer",
433
+ make_layer_config(bits=2, group_size=64),
434
+ )
435
+
436
+
437
+def test_wna16_xpu_int2_unsupported_config_still_raises(monkeypatch) -> None:
438
+ monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
439
+ monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
440
+
441
+ with pytest.raises(NotImplementedError, match="unsupported config"):
442
+ INCWna16Scheme().get_linear_method(
443
+ make_config(weight_bits=2, sym=False),
444
+ object(),
445
+ "layer",
446
+ make_layer_config(bits=2, sym=False),
447
+ )
448
+
449
+
450
+def test_inc_ark_linear_method_xpu_int2_create_weights(monkeypatch) -> None:
451
+ monkeypatch.setattr(
452
+ "vllm.model_executor.parameter.get_tensor_model_parallel_rank",
453
+ lambda: 0,
454
+ )
455
+ monkeypatch.setattr(
456
+ "vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
457
+ lambda: 1,
458
+ )
459
+
460
+ class DummyQuantLinear:
461
+ pass
462
+
463
+ class DummyLayer(torch.nn.Module):
464
+ pass
465
+
466
+ monkeypatch.setattr(
467
+ "vllm.model_executor.layers.quantization.inc.schemes.inc_ark_ops.get_ark_state",
468
+ lambda: (True, None, object(), DummyQuantLinear),
469
+ )
470
+
471
+ layer = DummyLayer()
472
+ method = INCARKLinearMethod(make_layer_config(bits=2, group_size=64))
473
+
474
+ method.create_weights(
475
+ layer=layer,
476
+ input_size_per_partition=64,
477
+ output_partition_sizes=[32, 32],
478
+ input_size=64,
479
+ output_size=64,
480
+ params_dtype=torch.bfloat16,
481
+ weight_loader=lambda *args, **kwargs: None,
482
+ )
483
+
484
+ assert method.pack_factor == 16
485
+ assert layer.qweight.shape == (4, 64)
486
+ assert layer.qweight.dtype == torch.int32
487
+ assert layer.scales.shape == (1, 64)
488
+ assert layer.scales.dtype == torch.bfloat16
489
+ assert layer.qzeros.shape == (1, 4)
490
+ assert layer.qzeros.dtype == torch.int32
491
+ assert layer.g_idx.shape == (64,)
492
+ assert layer.g_idx.dtype == torch.int32
493
+ assert layer.in_features == 64
494
+ assert layer.out_features == 64
495
+ assert layer.params_dtype == torch.bfloat16
496
+
497
+
498
def test_wna16_xpu_falls_back_when_ark_unavailable(monkeypatch) -> None:
499
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
500
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
@@ -452,10 +596,10 @@ def test_wna16_xpu_unsupported_config_still_raises(monkeypatch) -> None:
596
597
with pytest.raises(NotImplementedError, match="unsupported config"):
598
INCWna16Scheme().get_linear_method(
455
- make_config(sym=False),
599
+ make_config(weight_bits=2, sym=False),
600
object(),
601
"layer",
458
- make_layer_config(sym=False),
602
+ make_layer_config(bits=2, sym=False),
603
)
604
605
vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py
+10
-1
@@ -20,6 +20,8 @@ if TYPE_CHECKING:
20
21
logger = init_logger(__name__)
22
23
+XPU_WNA16_SUPPORTED_BITS = {2, 4}
24
+
25
26
class INCWna16Scheme(INCScheme):
27
@staticmethod
@@ -35,7 +37,7 @@ class INCWna16Scheme(INCScheme):
37
):
38
del config, layer
39
if current_platform.is_xpu():
38
- if layer_config.bits == 4 and layer_config.sym:
40
+ if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
41
from .inc_ark_ops import get_ark_state
42
from .inc_wna16_linear import (
43
INCARKLinearMethod,
@@ -45,6 +47,13 @@ class INCWna16Scheme(INCScheme):
47
is_ark_available, ark_error, _, _ = get_ark_state()
48
if is_ark_available:
49
return INCLinearMethod(INCARKLinearMethod(layer_config))
50
+ elif layer_config.bits == 2:
51
+ raise NotImplementedError(
52
+ "INC int2 on XPU requires the ARK backend. "
53
+ f"Layer: {prefix}. "
54
+ f"auto_round_kernel unavailable: "
55
+ f"{ark_error or 'unknown error'}"
56
+ )
57
58
logger.debug(
59
"ARK backend is unavailable for layer %s; "