A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 18:30 UTC
[Quantization][Autoround][XPU] Add `W4A16` Support (#37986)
Signed-off-by: yiliu30 <yi4.liu@intel.com>
Yi Liu committed
Apr 1, 2026 at 00:48 UTC
0dd25a44ea00931a7a055085d5b25f0d3b3fe8a2
2 files changed
+165
-11
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+1
@@ -42,6 +42,7 @@ docker run \
42
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
43
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
44
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
45
+ python3 examples/basic/offline_inference/generate.py --model OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc --block-size 64 --enforce-eager --max-model-len 8192
46
cd tests
47
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
48
pytest -v -s v1/engine
vllm/model_executor/layers/quantization/inc.py
+164
-11
@@ -6,14 +6,24 @@ from typing import TYPE_CHECKING, Any
6
7
import regex as re
8
import torch
9
+from torch.nn.parameter import Parameter
10
11
from vllm.logger import init_logger
11
-from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
12
+from vllm.model_executor.layers.linear import (
13
+ LinearBase,
14
+ LinearMethodBase,
15
+ UnquantizedLinearMethod,
16
+)
17
from vllm.model_executor.layers.quantization import (
18
QuantizationConfig,
19
QuantizationMethods,
20
)
21
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
22
+from vllm.model_executor.parameter import (
23
+ GroupQuantScaleParameter,
24
+ PackedvLLMParameter,
25
+ RowvLLMParameter,
26
+)
27
from vllm.platforms import current_platform
28
from vllm.scalar_type import scalar_types
29
@@ -402,16 +412,30 @@ class INCConfig(QuantizationConfig):
412
413
return None
414
405
- def apply_ipex_quant_layer(self, layer, prefix: str):
415
+ def apply_xpu_w4a16_quant_layer(self, layer, prefix: str):
416
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
417
if not self.check_quantized(weight_bits):
418
if isinstance(layer, (LinearBase, ParallelLMHead)):
419
return UnquantizedLinearMethod()
420
else:
421
return None
412
- raise NotImplementedError(
413
- "INC quantization is not supported during xpu kernel migration."
414
- )
422
+
423
+ if weight_bits != 4:
424
+ raise NotImplementedError(
425
+ f"INC on XPU only supports 4-bit quantization, "
426
+ f"got weight_bits={weight_bits}."
427
+ )
428
+ if not sym:
429
+ raise NotImplementedError(
430
+ "INC W4A16 on XPU only supports symmetric quantization for now."
431
+ )
432
+ if isinstance(layer, (LinearBase, ParallelLMHead)):
433
+ return INCXPULinearMethod(
434
+ weight_bits=weight_bits,
435
+ group_size=group_size,
436
+ sym=sym,
437
+ )
438
+ return None
439
440
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
441
if prefix and self.extra_config:
@@ -420,12 +444,8 @@ class INCConfig(QuantizationConfig):
444
layer_name == prefix or layer_name == f"model.{prefix}"
445
) and self.extra_config[layer_name].get("bits", 16) >= 16:
446
return UnquantizedLinearMethod()
423
- if (
424
- current_platform.is_cpu()
425
- or current_platform.is_xpu()
426
- or self.backend == "ipex"
427
- ):
428
- return self.apply_ipex_quant_layer(layer, prefix)
447
+ if current_platform.is_xpu():
448
+ return self.apply_xpu_w4a16_quant_layer(layer, prefix)
449
if "gptq" in self.packing_format or "gptq" in self.backend:
450
return self.apply_gptq_quant_layer(layer, prefix)
451
if "awq" in self.packing_format or "awq" in self.backend:
@@ -440,3 +460,136 @@ class INCConfig(QuantizationConfig):
460
if is_auto_round_format:
461
return cls.get_name()
462
return None
463
+
464
+
465
+class INCXPULinearMethod(LinearMethodBase):
466
+ """XPU linear method for INC w4a16 GPTQ quantization (symmetric only).
467
+
468
+ Repacks GPTQ weights from [in_packed, out] to oneDNN [out, in_packed]
469
+ layout and calls torch.ops._xpu_C.int4_gemm_w4a16.
470
+
471
+ GPTQ format: qweight [in_packed, out] with sequential nibble order.
472
+
473
+ Note: Asymmetric quantization (sym=false) is not for now.
474
+
475
+ FIXME(yiliu30): Refine the implementation to reuse XPUwNa16LinearKernel.
476
+ """
477
+
478
+ def __init__(self, weight_bits: int, group_size: int, sym: bool):
479
+ self.weight_bits = weight_bits
480
+ self.group_size = group_size
481
+ self.sym = sym
482
+ self.pack_factor = 32 // weight_bits
483
+
484
+ def create_weights(
485
+ self,
486
+ layer: torch.nn.Module,
487
+ input_size_per_partition: int,
488
+ output_partition_sizes: list[int],
489
+ input_size: int,
490
+ output_size: int,
491
+ params_dtype: torch.dtype,
492
+ **extra_weight_attrs,
493
+ ):
494
+ del output_size # Unused.
495
+ output_size_per_partition = sum(output_partition_sizes)
496
+ weight_loader = extra_weight_attrs.get("weight_loader")
497
+ scales_and_zp_size = input_size_per_partition // self.group_size
498
+
499
+ # GPTQ: qweight [in // pack_factor, out] packed along input dim
500
+ qweight = PackedvLLMParameter(
501
+ data=torch.empty(
502
+ input_size_per_partition // self.pack_factor,
503
+ output_size_per_partition,
504
+ dtype=torch.int32,
505
+ ),
506
+ input_dim=0,
507
+ output_dim=1,
508
+ packed_dim=0,
509
+ packed_factor=self.pack_factor,
510
+ weight_loader=weight_loader,
511
+ )
512
+ # scales: [num_groups, out] params_dtype
513
+ scales = GroupQuantScaleParameter(
514
+ data=torch.empty(
515
+ scales_and_zp_size,
516
+ output_size_per_partition,
517
+ dtype=params_dtype,
518
+ ),
519
+ input_dim=0,
520
+ output_dim=1,
521
+ weight_loader=weight_loader,
522
+ )
523
+ # qzeros: [num_groups, out // pack_factor] int32
524
+ qzeros = PackedvLLMParameter(
525
+ data=torch.empty(
526
+ scales_and_zp_size,
527
+ output_size_per_partition // self.pack_factor,
528
+ dtype=torch.int32,
529
+ ),
530
+ input_dim=0,
531
+ output_dim=1,
532
+ packed_dim=1,
533
+ packed_factor=self.pack_factor,
534
+ weight_loader=weight_loader,
535
+ )
536
+
537
+ layer.register_parameter("qweight", qweight)
538
+ layer.register_parameter("scales", scales)
539
+ layer.register_parameter("qzeros", qzeros)
540
+
541
+ # GPTQ checkpoints may include g_idx for activation reordering.
542
+ # Register it so the weight loader doesn't error on unexpected keys.
543
+ g_idx = RowvLLMParameter(
544
+ data=torch.tensor(
545
+ [i // self.group_size for i in range(input_size_per_partition)],
546
+ dtype=torch.int32,
547
+ ),
548
+ input_dim=0,
549
+ weight_loader=weight_loader,
550
+ )
551
+ layer.register_parameter("g_idx", g_idx)
552
+
553
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
554
+ """Repack GPTQ weights into kernel-ready NT layout."""
555
+ device = layer.qweight.data.device
556
+
557
+ # oneDNN int4 kernel requires strides[0]==1 ("NT format"), but GPTQ
558
+ # checkpoint is [K_packed, N] contiguous with strides (N, 1).
559
+ # Two transposes are needed — neither alone can achieve this:
560
+ # 1. .t().contiguous() → [N, K_packed] contiguous in memory
561
+ # 2. .t() → [K_packed, N] view with strides (1, K_packed)
562
+ # The result has the same logical shape but strides[0]==1 as required.
563
+ qweight_ct = layer.qweight.data.t().contiguous()
564
+ layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
565
+
566
+ # Scales: [num_groups, out] — no change needed
567
+ layer.scales = Parameter(layer.scales.data, requires_grad=False)
568
+
569
+ # Symmetric: GPTQ v1 stores qzeros=7, effective zp = 7+1 = 8
570
+ # Kernel expects int8 scalar = 8
571
+ layer.qzeros = Parameter(
572
+ torch.tensor([8], dtype=torch.int8, device=device),
573
+ requires_grad=False,
574
+ )
575
+
576
+ def apply(
577
+ self,
578
+ layer: torch.nn.Module,
579
+ x: torch.Tensor,
580
+ bias: torch.Tensor | None = None,
581
+ ) -> torch.Tensor:
582
+ # qweight is already in NT layout [K_packed, N] (strides (1, K_packed))
583
+ # from process_weights_after_loading — pass directly to kernel.
584
+ out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
585
+ reshaped_x = x.reshape(-1, x.shape[-1])
586
+ out = torch.ops._xpu_C.int4_gemm_w4a16(
587
+ reshaped_x,
588
+ layer.qweight,
589
+ bias,
590
+ layer.scales,
591
+ layer.qzeros,
592
+ self.group_size,
593
+ None, # g_idx not needed: desc_act is always False for INC models
594
+ )
595
+ return out.reshape(out_shape)