A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 08:45 UTC
[XPU] fix broken fp8 online quantization for XPU platform (#30831)
Signed-off-by: Yan Ma <yan.ma@intel.com>
Yan Ma committed
Dec 17, 2025 at 16:28 UTC
4f735babb7353987137b85ec0465e594e9ed1384
1 file changed
+35
vllm/model_executor/layers/quantization/ipex_quant.py
+35
@@ -27,6 +27,10 @@ from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
27
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
28
from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
29
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
30
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
31
+ maybe_create_device_identity,
32
+)
33
+from vllm.model_executor.parameter import ModelWeightParameter
34
from vllm.model_executor.utils import set_weight_attrs
35
from vllm.platforms import current_platform
36
@@ -305,6 +309,37 @@ class XPUFp8LinearMethod(Fp8LinearMethod):
309
def __init__(self, quant_config: Fp8Config):
310
super().__init__(quant_config)
311
312
+ def create_weights(
313
+ self,
314
+ layer: torch.nn.Module,
315
+ input_size_per_partition: int,
316
+ output_partition_sizes: list[int],
317
+ input_size: int,
318
+ output_size: int,
319
+ params_dtype: torch.dtype,
320
+ **extra_weight_attrs,
321
+ ):
322
+ maybe_create_device_identity()
323
+
324
+ output_size_per_partition = sum(output_partition_sizes)
325
+ weight_loader = extra_weight_attrs.get("weight_loader")
326
+ layer.logical_widths = output_partition_sizes
327
+ layer.input_size_per_partition = input_size_per_partition
328
+ layer.output_size_per_partition = output_size_per_partition
329
+ layer.orig_dtype = params_dtype
330
+ layer.weight_block_size = None
331
+ weight = ModelWeightParameter(
332
+ data=torch.empty(
333
+ output_size_per_partition,
334
+ input_size_per_partition,
335
+ dtype=params_dtype,
336
+ ),
337
+ input_dim=1,
338
+ output_dim=0,
339
+ weight_loader=weight_loader,
340
+ )
341
+ layer.register_parameter("weight", weight)
342
+
343
def process_weights_after_loading(self, layer: Module) -> None:
344
# If checkpoint not serialized fp8, quantize the weights.
345
if not self.quant_config.is_checkpoint_fp8_serialized: