A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 01:15 UTC
[XPU][1/N] Deprecate ipex and switch to vllm-xpu-kernels for xpu platform (#33379)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Kunshang Ji committed
Feb 3, 2026 at 14:46 UTC
e10604480bb8177d563253194667ee9c1590e31a
18 files changed
+144
-921
.buildkite/scripts/hardware_ci/run-xpu-test.sh
+1
-2
@@ -38,10 +38,9 @@ docker run \
38
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
39
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend ray
40
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
41
- python3 examples/offline_inference/basic/generate.py --model Intel/Qwen2.5-0.5B-W4A16-G128-AutoRound-LLMC-TEST-ONLY --enforce-eager
41
python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
42
cd tests
44
- pytest -v -s v1/core
43
+ pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py
44
pytest -v -s v1/engine
45
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py
46
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py
docker/Dockerfile.xpu
+10
-4
@@ -1,8 +1,8 @@
1
-FROM intel/deep-learning-essentials:2025.2.2-0-devel-ubuntu24.04 AS vllm-base
1
+FROM intel/deep-learning-essentials:2025.3.2-0-devel-ubuntu24.04 AS vllm-base
2
3
RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
4
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
5
- add-apt-repository -y ppa:kobuk-team/intel-graphics-staging
5
+ add-apt-repository -y ppa:kobuk-team/intel-graphics
6
7
RUN apt clean && apt-get update -y && \
8
apt-get install -y --no-install-recommends --fix-missing \
@@ -25,10 +25,13 @@ RUN apt clean && apt-get update -y && \
25
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
26
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1
27
28
-RUN apt install -y libze1 libze-dev libze-intel-gpu1 intel-opencl-icd libze-intel-gpu-raytracing intel-ocloc
28
+RUN apt update && apt upgrade -y && \
29
+ apt install -y libze1 libze-dev libze-intel-gpu1 intel-opencl-icd libze-intel-gpu-raytracing intel-ocloc && \
30
+ apt install -y intel-oneapi-compiler-dpcpp-cpp-2025.3
31
+
32
33
# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2.
31
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.6_offline.sh"
34
+ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh"
35
RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \
36
bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
37
rm "${ONECCL_INSTALLER}" && \
@@ -85,6 +88,9 @@ RUN python3 -m pip install -e tests/vllm_test_utils
88
ENV NIXL_VERSION=0.7.0
89
RUN python3 /workspace/vllm/tools/install_nixl_from_source_ubuntu.py
90
91
+# FIX triton
92
+RUN --mount=type=cache,target=/root/.cache/pip pip uninstall triton triton-xpu -y && pip install triton-xpu==3.6.0 --extra-index-url=https://download.pytorch.org/whl/xpu
93
+
94
# PyJWT-2.7.0 will influence some wheel behaviors, remove its dist-info to avoid conflicts
95
RUN rm /usr/lib/python3/dist-packages/PyJWT-2.7.0.dist-info/ -rf
96
requirements/xpu.txt
+2
-2
@@ -11,8 +11,8 @@ jinja2>=3.1.6
11
datasets # for benchmark scripts
12
numba == 0.61.2 # Required for N-gram speculative decoding
13
--extra-index-url=https://download.pytorch.org/whl/xpu
14
-torch==2.9.0+xpu
14
+torch==2.10.0+xpu
15
torchaudio
16
torchvision
17
18
-intel-extension-for-pytorch @ https://intel-extension-for-pytorch.s3.us-east-1.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.9.10.post0%2Bxpu-cp312-cp312-linux_x86_64.whl
18
+vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.0/vllm_xpu_kernels-0.1.0-cp312-cp312-linux_x86_64.whl
\ No newline at end of file
vllm/_ipex_ops.py
+73
-360
@@ -1,273 +1,59 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from typing import TYPE_CHECKING
5
6
import torch
7
+from vllm_xpu_kernels.flash_attn_interface import flash_attn_varlen_func
8
9
from vllm.logger import init_logger
8
-from vllm.platforms import current_platform
10
11
logger = init_logger(__name__)
12
12
-try:
13
- import intel_extension_for_pytorch as ipex
14
-except ImportError as e:
15
- logger.debug("Import error msg: %s", e.msg)
13
+if TYPE_CHECKING:
14
15
+ def register_fake(fn):
16
+ return lambda name: fn
17
+else:
18
+ try:
19
+ from torch.library import register_fake
20
+ except ImportError:
21
+ from torch.library import impl_abstract as register_fake
22
18
-class ipex_ops:
19
- @staticmethod
20
- def _reshape_activation_tensor(
21
- x: torch.Tensor,
22
- ) -> tuple[torch.Tensor, torch.Tensor]:
23
- num = x.size(0)
24
- d = x.size(1) // 2
25
- x = x.reshape(num, 2, d)
26
- x1, x2 = torch.chunk(x, chunks=2, dim=1)
27
- x1 = x1.reshape(num, d)
28
- x2 = x2.reshape(num, d)
29
- return x1, x2
30
-
31
- @staticmethod
32
- def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
33
- ipex.llm.functional.silu_and_mul(x, out)
34
-
35
- @staticmethod
36
- def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
37
- ipex.llm.functional.gelu_and_mul(x, out)
38
-
39
- @staticmethod
40
- def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
41
- ipex.llm.functional.gelu_and_mul(x, out)
23
+if hasattr(torch.ops._xpu_C, "fp8_gemm_w8a16"):
24
43
- @staticmethod
44
- def gelu_fast(x: torch.Tensor) -> torch.Tensor:
45
- return torch.nn.functional.gelu(x)
46
-
47
- @staticmethod
48
- def gelu_new(x: torch.Tensor) -> torch.Tensor:
49
- return torch.nn.functional.gelu(x)
50
-
51
- @staticmethod
52
- def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
53
- ipex.llm.functional.gelu_quick(x, out)
54
-
55
- @staticmethod
56
- def paged_attention_v1(
57
- out: torch.Tensor,
58
- query: torch.Tensor,
59
- key_cache: torch.Tensor,
60
- value_cache: torch.Tensor,
61
- num_kv_heads: int,
62
- scale: float,
63
- block_tables: torch.Tensor,
64
- context_lens: torch.Tensor,
65
- block_size: int,
66
- max_context_len: int,
67
- alibi_slopes: torch.Tensor | None,
68
- kv_cache_dtype: str,
69
- k_scale: float,
70
- v_scale: float,
71
- tp_rank: int = 0,
72
- blocksparse_local_blocks: int = 0,
73
- blocksparse_vert_stride: int = 0,
74
- blocksparse_block_size: int = 64,
75
- blocksparse_head_sliding_step: int = 0,
76
- ) -> None:
77
- assert kv_cache_dtype == "auto"
78
- num_heads = out.size(1)
79
- num_queries_per_tokens = num_heads // num_kv_heads
80
- ipex.llm.modules.PagedAttention.single_query_kv_attention(
81
- out,
82
- query.contiguous(),
83
- key_cache.view_as(value_cache),
84
- value_cache,
85
- num_queries_per_tokens,
86
- scale,
87
- block_tables,
88
- context_lens,
89
- block_size,
90
- max_context_len,
91
- alibi_slopes,
92
- )
93
-
94
- @staticmethod
95
- def paged_attention_v2(
96
- out: torch.Tensor,
97
- exp_sum: torch.Tensor,
98
- max_logits: torch.Tensor,
99
- tmp_out: torch.Tensor,
100
- query: torch.Tensor,
101
- key_cache: torch.Tensor,
102
- value_cache: torch.Tensor,
103
- num_kv_heads: int,
104
- scale: float,
105
- block_tables: torch.Tensor,
106
- context_lens: torch.Tensor,
107
- block_size: int,
108
- max_context_len: int,
109
- alibi_slopes: torch.Tensor | None,
110
- kv_cache_dtype: str,
111
- k_scale: float,
112
- v_scale: float,
113
- tp_rank: int = 0,
114
- blocksparse_local_blocks: int = 0,
115
- blocksparse_vert_stride: int = 0,
116
- blocksparse_block_size: int = 64,
117
- blocksparse_head_sliding_step: int = 0,
118
- ) -> None:
119
- assert kv_cache_dtype == "auto"
120
- num_heads = out.size(1)
121
- num_queries_per_tokens = num_heads // num_kv_heads
122
- ipex.llm.modules.PagedAttention.single_query_kv_attention(
123
- out,
124
- query.contiguous(),
125
- key_cache.view_as(value_cache),
126
- value_cache,
127
- num_queries_per_tokens,
128
- scale,
129
- block_tables,
130
- context_lens,
131
- block_size,
132
- max_context_len,
133
- alibi_slopes,
134
- )
135
-
136
- @staticmethod
137
- def rotary_embedding(
138
- positions: torch.Tensor, # [batch_size, seq_len]
139
- query: torch.Tensor, # [batch_size, seq_len, num_heads*head_size]
140
- key: torch.Tensor, # [batch_size, seq_len, num_kv_heads*head_size]
141
- head_size: int,
142
- cos_sin_cache: torch.Tensor, # [cos_sin_dim, rot_dim]
143
- is_neox: bool,
144
- ) -> None:
145
- rot_dim = cos_sin_cache.size(1)
146
- ipex.llm.functional.rotary_embedding_batched(
147
- positions, query, key, head_size, cos_sin_cache, is_neox, rot_dim
148
- )
149
-
150
- @staticmethod
151
- def rms_norm(
152
- input: torch.Tensor, weight: torch.Tensor, epsilon: float
25
+ @register_fake("_xpu_C::fp8_gemm_w8a16")
26
+ def _fp8_gemm_w8a16_fake(
27
+ input: torch.Tensor,
28
+ q_weight: torch.Tensor,
29
+ weight_scale: torch.Tensor,
30
+ bias: torch.Tensor | None = None,
31
) -> torch.Tensor:
154
- out = torch.empty_like(input)
155
- torch.ops.torch_ipex.rms_norm_vllm(out, input.contiguous(), weight, epsilon)
156
- return out
32
+ input_2d = input.view(-1, input.shape[-1])
33
+ M = input_2d.size(0)
34
+ N = q_weight.size(1)
35
+ return torch.empty((M, N), dtype=input.dtype, device=input.device)
36
158
- @staticmethod
159
- def fused_add_rms_norm(
160
- input: torch.Tensor,
161
- residual: torch.Tensor,
162
- weight: torch.Tensor,
163
- epsilon: float,
164
- ) -> None:
165
- torch.ops.torch_ipex.fused_add_rms_norm_vllm(input, residual, weight, epsilon)
37
167
- @staticmethod
168
- def varlen_attention(
169
- query: torch.Tensor,
170
- key: torch.Tensor,
171
- value: torch.Tensor,
172
- out: torch.Tensor,
173
- seqlen_q: torch.Tensor,
174
- seqlen_k: torch.Tensor,
175
- alibi_slopes: torch.Tensor | None,
176
- max_seqlen_q: int,
177
- max_seqlen_k: int,
178
- pdropout: float,
179
- softmax_scale: float,
180
- zero_tensors: bool,
181
- is_causal: bool,
182
- return_softmax: bool,
183
- gen_: torch.Generator,
184
- window_size_left: float,
185
- window_size_right: float,
186
- logits_soft_cap: float,
187
- ) -> None:
188
- if ipex.__version__.endswith("cpu"):
189
- if logits_soft_cap != 0.0:
190
- raise ValueError("IPEX CPU does not support logits_soft_cap")
191
- assert alibi_slopes is None
192
- assert window_size_left < 0 and window_size_right < 0
193
- ipex.llm.functional.varlen_attention(
194
- query.contiguous(),
195
- key.contiguous(),
196
- value.contiguous(),
197
- out,
198
- seqlen_q.int(),
199
- seqlen_k.int(),
200
- max_seqlen_q,
201
- max_seqlen_k,
202
- pdropout,
203
- softmax_scale,
204
- zero_tensors,
205
- is_causal,
206
- return_softmax,
207
- gen_,
208
- )
209
- else: # XPU build
210
- ipex.llm.functional.varlen_attention(
211
- query.contiguous(),
212
- key.contiguous(),
213
- value.contiguous(),
214
- out,
215
- seqlen_q.int(),
216
- seqlen_k.int(),
217
- alibi_slopes,
218
- max_seqlen_q,
219
- max_seqlen_k,
220
- pdropout,
221
- softmax_scale,
222
- zero_tensors,
223
- is_causal,
224
- return_softmax,
225
- gen_,
226
- window_size_left,
227
- window_size_right,
228
- logits_soft_cap,
229
- )
38
+if hasattr(torch.ops._xpu_C, "int4_gemm_w4a16"):
39
231
- @staticmethod
232
- def reshape_and_cache(
233
- key: torch.Tensor,
234
- value: torch.Tensor,
235
- key_cache: torch.Tensor,
236
- value_cache: torch.Tensor,
237
- slot_mapping: torch.Tensor,
238
- kv_cache_dtype: str,
239
- k_scale: float,
240
- v_scale: float,
241
- ) -> None:
242
- assert kv_cache_dtype == "auto"
243
- ipex.llm.modules.PagedAttention.reshape_and_cache(
244
- key, value, key_cache, value_cache, slot_mapping
245
- )
40
+ @register_fake("_xpu_C::int4_gemm_w4a16")
41
+ def _int4_gemm_w4a16_fake(
42
+ input: torch.Tensor,
43
+ q_weight: torch.Tensor,
44
+ bias: torch.Tensor | None,
45
+ weight_scale: torch.Tensor,
46
+ qzeros: torch.Tensor,
47
+ group_size: int,
48
+ group_idx: torch.Tensor | None = None,
49
+ ) -> torch.Tensor:
50
+ input_2d = input.view(-1, input.shape[-1])
51
+ M = input_2d.size(0)
52
+ N = q_weight.size(1)
53
+ return torch.empty((M, N), dtype=input.dtype, device=input.device)
54
247
- @staticmethod
248
- def reshape_and_cache_flash(
249
- key: torch.Tensor,
250
- value: torch.Tensor,
251
- key_cache: torch.Tensor,
252
- value_cache: torch.Tensor,
253
- slot_mapping: torch.Tensor,
254
- kv_cache_dtype: str,
255
- k_scale: torch.Tensor | None = None,
256
- v_scale: torch.Tensor | None = None,
257
- k_scale_float: float = 1.0,
258
- v_scale_float: float = 1.0,
259
- ) -> None:
260
- ipex.llm.modules.PagedAttention.reshape_and_cache_flash(
261
- key,
262
- value,
263
- key_cache,
264
- value_cache,
265
- slot_mapping,
266
- kv_cache_dtype,
267
- k_scale_float,
268
- v_scale_float,
269
- )
55
56
+class ipex_ops:
57
@staticmethod
58
def flash_attn_varlen_func(
59
q: torch.Tensor,
@@ -295,8 +81,21 @@ class ipex_ops:
81
k_descale=None,
82
v_descale=None,
83
num_splits=0,
84
+ return_softmax_lse: bool | None = False,
85
s_aux: torch.Tensor | None = None,
86
):
87
+ assert cu_seqlens_k is not None or seqused_k is not None, (
88
+ "cu_seqlens_k or seqused_k must be provided"
89
+ )
90
+ assert cu_seqlens_k is None or seqused_k is None, (
91
+ "cu_seqlens_k and seqused_k cannot be provided at the same time"
92
+ )
93
+ assert block_table is None or seqused_k is not None, (
94
+ "when enable block_table, seqused_k is needed"
95
+ )
96
+ assert block_table is not None or cu_seqlens_k is not None, (
97
+ "when block_table is disabled, cu_seqlens_k is needed"
98
+ )
99
if out is None:
100
out = torch.empty(q.shape, dtype=q.dtype, device=q.device)
101
real_window_size: tuple[int, int]
@@ -304,56 +103,31 @@ class ipex_ops:
103
real_window_size = (-1, -1)
104
else:
105
assert len(window_size) == 2
307
- real_window_size = (window_size[0], window_size[1])
106
+ real_window_size = (window_size[0], window_size[1]) # noqa: F841
107
108
+ # In encode attention, v maybe not contiguous and current
109
+ # kernel can't handle it
110
if block_table is None:
310
- assert cu_seqlens_k is not None, (
311
- "cu_seqlens_k can't be None when calling varlen_attention."
312
- )
313
- if softmax_scale is None:
314
- softmax_scale = q.shape[-1] ** (-0.5)
315
- ipex_ops.varlen_attention(
316
- q.contiguous(),
317
- k.contiguous(),
318
- v.contiguous(),
319
- out,
320
- cu_seqlens_q,
321
- cu_seqlens_k,
322
- None,
323
- max_seqlen_q,
324
- max_seqlen_k,
325
- 0.0,
326
- softmax_scale,
327
- False,
328
- causal,
329
- False,
330
- None,
331
- real_window_size[0],
332
- real_window_size[1],
333
- -1,
334
- )
335
- return out
336
- else:
337
- return ipex.llm.modules.PagedAttention.flash_attn_varlen_func(
338
- out,
339
- q.contiguous(),
340
- k,
341
- v,
342
- cu_seqlens_q,
343
- seqused_k,
344
- max_seqlen_q,
345
- max_seqlen_k,
346
- softmax_scale,
347
- causal,
348
- block_table,
349
- alibi_slopes,
350
- sink=s_aux,
351
- softcap=softcap,
352
- window_size_left=real_window_size[0],
353
- window_size_right=real_window_size[1],
354
- k_scale=1.0,
355
- v_scale=1.0,
356
- )
111
+ v = v.contiguous()
112
+ return flash_attn_varlen_func(
113
+ out=out,
114
+ q=q.contiguous(),
115
+ k=k,
116
+ v=v,
117
+ cu_seqlens_q=cu_seqlens_q,
118
+ cu_seqlens_k=cu_seqlens_k,
119
+ seqused_k=seqused_k,
120
+ max_seqlen_q=max_seqlen_q,
121
+ max_seqlen_k=max_seqlen_k,
122
+ softmax_scale=softmax_scale,
123
+ causal=causal,
124
+ block_table=block_table,
125
+ s_aux=s_aux,
126
+ window_size=real_window_size,
127
+ # alibi_slopes = alibi_slopes,
128
+ # softcap=softcap,
129
+ return_softmax_lse=return_softmax_lse,
130
+ )
131
132
@staticmethod
133
def get_scheduler_metadata(
@@ -382,64 +156,3 @@ class ipex_ops:
156
"get_scheduler_metadata is not implemented for ipex_ops, returning None."
157
)
158
return None
385
-
386
- @staticmethod
387
- def swap_blocks(
388
- src: torch.Tensor, dst: torch.Tensor, block_mapping: torch.Tensor
389
- ) -> None:
390
- torch.xpu.swap_blocks(src, dst, block_mapping) # type: ignore
391
-
392
- @staticmethod
393
- def scaled_fp8_quant(
394
- input: torch.Tensor,
395
- scale: torch.Tensor | None = None,
396
- num_token_padding: int | None = None,
397
- scale_ub: torch.Tensor | None = None,
398
- use_per_token_if_dynamic: bool = False,
399
- output: torch.Tensor | None = None,
400
- ) -> tuple[torch.Tensor, torch.Tensor]:
401
- """
402
- Quantize input tensor to FP8 and return quantized tensor and scale.
403
-
404
- This function is designed for both static and dynamic quantization:
405
- If you provide the scale, it will use static scaling and if you omit
406
- it, the scale will be determined dynamically. Currently, XPU platform
407
- only supports dynamic quantization. The function also allows optional
408
- padding of the output tensors for downstream kernels that will benefit
409
- from padding.
410
-
411
- Args:
412
- input: The input tensor to be quantized to FP8
413
- scale: Optional scaling factor for the FP8 quantization
414
- scale_ub: Optional upper bound for scaling factor in dynamic
415
- per token case
416
- num_token_padding: If specified, pad the first dimension
417
- of the output to at least this value.
418
- use_per_token_if_dynamic: Whether to do per_tensor or per_token
419
- in the dynamic quantization case.
420
-
421
- Returns:
422
- tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
423
- scaling factor.
424
- """
425
- # This code assumes batch_dim and num_tokens are flattened
426
- assert input.ndim == 2
427
- shape: tuple[int, int] | torch.Size = input.shape
428
- out_dtype: torch.dtype = current_platform.fp8_dtype()
429
- if num_token_padding:
430
- shape = (max(num_token_padding, input.shape[0]), shape[1])
431
- if output is None:
432
- output = torch.empty(shape, device=input.device, dtype=out_dtype)
433
- else:
434
- assert num_token_padding is None, (
435
- "padding not supported if output passed in"
436
- )
437
- assert output.dtype == out_dtype
438
- assert scale is None, "only dynamic fp8 quantization supported on XPU"
439
- assert not use_per_token_if_dynamic, (
440
- "per token dynamic fp8 quantization not supported on XPU"
441
- )
442
- scale = torch.zeros(1, device=input.device, dtype=torch.float32)
443
- torch.ops.torch_ipex.dynamic_scaled_fp8_quant(output, input, scale)
444
-
445
- return output, scale
vllm/config/model.py
-1
@@ -877,7 +877,6 @@ class ModelConfig:
877
overrides = [
878
"gptq_marlin",
879
"awq_marlin",
880
- "ipex",
880
"inc",
881
"moe_wna16",
882
"modelopt",
vllm/model_executor/layers/activation.py
+29
-53
@@ -129,12 +129,8 @@ class SiluAndMul(CustomOp):
129
130
def __init__(self, *, compile_native: bool = True):
131
super().__init__(compile_native=compile_native)
132
- if current_platform.is_cuda_alike():
132
+ if current_platform.is_cuda_alike() or current_platform.is_xpu():
133
self.op = torch.ops._C.silu_and_mul
134
- elif current_platform.is_xpu():
135
- from vllm._ipex_ops import ipex_ops
136
-
137
- self.op = ipex_ops.silu_and_mul
134
elif current_platform.is_cpu():
135
self._forward_method = self.forward_native
136
@@ -152,11 +148,7 @@ class SiluAndMul(CustomOp):
148
return out
149
150
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
155
- d = x.shape[-1] // 2
156
- output_shape = x.shape[:-1] + (d,)
157
- out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
158
- self.op(out, x)
159
- return out
151
+ return self.forward_cuda(x)
152
153
154
# --8<-- [start:mul_and_silu]
@@ -175,12 +167,8 @@ class MulAndSilu(CustomOp):
167
168
def __init__(self):
169
super().__init__()
178
- if current_platform.is_cuda_alike():
170
+ if current_platform.is_cuda_alike() or current_platform.is_xpu():
171
self.op = torch.ops._C.mul_and_silu
180
- elif current_platform.is_xpu():
181
- from vllm._ipex_ops import ipex_ops
182
-
183
- self.op = ipex_ops.silu_and_mul
172
elif current_platform.is_cpu():
173
self._forward_method = self.forward_native
174
@@ -196,8 +184,8 @@ class MulAndSilu(CustomOp):
184
self.op(out, x)
185
return out
186
199
- # TODO implement forward_xpu for MulAndSilu
200
- # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
187
+ def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
188
+ return self.forward_cuda(x)
189
190
191
# --8<-- [start:gelu_and_mul_sparse]
@@ -278,7 +266,11 @@ class GeluAndMul(CustomOp):
266
self.approximate = approximate
267
if approximate not in ("none", "tanh"):
268
raise ValueError(f"Unknown approximate mode: {approximate}")
281
- if current_platform.is_cuda_alike() or current_platform.is_cpu():
269
+ if (
270
+ current_platform.is_cuda_alike()
271
+ or current_platform.is_cpu()
272
+ or current_platform.is_xpu()
273
+ ):
274
if approximate == "none":
275
self.op = torch.ops._C.gelu_and_mul
276
elif approximate == "tanh":
@@ -289,13 +281,6 @@ class GeluAndMul(CustomOp):
281
"with torch.compile. For native implementation, fallback to 'none' "
282
"approximation. The custom kernel implementation is unaffected."
283
)
292
- elif current_platform.is_xpu():
293
- from vllm._ipex_ops import ipex_ops
294
-
295
- if approximate == "none":
296
- self.op = ipex_ops.gelu_and_mul
297
- else:
298
- self.op = ipex_ops.gelu_tanh_and_mul
284
285
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
286
"""PyTorch-native implementation equivalent to forward()."""
@@ -314,11 +299,7 @@ class GeluAndMul(CustomOp):
299
return out
300
301
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
317
- d = x.shape[-1] // 2
318
- output_shape = x.shape[:-1] + (d,)
319
- out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
320
- self.op(out, x)
321
- return out
302
+ return self.forward_cuda(x)
303
304
def extra_repr(self) -> str:
305
return f"approximate={repr(self.approximate)}"
@@ -401,12 +382,12 @@ class NewGELU(CustomOp):
382
383
def __init__(self):
384
super().__init__()
404
- if current_platform.is_cuda_alike() or current_platform.is_cpu():
385
+ if (
386
+ current_platform.is_cuda_alike()
387
+ or current_platform.is_cpu()
388
+ or current_platform.is_xpu()
389
+ ):
390
self.op = torch.ops._C.gelu_new
406
- elif current_platform.is_xpu():
407
- from vllm._ipex_ops import ipex_ops
408
-
409
- self.op = ipex_ops.gelu_new
391
392
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
393
"""PyTorch-native implementation equivalent to forward()."""
@@ -419,7 +400,7 @@ class NewGELU(CustomOp):
400
return out
401
402
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
422
- return self.op(x)
403
+ return self.forward_cuda(x)
404
405
406
# --8<-- [start:gelu_fast]
@@ -429,12 +410,12 @@ class FastGELU(CustomOp):
410
411
def __init__(self):
412
super().__init__()
432
- if current_platform.is_cuda_alike() or current_platform.is_cpu():
413
+ if (
414
+ current_platform.is_cuda_alike()
415
+ or current_platform.is_cpu()
416
+ or current_platform.is_xpu()
417
+ ):
418
self.op = torch.ops._C.gelu_fast
434
- elif current_platform.is_xpu():
435
- from vllm._ipex_ops import ipex_ops
436
-
437
- self.op = ipex_ops.gelu_fast
419
420
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
421
"""PyTorch-native implementation equivalent to forward()."""
@@ -446,7 +427,7 @@ class FastGELU(CustomOp):
427
return out
428
429
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
449
- return self.op(x)
430
+ return self.forward_cuda(x)
431
432
433
# --8<-- [start:quick_gelu]
@@ -457,12 +438,12 @@ class QuickGELU(CustomOp):
438
439
def __init__(self):
440
super().__init__()
460
- if current_platform.is_cuda_alike() or current_platform.is_cpu():
441
+ if (
442
+ current_platform.is_cuda_alike()
443
+ or current_platform.is_cpu()
444
+ or current_platform.is_xpu()
445
+ ):
446
self.op = torch.ops._C.gelu_quick
462
- elif current_platform.is_xpu():
463
- from vllm._ipex_ops import ipex_ops
464
-
465
- self.op = ipex_ops.gelu_quick
447
448
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
449
"""PyTorch-native implementation equivalent to forward()."""
@@ -474,12 +455,7 @@ class QuickGELU(CustomOp):
455
return out
456
457
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
477
- out = torch.empty_like(x)
478
- self.op(out, x)
479
- return out
480
-
481
- # TODO implement forward_xpu for QuickGELU
482
- # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
458
+ return self.forward_cuda(x)
459
460
461
# --8<-- [start:relu2]
vllm/model_executor/layers/layernorm.py
+1
-18
@@ -231,24 +231,7 @@ class RMSNorm(CustomOp):
231
x: torch.Tensor,
232
residual: torch.Tensor | None = None,
233
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
234
- if self.variance_size_override is not None:
235
- return self.forward_native(x, residual)
236
-
237
- from vllm._ipex_ops import ipex_ops as ops
238
-
239
- if residual is not None:
240
- ops.fused_add_rms_norm(
241
- x,
242
- residual,
243
- self.weight.data,
244
- self.variance_epsilon,
245
- )
246
- return x, residual
247
- return ops.rms_norm(
248
- x,
249
- self.weight.data,
250
- self.variance_epsilon,
251
- )
234
+ return self.forward_cuda(x, residual)
235
236
def extra_repr(self) -> str:
237
s = f"hidden_size={self.weight.data.size(0)}"
vllm/model_executor/layers/linear.py
-2
@@ -60,8 +60,6 @@ WEIGHT_LOADER_V2_SUPPORTED = [
60
"ModelOptFp8LinearMethod",
61
"ModelOptFp8PcPtLinearMethod",
62
"ModelOptFp8PbWoLinearMethod",
63
- "IPEXAWQLinearMethod",
64
- "IPEXGPTQLinearMethod",
63
"QuarkLinearMethod",
64
"ModelOptNvFp4LinearMethod",
65
"PetitNvFp4LinearMethod",
vllm/model_executor/layers/quantization/__init__.py
-4
@@ -24,7 +24,6 @@ QuantizationMethods = Literal[
24
"compressed-tensors",
25
"bitsandbytes",
26
"experts_int8",
27
- "ipex",
27
"quark",
28
"moe_wna16",
29
"torchao",
@@ -41,7 +40,6 @@ DEPRECATED_QUANTIZATION_METHODS = [
40
"fbgemm_fp8",
41
"fp_quant",
42
"experts_int8",
44
- "ipex",
43
"petit_nvfp4",
44
]
45
@@ -121,7 +119,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
119
from .gptq import GPTQConfig
120
from .gptq_marlin import GPTQMarlinConfig
121
from .inc import INCConfig
124
- from .ipex_quant import IPEXConfig
122
from .modelopt import ModelOptFp8Config, ModelOptNvFp4Config
123
from .moe_wna16 import MoeWNA16Config
124
from .mxfp4 import Mxfp4Config
@@ -144,7 +141,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
141
"bitsandbytes": BitsAndBytesConfig,
142
"ptpc_fp8": PTPCFp8Config,
143
"experts_int8": ExpertsInt8Config,
147
- "ipex": IPEXConfig,
144
"quark": QuarkConfig,
145
"moe_wna16": MoeWNA16Config,
146
"torchao": TorchAOConfig,
vllm/model_executor/layers/quantization/fp8.py
+2
-31
@@ -184,39 +184,10 @@ class Fp8Config(QuantizationConfig):
184
def get_xpu_quant_method(
185
self, layer: torch.nn.Module, prefix: str
186
) -> "QuantizeMethodBase | None":
187
- from vllm.model_executor.layers.quantization.ipex_quant import (
188
- XPUFp8LinearMethod,
189
- XPUFp8MoEMethod,
187
+ raise NotImplementedError(
188
+ "FP8 quantization is not supported during xpu kernel migration."
189
)
190
192
- fp8_config = Fp8Config(
193
- is_checkpoint_fp8_serialized=self.is_checkpoint_fp8_serialized,
194
- activation_scheme=self.activation_scheme,
195
- ignored_layers=self.ignored_layers,
196
- weight_block_size=self.weight_block_size,
197
- )
198
-
199
- if isinstance(layer, LinearBase):
200
- if is_layer_skipped(
201
- prefix=prefix,
202
- ignored_layers=self.ignored_layers,
203
- fused_mapping=self.packed_modules_mapping,
204
- ):
205
- return UnquantizedLinearMethod()
206
- return XPUFp8LinearMethod(fp8_config)
207
- elif isinstance(layer, FusedMoE):
208
- if is_layer_skipped(
209
- prefix=prefix,
210
- ignored_layers=self.ignored_layers,
211
- fused_mapping=self.packed_modules_mapping,
212
- ):
213
- return UnquantizedFusedMoEMethod(layer.moe_config)
214
-
215
- return XPUFp8MoEMethod(fp8_config, layer)
216
- elif isinstance(layer, Attention):
217
- return Fp8KVCacheMethod(self)
218
- return None
219
-
191
def get_quant_method(
192
self, layer: torch.nn.Module, prefix: str
193
) -> "QuantizeMethodBase | None":
vllm/model_executor/layers/quantization/inc.py
+2
-24
@@ -38,7 +38,6 @@ class INCConfig(QuantizationConfig):
38
"awq",
39
"awq:marlin",
40
"marlin",
41
- "ipex",
41
}
42
43
def __init__(
@@ -410,31 +409,10 @@ class INCConfig(QuantizationConfig):
409
return UnquantizedLinearMethod()
410
else:
411
return None
413
- from vllm.model_executor.layers.quantization.ipex_quant import (
414
- IPEXAWQLinearMethod,
415
- IPEXConfig,
416
- IPEXGPTQLinearMethod,
412
+ raise NotImplementedError(
413
+ "INC quantization is not supported during xpu kernel migration."
414
)
415
419
- if isinstance(layer, (LinearBase, ParallelLMHead)):
420
- if "awq" in self.packing_format:
421
- config = IPEXConfig(
422
- method="awq", weight_bits=weight_bits, group_size=group_size
423
- )
424
- return IPEXAWQLinearMethod(config)
425
- elif "gptq" in self.packing_format:
426
- config = IPEXConfig(
427
- method="gptq", weight_bits=weight_bits, group_size=group_size
428
- )
429
- return IPEXGPTQLinearMethod(config)
430
- else:
431
- raise ValueError(
432
- f"ipex backend only supports awq "
433
- f"and gptq format,but got {self.packing_format}"
434
- )
435
- else:
436
- return None
437
-
416
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
417
if prefix and self.extra_config:
418
for layer_name in self.extra_config:
vllm/model_executor/layers/quantization/ipex_quant.py
deleted
-403
@@ -1,403 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Any
5
-
6
-import torch
7
-from packaging import version
8
-from torch.nn import Module
9
-
10
-from vllm._ipex_ops import ipex_ops as ops
11
-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
12
-from vllm.model_executor.layers.linear import (
13
- LinearBase,
14
- LinearMethodBase,
15
- UnquantizedLinearMethod,
16
-)
17
-from vllm.model_executor.layers.quantization import (
18
- QuantizationConfig,
19
- QuantizationMethods,
20
-)
21
-from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
22
-from vllm.model_executor.layers.quantization.fp8 import (
23
- Fp8Config,
24
- Fp8LinearMethod,
25
- Fp8OnlineMoEMethod,
26
-)
27
-from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
28
-from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
29
-from vllm.model_executor.utils import replace_parameter
30
-from vllm.platforms import current_platform
31
-
32
-MIN_IPEX_VERSION = "2.6.0"
33
-
34
-
35
-class IPEXConfig(QuantizationConfig):
36
- """INT8 quantization config class using IPEX for the CPU/XPU backend,
37
- including AWQ, GPTQ.
38
- """
39
-
40
- IPEX_QUANT_METHOD_MAP = {
41
- "awq": 1,
42
- "gptq": 0,
43
- }
44
-
45
- def __init__(
46
- self,
47
- method: str,
48
- weight_bits: int,
49
- group_size: int,
50
- modules_to_not_convert: list[str] | None = None,
51
- desc_act: bool | None = None,
52
- lm_head_quantized: bool | None = None,
53
- is_sym: bool | None = None,
54
- ) -> None:
55
- super().__init__()
56
- self.method = method
57
- self.weight_bits = weight_bits
58
- self.group_size = group_size
59
- self.modules_to_not_convert = modules_to_not_convert or []
60
- self.desc_act = desc_act
61
- self.lm_head_quantized = lm_head_quantized
62
- self.is_sym = is_sym
63
- self.pack_factor = 32 // self.weight_bits
64
-
65
- if self.weight_bits not in [4]:
66
- raise ValueError(
67
- f"IPEX quantization supports weight bits [4], "
68
- f"but got {self.weight_bits}."
69
- )
70
-
71
- if self.method not in ["awq", "gptq"]:
72
- raise ValueError(
73
- f"IPEX quantization supports [awq, gptq], but got {self.method}."
74
- )
75
-
76
- def __repr__(self) -> str:
77
- return (
78
- f"IPEXConfig(method={self.method},"
79
- f"weight_bits={self.weight_bits}, "
80
- f"group_size={self.group_size})"
81
- )
82
-
83
- @classmethod
84
- def get_name(cls) -> QuantizationMethods:
85
- return "ipex"
86
-
87
- @classmethod
88
- def get_supported_act_dtypes(cls) -> list[torch.dtype]:
89
- return [torch.bfloat16, torch.float16]
90
-
91
- @classmethod
92
- def get_min_capability(cls) -> int:
93
- return -1
94
-
95
- @staticmethod
96
- def get_config_filenames() -> list[str]:
97
- return [
98
- "quant_config.json",
99
- "quantize_config.json",
100
- ]
101
-
102
- @classmethod
103
- def from_config(cls, config: dict[str, Any]) -> "IPEXConfig":
104
- method = cls.get_from_keys(config, ["quant_method"]).lower()
105
- if method == "awq":
106
- weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
107
- group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
108
- modules_to_not_convert = cls.get_from_keys_or(
109
- config, ["modules_to_not_convert"], None
110
- )
111
- is_sym = not cls.get_from_keys_or(config, ["zero_point"], default=False)
112
- return cls(
113
- method,
114
- weight_bits,
115
- group_size,
116
- modules_to_not_convert,
117
- False,
118
- False,
119
- is_sym,
120
- )
121
- # otherwise for gptq
122
- weight_bits = cls.get_from_keys(config, ["bits"])
123
- group_size = cls.get_from_keys(config, ["group_size"])
124
- lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
125
- desc_act = cls.get_from_keys_or(config, ["desc_act"], default=False)
126
- is_sym = cls.get_from_keys_or(config, ["sym"], default=True)
127
- return cls(
128
- method, weight_bits, group_size, [], desc_act, lm_head_quantized, is_sym
129
- )
130
-
131
- @classmethod
132
- def override_quantization_method(
133
- cls, hf_quant_cfg, user_quant
134
- ) -> QuantizationMethods | None:
135
- if not current_platform.is_xpu():
136
- return None
137
-
138
- quant_method = hf_quant_cfg.get("quant_method", "").lower()
139
-
140
- if quant_method in ["awq", "gptq"]:
141
- return cls.get_name()
142
-
143
- return None
144
-
145
- def get_quant_method(
146
- self, layer: torch.nn.Module, prefix: str
147
- ) -> "LinearMethodBase | None":
148
- if isinstance(layer, LinearBase):
149
- if self.method == "awq":
150
- if is_layer_skipped(
151
- prefix,
152
- self.modules_to_not_convert,
153
- self.packed_modules_mapping,
154
- skip_with_substr=True,
155
- ):
156
- return UnquantizedLinearMethod()
157
- return IPEXAWQLinearMethod(self)
158
- if self.method == "gptq":
159
- return IPEXGPTQLinearMethod(self)
160
- return None
161
-
162
-
163
-class IPEXGPTQLinearMethod(GPTQLinearMethod):
164
- """GPTQ linear method using IPEX for the CPU/XPU backend."""
165
-
166
- def __init__(self, quant_config: IPEXConfig):
167
- self.quant_config = quant_config # type: ignore
168
-
169
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
170
- bias = layer.bias if not layer.skip_bias_add else None
171
-
172
- try:
173
- import intel_extension_for_pytorch as ipex
174
-
175
- if version.parse(ipex.__version__) < version.parse(MIN_IPEX_VERSION):
176
- raise ImportError(
177
- "intel_extension_for_pytorch version is "
178
- "wrong. Please install "
179
- f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}."
180
- )
181
- except ImportError as err:
182
- raise ImportError(
183
- "Please install "
184
- f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
185
- f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
186
- " to use IPEX-AWQ linear method."
187
- ) from err
188
- # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
189
- # with better performance.
190
- lowp_mode = ipex.quantization.WoqLowpMode.INT8
191
- # The weight will be de-packed from INT4 to INT8.
192
- weight_dtype = ipex.quantization.WoqWeightDtype.INT4
193
- # The float activation will be quantized (dynamic, per-token) to INT8.
194
- act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH_IC_BLOCK
195
-
196
- assert isinstance(self.quant_config, IPEXConfig)
197
- qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
198
- weight_dtype=weight_dtype,
199
- lowp_mode=lowp_mode,
200
- act_quant_mode=act_quant_mode,
201
- group_size=self.quant_config.group_size,
202
- )
203
- layer.ipex_output_size = layer.qweight.shape[-1]
204
- g_idx = layer.g_idx if self.quant_config.desc_act else None
205
- layer.ipex_qlinear = (
206
- ipex.llm.quantization.woq_linear.IPEXWeightOnlyQuantizedLinear.from_weight(
207
- layer.qweight,
208
- layer.scales,
209
- layer.qzeros,
210
- layer.qweight.size(0),
211
- layer.ipex_output_size,
212
- qconfig=qconfig,
213
- g_idx=g_idx,
214
- bias=bias,
215
- group_size=self.quant_config.group_size,
216
- quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["gptq"],
217
- weight_qscheme="sym" if self.quant_config.is_sym else "asym",
218
- )
219
- )
220
-
221
- def apply(
222
- self,
223
- layer: torch.nn.Module,
224
- x: torch.Tensor,
225
- bias: torch.Tensor | None = None,
226
- ) -> torch.Tensor:
227
- reshaped_x = x.reshape(-1, x.shape[-1])
228
- out = layer.ipex_qlinear(reshaped_x)
229
- return out.reshape(x.shape[:-1] + (layer.ipex_output_size,))
230
-
231
-
232
-class IPEXAWQLinearMethod(AWQLinearMethod):
233
- """AWQ linear method using IPEX for the CPU/XPU backend."""
234
-
235
- def __init__(self, quant_config: IPEXConfig):
236
- self.quant_config = quant_config # type: ignore
237
-
238
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
239
- super().process_weights_after_loading(layer=layer)
240
-
241
- bias = layer.bias if not layer.skip_bias_add else None
242
-
243
- try:
244
- import intel_extension_for_pytorch as ipex
245
-
246
- if version.parse(ipex.__version__) < version.parse(MIN_IPEX_VERSION):
247
- raise ImportError(
248
- "intel_extension_for_pytorch version is "
249
- "wrong. Please install "
250
- f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}."
251
- )
252
- except ImportError as err:
253
- raise ImportError(
254
- "Please install "
255
- f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
256
- f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
257
- " to use IPEX-AWQ linear method."
258
- ) from err
259
-
260
- # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
261
- # with better performance.
262
- lowp_mode = ipex.quantization.WoqLowpMode.INT8
263
- # The weight will be de-packed from INT4 to INT8.
264
- weight_dtype = ipex.quantization.WoqWeightDtype.INT4
265
- # The float activation will be quantized (dynamic, per-token) to INT8.
266
- act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH
267
-
268
- assert isinstance(self.quant_config, IPEXConfig)
269
- qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
270
- weight_dtype=weight_dtype,
271
- lowp_mode=lowp_mode,
272
- act_quant_mode=act_quant_mode,
273
- group_size=self.quant_config.group_size,
274
- )
275
-
276
- layer.ipex_output_size = layer.qweight.size(1) * self.quant_config.pack_factor
277
- layer.ipex_qlinear = (
278
- ipex.llm.quantization.woq_linear.IPEXWeightOnlyQuantizedLinear.from_weight(
279
- layer.qweight,
280
- layer.scales,
281
- layer.qzeros,
282
- layer.qweight.size(0),
283
- layer.ipex_output_size,
284
- qconfig=qconfig,
285
- bias=bias,
286
- group_size=self.quant_config.group_size,
287
- quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["awq"], # type: ignore
288
- weight_qscheme="sym" if self.quant_config.is_sym else "asym",
289
- )
290
- )
291
-
292
- def apply(
293
- self,
294
- layer: torch.nn.Module,
295
- x: torch.Tensor,
296
- bias: torch.Tensor | None = None,
297
- ) -> torch.Tensor:
298
- reshaped_x = x.reshape(-1, x.shape[-1])
299
- out = layer.ipex_qlinear(reshaped_x)
300
- return out.reshape(x.shape[:-1] + (layer.ipex_output_size,))
301
-
302
-
303
-class XPUFp8LinearMethod(Fp8LinearMethod):
304
- def __init__(self, quant_config: Fp8Config):
305
- super().__init__(quant_config)
306
-
307
- def process_weights_after_loading(self, layer: Module) -> None:
308
- if getattr(layer, "_already_called_process_weights_after_loading", False):
309
- return
310
- # If checkpoint not serialized fp8, quantize the weights.
311
- if not self.quant_config.is_checkpoint_fp8_serialized:
312
- qweight, weight_scale = ops.scaled_fp8_quant(layer.weight, scale=None)
313
- # Update the layer with the new values.
314
- replace_parameter(layer, "weight", qweight.data)
315
- replace_parameter(layer, "weight_scale", weight_scale.data)
316
- layer.input_scale = None
317
-
318
- def apply(
319
- self,
320
- layer: torch.nn.Module,
321
- x: torch.Tensor,
322
- bias: torch.Tensor | None = None,
323
- ) -> torch.Tensor:
324
- weight = layer.weight.data
325
- weight_scale = layer.weight_scale.data
326
- output = torch.ops.torch_ipex.fp8_gemm_w8a16(
327
- x, weight, True, weight_scale, bias
328
- )
329
- return output
330
-
331
-
332
-class XPUFp8MoEMethod(Fp8OnlineMoEMethod):
333
- def __init__(self, quant_config: Fp8Config, layer: torch.nn.Module):
334
- super().__init__(quant_config, layer)
335
- self.quant_config = quant_config
336
-
337
- def process_weights_after_loading(self, layer: Module) -> None:
338
- if getattr(layer, "_already_called_process_weights_after_loading", False):
339
- return
340
- if not self.quant_config.is_checkpoint_fp8_serialized:
341
- fp8_dtype = current_platform.fp8_dtype()
342
- w13_weight = torch.empty_like(layer.w13_weight.data, dtype=fp8_dtype)
343
- w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype)
344
-
345
- # Re-initialize w13_scale because we directly quantize
346
- # merged w13 weights and generate a single scaling factor.
347
- layer.w13_weight_scale = torch.nn.Parameter(
348
- torch.ones(
349
- layer.local_num_experts,
350
- dtype=torch.float32,
351
- device=w13_weight.device,
352
- ),
353
- requires_grad=False,
354
- )
355
- for expert in range(layer.local_num_experts):
356
- w13_weight[expert, :, :], layer.w13_weight_scale[expert] = (
357
- ops.scaled_fp8_quant(layer.w13_weight.data[expert, :, :])
358
- )
359
- w2_weight[expert, :, :], layer.w2_weight_scale[expert] = (
360
- ops.scaled_fp8_quant(layer.w2_weight.data[expert, :, :])
361
- )
362
- replace_parameter(layer, "w13_weight", w13_weight)
363
- replace_parameter(layer, "w2_weight", w2_weight)
364
-
365
- import intel_extension_for_pytorch as ipex
366
-
367
- ep_rank_start = self.moe.ep_rank * self.moe.num_local_experts
368
- layer.ipex_fusion = ipex.llm.modules.GatedMLPMOE(
369
- layer.w13_weight,
370
- layer.w2_weight,
371
- w1_scale_inv=layer.w13_weight_scale,
372
- w2_scale_inv=layer.w2_weight_scale,
373
- a1_scale_inv=layer.w13_input_scale,
374
- a2_scale_inv=layer.w2_input_scale,
375
- use_prepack=True,
376
- experts_start_id=ep_rank_start,
377
- )
378
-
379
- def get_fused_moe_quant_config(
380
- self, layer: torch.nn.Module
381
- ) -> FusedMoEQuantConfig | None:
382
- return None
383
-
384
- @property
385
- def is_monolithic(self) -> bool:
386
- return True
387
-
388
- def apply_monolithic(
389
- self,
390
- layer: torch.nn.Module,
391
- x: torch.Tensor,
392
- router_logits: torch.Tensor,
393
- ) -> torch.Tensor:
394
- return layer.ipex_fusion(
395
- x,
396
- layer.use_grouped_topk,
397
- layer.top_k,
398
- router_logits,
399
- layer.renormalize,
400
- layer.topk_group,
401
- layer.num_expert_group,
402
- custom_routing_function=layer.custom_routing_function,
403
- )
vllm/model_executor/layers/rotary_embedding/base.py
+2
-5
@@ -232,17 +232,14 @@ class RotaryEmbedding(RotaryEmbeddingBase):
232
query: torch.Tensor,
233
key: torch.Tensor | None = None,
234
) -> tuple[torch.Tensor, torch.Tensor | None]:
235
- from vllm._ipex_ops import ipex_ops as ops
236
-
235
self._match_cos_sin_cache_dtype(query)
236
# ops.rotary_embedding() is an in-place operation
237
# that updates the query and key tensors.
238
if key is None:
241
- # XPU kernel doesn't support key=None so fall back to native impl
242
- # TODO(sarckk): add support for optional key in
243
- # ipex.llm.functional.rotary_embedding_batched
239
return self.forward_native(positions, query, key)
240
else:
241
+ from vllm import _custom_ops as ops
242
+
243
ops.rotary_embedding(
244
positions,
245
query,
vllm/platforms/__init__.py
-2
@@ -132,8 +132,6 @@ def xpu_platform_plugin() -> str | None:
132
is_xpu = False
133
logger.debug("Checking if XPU platform is available.")
134
try:
135
- # installed IPEX if the machine has XPUs.
136
- import intel_extension_for_pytorch # noqa: F401
135
import torch
136
137
if supports_xccl():
vllm/platforms/xpu.py
+11
-3
@@ -7,6 +7,11 @@ from typing import TYPE_CHECKING
7
8
import torch
9
10
+# import custom ops, trigger op registration
11
+import vllm_xpu_kernels._C # noqa
12
+import vllm_xpu_kernels._moe_C # noqa
13
+import vllm_xpu_kernels._xpu_C # noqa
14
+
15
from vllm.logger import init_logger
16
from vllm.v1.attention.backends.registry import AttentionBackendEnum
17
@@ -55,6 +60,9 @@ class XPUPlatform(Platform):
60
dtype = attn_selector_config.dtype
61
if attn_selector_config.use_sparse:
62
raise NotImplementedError("Sparse Attention is not supported on XPU.")
63
+ if attn_selector_config.use_mla:
64
+ logger.info_once("Using Triton MLA backend on V1 engine.")
65
+ return AttentionBackendEnum.TRITON_MLA.get_path()
66
if selected_backend == AttentionBackendEnum.TRITON_ATTN:
67
logger.info_once("Using Triton backend.")
68
return AttentionBackendEnum.TRITON_ATTN.get_path()
@@ -78,9 +86,9 @@ class XPUPlatform(Platform):
86
87
@classmethod
88
def get_supported_vit_attn_backends(cls) -> list["AttentionBackendEnum"]:
81
- # XPU only supports FLASH_ATTN for vision attention.
89
return [
90
AttentionBackendEnum.FLASH_ATTN,
91
+ AttentionBackendEnum.TORCH_SDPA,
92
]
93
94
@classmethod
@@ -145,7 +153,7 @@ class XPUPlatform(Platform):
153
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
154
cache_config = vllm_config.cache_config
155
model_config = vllm_config.model_config
148
- # in V1(or with ipex chunked prefill) block_size is 64
156
+ # in V1(or with chunked prefill) block_size is 64
157
if cache_config and cache_config.block_size is None:
158
cache_config.block_size = 64
159
@@ -206,7 +214,7 @@ class XPUPlatform(Platform):
214
215
@classmethod
216
def fp8_dtype(cls) -> torch.dtype:
209
- return torch.float8_e5m2
217
+ return torch.float8_e4m3fn
218
219
@classmethod
220
def is_data_center_gpu(cls) -> bool:
vllm/v1/attention/backends/fa_utils.py
+3
-2
@@ -16,12 +16,13 @@ if current_platform.is_cuda():
16
)
17
18
elif current_platform.is_xpu():
19
+ from vllm import _custom_ops as ops
20
+
21
+ reshape_and_cache_flash = ops.reshape_and_cache_flash
22
from vllm._ipex_ops import ipex_ops
23
21
- reshape_and_cache_flash = ipex_ops.reshape_and_cache_flash
24
flash_attn_varlen_func = ipex_ops.flash_attn_varlen_func # type: ignore[assignment]
25
get_scheduler_metadata = ipex_ops.get_scheduler_metadata # type: ignore[assignment]
24
-
26
elif current_platform.is_rocm():
27
try:
28
from flash_attn import flash_attn_varlen_func # type: ignore[no-redef]
vllm/v1/attention/backends/registry.py
-1
@@ -69,7 +69,6 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
69
"vllm.v1.attention.backends.mla.flashmla_sparse.FlashMLASparseBackend"
70
)
71
FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend"
72
- IPEX = "vllm.v1.attention.backends.ipex.IpexAttentionBackend"
72
NO_ATTENTION = "vllm.v1.attention.backends.no_attention.NoAttentionBackend"
73
FLEX_ATTENTION = "vllm.v1.attention.backends.flex_attention.FlexAttentionBackend"
74
TREE_ATTN = "vllm.v1.attention.backends.tree_attn.TreeAttentionBackend"
vllm/v1/worker/xpu_worker.py
+8
-4
@@ -1,10 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import gc
4
import os
5
from typing import Any
6
7
import torch
7
-import torch.distributed
8
9
from vllm.config import VllmConfig
10
from vllm.logger import init_logger
@@ -85,7 +85,14 @@ class XPUWorker(Worker):
85
current_platform.dist_backend,
86
)
87
88
+ # Set random seed.
89
+ set_random_seed(self.model_config.seed)
90
+
91
+ # Now take memory snapshot after NCCL is initialized
92
+ gc.collect()
93
torch.xpu.empty_cache()
94
+
95
+ # take current memory snapshot
96
self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device)
97
self.requested_memory = request_memory(init_snapshot, self.cache_config)
98
logger.debug("worker init memory snapshot: %r", self.init_snapshot)
@@ -93,9 +100,6 @@ class XPUWorker(Worker):
100
"worker requested memory: %sGiB", format_gib(self.requested_memory)
101
)
102
96
- # Set random seed.
97
- set_random_seed(self.model_config.seed)
98
-
103
# Initialize workspace manager
104
num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1
105
init_workspace_manager(self.device, num_ubatches)