A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[XPU] Fix the bug of LoRA logits on the XPU platform (#24081)
Signed-off-by: chzhang <chaojun.zhang@intel.com>
Chaojun Zhang committed
Sep 3, 2025 at 08:21 UTC
862f2ef893d9751db0a92bd2d4ae0e3d9677872f
3 files changed
+15
-5
vllm/lora/layers.py
+1
-1
@@ -1151,7 +1151,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1151
lora_logits = lora_logits.mT
1152
indices_padded = self.punica_wrapper.sampler_indices_padded
1153
1154
- if current_platform.is_tpu():
1154
+ if current_platform.is_tpu() or current_platform.is_xpu():
1155
indices_padded = indices_padded[:logits.size(0)]
1156
1157
lora_logits = (lora_logits.reshape(
vllm/lora/punica_wrapper/punica_xpu.py
+10
-3
@@ -225,6 +225,13 @@ class PunicaWrapperXPU(PunicaWrapperBase):
225
add_inputs=True,
226
**kwargs)
227
228
+ @property
229
+ def sampler_indices_padded(self) -> torch.Tensor:
230
+ """
231
+ This property provides access to padded sampler indices.
232
+ """
233
+ return self._sampler_indices_padded[:]
234
+
235
def add_lora_logits(self,
236
y: torch.Tensor,
237
x: torch.Tensor,
@@ -259,11 +266,11 @@ class PunicaWrapperXPU(PunicaWrapperBase):
266
buffer = torch.zeros((x.size(0), r),
267
dtype=torch.float32,
268
device=x.device)
262
-
263
- bgmv_shrink(x, lora_a_stacked, buffer, self.sampler_indices, scale)
269
+ sampler_indices = torch.narrow(self._sampler_indices, 0, 0, x.size(0))
270
+ bgmv_shrink(x, lora_a_stacked, buffer, sampler_indices, scale)
271
bgmv_expand(buffer,
272
lora_b_stacked,
273
y,
267
- self.sampler_indices,
274
+ sampler_indices,
275
add_inputs=True)
276
return y.view_as(y_org)
vllm/platforms/xpu.py
+4
-1
@@ -91,7 +91,7 @@ class XPUPlatform(Platform):
91
cache_config.block_size = 64
92
93
# lazy import to avoid circular import
94
- from vllm.config import CUDAGraphMode
94
+ from vllm.config import CompilationLevel, CUDAGraphMode
95
compilation_config = vllm_config.compilation_config
96
if compilation_config.cudagraph_mode is None or \
97
compilation_config.cudagraph_mode.max_cudagraph_mode() \
@@ -100,6 +100,9 @@ class XPUPlatform(Platform):
100
"cudagraphs. Fallback to cudagraph_mode=NONE")
101
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
102
103
+ if vllm_config.lora_config is not None:
104
+ compilation_config.level = CompilationLevel.NO_COMPILATION
105
+
106
# check and update parallel config
107
parallel_config = vllm_config.parallel_config
108
parallel_config.worker_cls = "vllm.v1.worker.xpu_worker.XPUWorker"