A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 16:30 UTC
[LoRA]Disable linear LoRA kernel PDL (#31777)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jan 6, 2026 at 23:12 UTC
cbd4690a03a458580370660dc443ee9de59416cd
4 files changed
+14
-10
docs/features/lora.md
+1
-1
@@ -277,7 +277,7 @@ The new format of `--lora-modules` is mainly to support the display of parent mo
277
278
## LoRA Support for Tower and Connector of Multi-Modal Model
279
280
-Currently, vLLM experimentally supports LoRA for the Tower and Connector components of multi-modal models. To enable this feature, you need to implement the corresponding token helper functions for the tower and connector. For more details on the rationale behind this approach, please refer to [PR 26674](https://github.com/vllm-project/vllm/pull/26674). We welcome contributions to extend LoRA support to additional models' tower and connector.
280
+Currently, vLLM experimentally supports LoRA for the Tower and Connector components of multi-modal models. To enable this feature, you need to implement the corresponding token helper functions for the tower and connector. For more details on the rationale behind this approach, please refer to [PR 26674](https://github.com/vllm-project/vllm/pull/26674). We welcome contributions to extend LoRA support to additional models' tower and connector. Please refer to [Issue 31479](https://github.com/vllm-project/vllm/issues/31479) to check the current model support status.
281
282
## Default LoRA Models For Multimodal Models
283
vllm/lora/ops/triton_ops/fused_moe_lora_op.py
+7
-3
@@ -231,9 +231,9 @@ def _fused_moe_lora_shrink(
231
num_stages: int,
232
split_k: int,
233
mul_routed_weight: bool = False,
234
+ use_gdc: bool = False,
235
) -> None:
236
w1_lora_a_stacked = lora_a_stacked[0]
236
- use_gdc = supports_pdl(qcurr_hidden_states.device)
237
shrink_config = {
238
"BLOCK_SIZE_M": block_size_m,
239
"BLOCK_SIZE_N": block_size_n,
@@ -326,6 +326,7 @@ def _fused_moe_lora_expand(
326
split_k: int,
327
mul_routed_weight: bool = False,
328
offset: int = 0,
329
+ use_gdc: bool = False,
330
) -> None:
331
b_ptr = _get_ptr(lora_b_stacked, device)
332
K = max_lora_rank
@@ -337,7 +338,6 @@ def _fused_moe_lora_expand(
338
-1, a_intermediate_cache1.shape[3]
339
)
340
340
- use_gdc = supports_pdl(a_intermediate_cache1.device)
341
expand_config = {
342
"BLOCK_SIZE_M": block_size_m,
343
"BLOCK_SIZE_N": block_size_n,
@@ -466,7 +466,7 @@ def _fused_moe_lora(
466
dtype=output.dtype,
467
device=device,
468
)
469
-
469
+ use_gdc = supports_pdl(device) and not fully_sharded
470
_fused_moe_lora_shrink(
471
a_intermediate_cache1,
472
qcurr_hidden_states,
@@ -495,6 +495,7 @@ def _fused_moe_lora(
495
shrink_num_stages,
496
shrink_split_k,
497
mul_routed_weight,
498
+ use_gdc=use_gdc,
499
)
500
501
if fully_sharded:
@@ -542,6 +543,7 @@ def _fused_moe_lora(
543
expand_split_k,
544
mul_routed_weight,
545
offset,
546
+ use_gdc=use_gdc,
547
)
548
549
@@ -604,6 +606,7 @@ def _fused_moe_lora_shrink_fake(
606
num_stages: int,
607
split_k: int,
608
mul_routed_weight: bool = False,
609
+ use_gdc: bool = False,
610
) -> None:
611
return
612
@@ -637,6 +640,7 @@ def _fused_moe_lora_expand_fake(
640
num_stages: int,
641
split_k: int,
642
mul_routed_weight: bool = False,
643
+ use_gdc: bool = False,
644
) -> None:
645
return
646
vllm/lora/ops/triton_ops/lora_expand_op.py
+3
-3
@@ -14,8 +14,6 @@ from vllm.lora.ops.triton_ops.utils import _get_lora_b_ptr, get_lora_op_configs
14
from vllm.triton_utils import tl, triton
15
from vllm.utils.torch_utils import direct_register_custom_op
16
17
-from .utils import supports_pdl
18
-
17
18
@triton.jit
19
def _lora_expand_kernel(
@@ -241,7 +239,9 @@ def _lora_expand(
239
# thread blocks simply exit.
240
MAX_LORAS,
241
)
244
- use_gdc = supports_pdl(inputs.device)
242
+ # We disable PDL temporarily because LoRA kernels are not launching back-to-back,
243
+ # making PDL invalid and affecting the kernel performance.
244
+ use_gdc = False # supports_pdl(inputs.device)
245
_lora_expand_kernel[grid](
246
inputs,
247
lora_ptr_tensor,
vllm/lora/ops/triton_ops/lora_shrink_op.py
+3
-3
@@ -14,8 +14,6 @@ from vllm.lora.ops.triton_ops.utils import _get_lora_a_ptr, get_lora_op_configs
14
from vllm.triton_utils import tl, triton
15
from vllm.utils.torch_utils import direct_register_custom_op
16
17
-from .utils import supports_pdl
18
-
17
18
@triton.jit
19
def _lora_shrink_kernel(
@@ -221,7 +219,9 @@ def _lora_shrink(
219
# thread blocks exit early.
220
MAX_LORAS,
221
)
224
- use_gdc = supports_pdl(inputs.device)
222
+ # We disable PDL temporarily because LoRA kernels are not launching back-to-back,
223
+ # making PDL invalid and affecting the kernel performance.
224
+ use_gdc = False # supports_pdl(inputs.device)
225
_lora_shrink_kernel[grid](
226
inputs,
227
lora_ptr_tensor,