A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 04:15 UTC
[Model Runner V2] Fix lora `Triton Error [CUDA]: device-side assert triggered` (#43139)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
Wentao Ye committed
May 20, 2026 at 21:00 UTC
9640970de20b15ade9eb3859825637f64e81ed8c
1 file changed
+14
-1
vllm/v1/worker/gpu/model_runner.py
+14
-1
@@ -37,6 +37,7 @@ from vllm.distributed.parallel_state import (
37
)
38
from vllm.forward_context import BatchDescriptor, set_forward_context
39
from vllm.logger import init_logger
40
+from vllm.lora.layers import LoRAMapping
41
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
42
initialize_mamba_ssu_backend,
43
)
@@ -1084,7 +1085,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1085
)
1086
block_tables = None
1087
slot_mappings = None
1087
- # FIXME(woosuk): Fix warmup for LoRA.
1088
+ if self.lora_config:
1089
+ # program a no-LoRA mapping here so kernels early-exit instead of
1090
+ # reading uninitialized metadata during dummy runs.
1091
+ # FIXME: Replace this with LoRA warmup:
1092
+ # https://github.com/vllm-project/vllm/pull/35536
1093
+ assert hasattr(self, "lora_manager")
1094
+ self.lora_manager._adapter_manager.set_adapter_mapping(
1095
+ LoRAMapping(
1096
+ index_mapping=(0,) * input_batch.num_tokens_after_padding,
1097
+ prompt_mapping=(0,) * input_batch.num_reqs,
1098
+ is_prefill=True,
1099
+ )
1100
+ )
1101
1102
attn_metadata = None
1103
slot_mappings_by_layer = None