A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 12:15 UTC
[Bugfix] Fix getting device for MoE LoRA (#29475)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 27, 2025 at 15:16 UTC
c069086b9c9b8212b0a8544eb25b6af65c16762d
2 files changed
+12
-1
vllm/lora/layers/fused_moe.py
+3
-1
@@ -30,6 +30,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
30
FusedMoEModularMethod,
31
)
32
33
+from .utils import _get_lora_device
34
+
35
36
class FusedMoEWithLoRA(BaseLayerWithLoRA):
37
def __init__(self, base_layer: FusedMoE) -> None:
@@ -41,7 +43,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
43
)
44
self.tp_size = get_tensor_model_parallel_world_size()
45
self.tp_rank = get_tensor_model_parallel_rank()
44
- self.device = base_layer.w2_weight.device
46
+ self.device = _get_lora_device(base_layer)
47
self._w13_slices = 2
48
self._inject_lora_into_fused_moe()
49
vllm/lora/layers/utils.py
+9
@@ -33,6 +33,15 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
33
# HQQ marlin
34
elif hasattr(base_layer, "W_q"):
35
return base_layer.W_q.device
36
+ # MoE layer
37
+ elif hasattr(base_layer, "w2_weight"):
38
+ return base_layer.w2_weight.device
39
+ # MoE Compressed Tensor
40
+ elif hasattr(base_layer, "w2_weight_packed"):
41
+ return base_layer.w2_weight_packed.device
42
+ # MoE GPTQ/AWQ/GGUF
43
+ elif hasattr(base_layer, "w2_qweight"):
44
+ return base_layer.w2_qweight.device
45
else:
46
raise ValueError(f"Unsupported base layer: {base_layer}")
47