A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Fix LoRA with PP (#7292)
Murali Andoorveedu committed
Aug 8, 2024 at 00:02 UTC
6dffa4b0a6120159ef2fe44d695a46817aff65bc
1 file changed
+3
vllm/lora/models.py
+3
@@ -25,6 +25,7 @@ from vllm.lora.punica import PunicaWrapper
25
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26
parse_fine_tuned_lora_name, replace_submodule)
27
from vllm.model_executor.models.interfaces import SupportsLoRA
28
+from vllm.model_executor.models.utils import PPMissingLayer
29
from vllm.utils import is_pin_memory_available
30
31
logger = init_logger(__name__)
@@ -432,6 +433,8 @@ class LoRAModelManager(AdapterModelManager):
433
def _create_lora_modules(self):
434
for module_name, module in self.model.named_modules(
435
remove_duplicate=False):
436
+ if isinstance(module, PPMissingLayer):
437
+ continue
438
if not self._match_target_modules(module_name):
439
continue
440
parts = module_name.split(".")[-1]