@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6dffa4b0a6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix LoRA with PP (#7292)

Murali Andoorveedu committed Aug 8, 2024 at 00:02 UTC 6dffa4b0a6120159ef2fe44d695a46817aff65bc
1 file changed +3
vllm/lora/models.py
+3
@@ -25,6 +25,7 @@ from vllm.lora.punica import PunicaWrapper
25 from vllm.lora.utils import (from_layer, from_layer_logits_processor,
26 parse_fine_tuned_lora_name, replace_submodule)
27 from vllm.model_executor.models.interfaces import SupportsLoRA
28 +from vllm.model_executor.models.utils import PPMissingLayer
29 from vllm.utils import is_pin_memory_available
30
31 logger = init_logger(__name__)
@@ -432,6 +433,8 @@ class LoRAModelManager(AdapterModelManager):
433 def _create_lora_modules(self):
434 for module_name, module in self.model.named_modules(
435 remove_duplicate=False):
436 + if isinstance(module, PPMissingLayer):
437 + continue
438 if not self._match_target_modules(module_name):
439 continue
440 parts = module_name.split(".")[-1]