A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Adjust max_position_embeddings for LoRA compatibility (#8957)
Jee Jee Li committed
Sep 30, 2024 at 20:57 UTC
1cabfcefb64a489c8ff9dcb289b4dd47cf8f89cf
1 file changed
+9
-2
vllm/worker/model_runner.py
+9
-2
@@ -1037,9 +1037,17 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1037
assert supports_lora(
1038
self.model
1039
), f"{self.model.__class__.__name__} does not support LoRA yet."
1040
+
1041
if supports_multimodal(self.model):
1042
logger.warning("Regarding multimodal models, vLLM currently "
1043
"only supports adding LoRA to language model.")
1044
+ # It's necessary to distinguish between the max_position_embeddings
1045
+ # of VLMs and LLMs.
1046
+ if hasattr(self.model.config, "max_position_embeddings"):
1047
+ max_pos_embeddings = self.model.config.max_position_embeddings
1048
+ else:
1049
+ max_pos_embeddings = (
1050
+ self.model.config.text_config.max_position_embeddings)
1051
1052
self.lora_manager = LRUCacheWorkerLoRAManager(
1053
self.scheduler_config.max_num_seqs,
@@ -1049,8 +1057,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1057
self.device,
1058
self.model.embedding_modules,
1059
self.model.embedding_padding_modules,
1052
- max_position_embeddings=self.model.config.
1053
- max_position_embeddings,
1060
+ max_position_embeddings=max_pos_embeddings,
1061
)
1062
self.model = self.lora_manager.create_lora_manager(self.model)
1063