@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1cabfcefb6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Misc] Adjust max_position_embeddings for LoRA compatibility (#8957)

Jee Jee Li committed Sep 30, 2024 at 20:57 UTC 1cabfcefb64a489c8ff9dcb289b4dd47cf8f89cf
1 file changed +9 -2
vllm/worker/model_runner.py
+9 -2
@@ -1037,9 +1037,17 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1037 assert supports_lora(
1038 self.model
1039 ), f"{self.model.__class__.__name__} does not support LoRA yet."
1040 +
1041 if supports_multimodal(self.model):
1042 logger.warning("Regarding multimodal models, vLLM currently "
1043 "only supports adding LoRA to language model.")
1044 + # It's necessary to distinguish between the max_position_embeddings
1045 + # of VLMs and LLMs.
1046 + if hasattr(self.model.config, "max_position_embeddings"):
1047 + max_pos_embeddings = self.model.config.max_position_embeddings
1048 + else:
1049 + max_pos_embeddings = (
1050 + self.model.config.text_config.max_position_embeddings)
1051
1052 self.lora_manager = LRUCacheWorkerLoRAManager(
1053 self.scheduler_config.max_num_seqs,
@@ -1049,8 +1057,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1057 self.device,
1058 self.model.embedding_modules,
1059 self.model.embedding_padding_modules,
1052 - max_position_embeddings=self.model.config.
1053 - max_position_embeddings,
1060 + max_position_embeddings=max_pos_embeddings,
1061 )
1062 self.model = self.lora_manager.create_lora_manager(self.model)
1063