A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Improve static type checking in `LoRAModelRunnerMixin` (#17104)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 24, 2025 at 14:14 UTC
21f4f1c9a4df20ae4e744ad46a426aaf981b9412
1 file changed
+6
-10
vllm/v1/worker/lora_model_runner_mixin.py
+6
-10
@@ -28,20 +28,16 @@ class LoRAModelRunnerMixin:
28
scheduler_config: SchedulerConfig,
29
lora_config: LoRAConfig, device: str) -> nn.Module:
30
31
- assert supports_lora(
32
- model), f"{model.__class__.__name__} does not support LoRA yet."
31
+ if not supports_lora(model):
32
+ raise ValueError(
33
+ f"{model.__class__.__name__} does not support LoRA yet.")
34
35
if supports_multimodal(model):
36
logger.warning("Regarding multimodal models, vLLM currently "
37
"only supports adding LoRA to language model.")
38
38
- # It's necessary to distinguish between the max_position_embeddings
39
- # of VLMs and LLMs.
40
- if hasattr(model.config, "max_position_embeddings"):
41
- max_pos_embeddings = model.config.max_position_embeddings
42
- else:
43
- max_pos_embeddings = (
44
- model.config.text_config.max_position_embeddings)
39
+ # Use get_text_config() in case of multimodal models
40
+ text_config = model_config.hf_config.get_text_config()
41
42
# Add LoRA Manager to the Model Runner
43
self.lora_manager = LRUCacheWorkerLoRAManager(
@@ -52,7 +48,7 @@ class LoRAModelRunnerMixin:
48
device,
49
model.embedding_modules,
50
model.embedding_padding_modules,
55
- max_position_embeddings=max_pos_embeddings,
51
+ max_position_embeddings=text_config.max_position_embeddings,
52
)
53
return self.lora_manager.create_lora_manager(model)
54