A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Misc] Gemma3ForConditionalGeneration supports LoRA (#14797)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 14, 2025 at 16:07 UTC
989ecd200701bc011a186956a7b599bacf890d97
1 file changed
+14
-3
vllm/model_executor/models/gemma3_mm.py
+14
-3
@@ -12,6 +12,7 @@ from vllm.config import VllmConfig
12
from vllm.logger import init_logger
13
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
14
from vllm.model_executor.layers.sampler import SamplerOutput
15
+from vllm.model_executor.models.module_mapping import MultiModelKeys
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
@@ -23,7 +24,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
24
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
25
from vllm.sequence import IntermediateTensors
26
26
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
27
+from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
28
+ SupportsMultiModal, SupportsPP)
29
from .siglip import SiglipVisionModel
30
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
31
maybe_prefix, merge_multimodal_embeddings)
@@ -371,8 +373,8 @@ class Gemma3MultiModalProjector(nn.Module):
373
@MULTIMODAL_REGISTRY.register_processor(Gemma3MultiModalProcessor,
374
info=Gemma3ProcessingInfo,
375
dummy_inputs=Gemma3DummyInputsBuilder)
374
-class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal,
375
- SupportsPP):
376
+class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
377
+ SupportsLoRA):
378
packed_modules_mapping = {
379
"qkv_proj": [
380
"q_proj",
@@ -614,3 +616,12 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal,
616
torch.Tensor]]) -> Set[str]:
617
loader = AutoWeightsLoader(self)
618
return loader.load_weights(weights)
619
+
620
+ def get_mm_mapping(self) -> MultiModelKeys:
621
+ """
622
+ Get the module prefix in multimodal models
623
+ """
624
+ return MultiModelKeys.from_string_field(
625
+ language_model="language_model",
626
+ connector="multi_modal_projector",
627
+ tower_model="vision_tower")