A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[LoRA] Support dynamically initialize `packed_modules_mapping` for VLM with arbitrary components (#18987)
Signed-off-by: isotr0py <2037008807@qq.com> Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jun 1, 2025 at 11:06 UTC
a35ca765a52fff242edf0e9fd3203ea2534aed58
7 files changed
+32
-38
vllm/lora/models.py
+3
-3
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import copy
3
import math
4
import os
5
from collections.abc import Sequence
@@ -34,6 +33,7 @@ from vllm.model_executor.models import SupportsLoRA, supports_multimodal
33
from vllm.model_executor.models.interfaces import is_pooling_model
34
from vllm.model_executor.models.module_mapping import MultiModelKeys
35
from vllm.model_executor.models.utils import PPMissingLayer, WeightsMapper
36
+from vllm.model_executor.utils import get_packed_modules_mapping
37
from vllm.utils import is_pin_memory_available
38
39
logger = init_logger(__name__)
@@ -364,8 +364,8 @@ class LoRAModelManager(AdapterModelManager):
364
# We need to replace rotary emb layer to do batch computation
365
# for long lora.
366
self.supported_lora_modules.append("rotary_emb")
367
- self.packed_modules_mapping = copy.deepcopy(
368
- self.model.packed_modules_mapping)
367
+
368
+ self.packed_modules_mapping = get_packed_modules_mapping(self.model)
369
# Used to indicate whether the model is a multimodal model
370
self.supports_mm: bool = (
371
supports_multimodal(self.model)
vllm/model_executor/model_loader/bitsandbytes_loader.py
+4
-4
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# ruff: noqa: SIM117
3
-import copy
3
import fnmatch
4
import glob
5
import itertools
@@ -36,7 +35,8 @@ from vllm.model_executor.model_loader.weight_utils import (
35
filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
36
pt_weights_iterator, safetensors_weights_iterator)
37
from vllm.model_executor.models import is_pooling_model
39
-from vllm.model_executor.utils import set_weight_attrs
38
+from vllm.model_executor.utils import (get_packed_modules_mapping,
39
+ set_weight_attrs)
40
from vllm.platforms import current_platform
41
42
logger = init_logger(__name__)
@@ -420,8 +420,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
420
f"Model {type(model).__name__} does not support BitsAndBytes "
421
"quantization yet. No 'packed_modules_mapping' found.")
422
self.is_pool_model=is_pooling_model(model)
423
- self.modules_mapping = ParamMapping(
424
- copy.deepcopy(model.packed_modules_mapping))
423
+
424
+ self.modules_mapping = ParamMapping(get_packed_modules_mapping(model))
425
426
# For some models like Molmo, we need to use hf_to_vllm_mapper
427
# to ensure correct loading of weights.
vllm/model_executor/models/intern_vit.py
+4
@@ -415,6 +415,10 @@ class InternVisionEncoder(nn.Module):
415
416
class InternVisionModel(nn.Module):
417
418
+ packed_modules_mapping = {
419
+ "qkv": ["qkv"],
420
+ }
421
+
422
def __init__(
423
self,
424
config: PretrainedConfig,
vllm/model_executor/models/internvl.py
-9
@@ -1019,15 +1019,6 @@ class InternVLMultiModalProcessor(
1019
class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP,
1020
SupportsLoRA):
1021
1022
- packed_modules_mapping = {
1023
- "wqkv": ["wqkv"],
1024
- "qkv": ["qkv"],
1025
- "gate_up_proj": [
1026
- "w1",
1027
- "w3",
1028
- ],
1029
- }
1030
-
1022
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
1023
super().__init__()
1024
vllm/model_executor/models/qwen2_5_vl.py
-11
@@ -821,17 +821,6 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
821
dummy_inputs=Qwen2_5_VLDummyInputsBuilder)
822
class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
823
SupportsLoRA, SupportsPP):
824
- packed_modules_mapping = {
825
- "qkv_proj": [
826
- "q_proj",
827
- "k_proj",
828
- "v_proj",
829
- ],
830
- "gate_up_proj": [
831
- "gate_proj",
832
- "up_proj",
833
- ],
834
- }
824
825
# To ensure correct weight loading and mapping.
826
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
vllm/model_executor/models/qwen2_vl.py
-11
@@ -1069,17 +1069,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
1069
dummy_inputs=Qwen2VLDummyInputsBuilder)
1070
class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1071
SupportsLoRA, SupportsPP):
1072
- packed_modules_mapping = {
1073
- "qkv_proj": [
1074
- "q_proj",
1075
- "k_proj",
1076
- "v_proj",
1077
- ],
1078
- "gate_up_proj": [
1079
- "gate_proj",
1080
- "up_proj",
1081
- ],
1082
- }
1072
1073
# To ensure correct weight loading and mapping.
1074
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
vllm/model_executor/utils.py
+21
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""Utils for model executor."""
3
+import copy
4
from typing import Any, Optional
5
6
import torch
@@ -51,3 +52,23 @@ def _make_synced_weight_loader(original_weight_loader):
52
torch._sync(param)
53
54
return _synced_weight_loader
55
+
56
+
57
+def get_packed_modules_mapping(model: torch.nn.Module) -> dict[str, list[str]]:
58
+ parent_map = copy.deepcopy(getattr(model, "packed_modules_mapping", {}))
59
+
60
+ # don't infer mapping if the model has defined it explicitly.
61
+ if parent_map:
62
+ return parent_map
63
+
64
+ # We only check main components instead of whole model submodules
65
+ for child in model.children():
66
+ child_map = getattr(child, "packed_modules_mapping", {})
67
+ if any((k in parent_map and parent_map[k] != v)
68
+ for k, v in child_map.items()):
69
+ raise ValueError(
70
+ f"Can't update {type(model).__name__}'s packed_modules_mapping "
71
+ f"safely because of conflicts from {type(child).__name__}.")
72
+ else:
73
+ parent_map.update(child_map)
74
+ return parent_map
\ No newline at end of file