A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] Add base class for LoRA-supported models (#5018)
Cyrus Leung committed
Jun 27, 2024 at 16:03 UTC
96354d6a2967a63eb5c0e56a2da2ead512ff1067
20 files changed
+270
-75
docs/source/models/lora.rst
+3
@@ -4,6 +4,9 @@ Using LoRA adapters
4
===================
5
6
This document shows you how to use `LoRA adapters <https://arxiv.org/abs/2106.09685>`_ with vLLM on top of a base model.
7
+
8
+LoRA adapters can be used with any vLLM model that implements :class:`~vllm.model_executor.models.interfaces.SupportsLoRA`.
9
+
10
Adapters can be efficiently served on a per request basis with minimal overhead. First we download the adapter(s) and save
11
them locally with
12
vllm/lora/lora.py
+2
-1
@@ -2,6 +2,7 @@ from typing import List, Optional
2
from typing import Sequence as GenericSequence
3
4
import torch
5
+import torch.types
6
7
from vllm.utils import is_pin_memory_available
8
@@ -64,7 +65,7 @@ class LoRALayerWeights:
65
output_dim: int,
66
rank: int,
67
dtype: torch.dtype,
67
- device: torch.device,
68
+ device: torch.types.Device,
69
embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
70
pin_memory = str(device) == "cpu" and is_pin_memory_available()
71
lora_a = torch.zeros([input_dim, rank],
vllm/lora/models.py
+3
-3
@@ -18,6 +18,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
18
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
20
parse_fine_tuned_lora_name, replace_submodule)
21
+from vllm.model_executor.models.interfaces import SupportsLoRA
22
from vllm.utils import LRUCache, is_pin_memory_available
23
24
logger = init_logger(__name__)
@@ -363,7 +364,7 @@ class LoRAModelManager:
364
365
def __init__(
366
self,
366
- model: nn.Module,
367
+ model: SupportsLoRA,
368
max_num_seqs: int,
369
max_num_batched_tokens: int,
370
vocab_size: int,
@@ -411,7 +412,7 @@ class LoRAModelManager:
412
# embeddings_indices
413
self.indices_len: List[Optional[int]] = [None] * 4
414
414
- self.model: nn.Module = model
415
+ self.model = model
416
if hasattr(self.model, "supported_lora_modules"):
417
self.supported_lora_modules = copy.deepcopy(
418
self.model.supported_lora_modules)
@@ -428,7 +429,6 @@ class LoRAModelManager:
429
self._active_loras: Dict[int, None] = {}
430
self._last_mapping: Optional[LoRAMapping] = None
431
self._create_lora_modules()
431
- self.model.lora_manager = self
432
433
@property
434
def capacity(self) -> int:
vllm/model_executor/model_loader/loader.py
+13
-7
@@ -32,7 +32,8 @@ from vllm.model_executor.model_loader.weight_utils import (
32
filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
33
get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
34
pt_weights_iterator, safetensors_weights_iterator)
35
-from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
35
+from vllm.model_executor.models.interfaces import (supports_lora,
36
+ supports_vision)
37
from vllm.model_executor.utils import set_weight_attrs
38
from vllm.utils import is_tpu
39
@@ -64,12 +65,15 @@ def _get_quantization_config(
65
66
67
def _get_model_initialization_kwargs(
67
- model_class: Type[nn.Module], lora_config: Optional[LoRAConfig],
68
- vision_language_config: Optional[VisionLanguageConfig]
68
+ model_class: Type[nn.Module],
69
+ lora_config: Optional[LoRAConfig],
70
+ vlm_config: Optional[VisionLanguageConfig],
71
) -> Dict[str, Any]:
72
"""Get extra kwargs for model initialization."""
73
extra_kwargs: Dict[str, Any] = {}
72
- if hasattr(model_class, "supported_lora_modules"):
74
+
75
+ if supports_lora(model_class):
76
+ # lora_config=None is used to disable LoRA
77
extra_kwargs["lora_config"] = lora_config
78
elif lora_config:
79
raise ValueError(
@@ -77,13 +81,15 @@ def _get_model_initialization_kwargs(
81
"but LoRA is enabled. Support for this model may "
82
"be added in the future. If this is important to you, "
83
"please open an issue on github.")
80
- elif issubclass(model_class, VisionLanguageModelBase):
81
- if vision_language_config is None:
84
+
85
+ if supports_vision(model_class):
86
+ if vlm_config is None:
87
raise ValueError("Provide `image_input_type` and other vision "
88
"related configurations through LLM entrypoint "
89
"or engine arguments.")
90
86
- extra_kwargs["vision_language_config"] = vision_language_config
91
+ extra_kwargs["vlm_config"] = vlm_config
92
+
93
return extra_kwargs
94
95
vllm/model_executor/models/baichuan.py
+9
-2
@@ -45,6 +45,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.sequence import SamplerOutput
47
48
+from .interfaces import SupportsLoRA
49
+
50
51
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
52
closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
@@ -292,7 +294,9 @@ class BaiChuanModel(nn.Module):
294
return hidden_states
295
296
295
-class BaiChuanBaseForCausalLM(nn.Module):
297
+class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
298
+ supports_lora = True
299
+
300
packed_modules_mapping = {
301
"W_pack": ["W_pack"],
302
"gate_up_proj": [
@@ -312,14 +316,17 @@ class BaiChuanBaseForCausalLM(nn.Module):
316
317
def __init__(
318
self,
315
- config,
319
+ config: PretrainedConfig,
320
position_embedding: str,
321
cache_config: Optional[CacheConfig] = None,
322
quant_config: Optional[QuantizationConfig] = None,
323
lora_config: Optional[LoRAConfig] = None,
324
):
325
super().__init__()
326
+
327
self.config = config
328
+ self.lora_config = lora_config
329
+
330
self.quant_config = quant_config
331
self.model = BaiChuanModel(config, position_embedding, cache_config,
332
quant_config)
vllm/model_executor/models/chatglm.py
+9
-2
@@ -28,6 +28,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
28
from vllm.sequence import SamplerOutput
29
from vllm.transformers_utils.configs import ChatGLMConfig
30
31
+from .interfaces import SupportsLoRA
32
+
33
34
class GLMAttention(nn.Module):
35
@@ -322,7 +324,9 @@ class ChatGLMModel(nn.Module):
324
return hidden_states
325
326
325
-class ChatGLMForCausalLM(nn.Module):
327
+class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
328
+ supports_lora = True
329
+
330
packed_modules_mapping = {
331
"query_key_value": ["query_key_value"],
332
"dense_h_to_4h": ["dense_h_to_4h"]
@@ -345,7 +349,10 @@ class ChatGLMForCausalLM(nn.Module):
349
lora_config: Optional[LoRAConfig] = None,
350
):
351
super().__init__()
348
- self.config: ChatGLMConfig = config
352
+
353
+ self.config = config
354
+ self.lora_config = lora_config
355
+
356
self.quant_config = quant_config
357
self.max_position_embeddings = getattr(config, "max_sequence_length",
358
8192)
vllm/model_executor/models/decilm.py
+2
-2
@@ -26,7 +26,7 @@
26
from typing import Iterable, Optional, Tuple
27
28
import torch
29
-from transformers import PretrainedConfig
29
+from transformers import LlamaConfig
30
31
from vllm.config import CacheConfig, LoRAConfig
32
from vllm.model_executor.layers.quantization.base_config import (
@@ -55,7 +55,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
55
56
def __init__(
57
self,
58
- config: Optional[PretrainedConfig] = None,
58
+ config: LlamaConfig,
59
cache_config: Optional[CacheConfig] = None,
60
quant_config: Optional[QuantizationConfig] = None,
61
lora_config: Optional[LoRAConfig] = None,
vllm/model_executor/models/gemma.py
+8
-2
@@ -41,6 +41,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import SamplerOutput
43
44
+from .interfaces import SupportsLoRA
45
+
46
logger = init_logger(__name__)
47
48
@@ -288,7 +290,9 @@ class GemmaModel(nn.Module):
290
return hidden_states
291
292
291
-class GemmaForCausalLM(nn.Module):
293
+class GemmaForCausalLM(nn.Module, SupportsLoRA):
294
+ supports_lora = True
295
+
296
packed_modules_mapping = {
297
"qkv_proj": [
298
"q_proj",
@@ -319,9 +323,11 @@ class GemmaForCausalLM(nn.Module):
323
quant_config: Optional[QuantizationConfig] = None,
324
lora_config: Optional[LoRAConfig] = None,
325
) -> None:
322
- del lora_config # Unused.
326
super().__init__()
327
+
328
self.config = config
329
+ self.lora_config = lora_config
330
+
331
self.quant_config = quant_config
332
self.model = GemmaModel(config, cache_config, quant_config)
333
self.logits_processor = LogitsProcessor(config.vocab_size)
vllm/model_executor/models/gpt_bigcode.py
+8
-1
@@ -41,6 +41,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
from vllm.sequence import SamplerOutput
43
44
+from .interfaces import SupportsLoRA
45
+
46
47
class GPTBigCodeAttention(nn.Module):
48
@@ -230,7 +232,9 @@ class GPTBigCodeModel(nn.Module):
232
return hidden_states
233
234
233
-class GPTBigCodeForCausalLM(nn.Module):
235
+class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
236
+ supports_lora = True
237
+
238
packed_modules_mapping = {"c_attn": ["c_attn"]}
239
240
supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]
@@ -250,7 +254,10 @@ class GPTBigCodeForCausalLM(nn.Module):
254
lora_config: Optional[LoRAConfig] = None,
255
):
256
super().__init__()
257
+
258
self.config = config
259
+ self.lora_config = lora_config
260
+
261
self.quant_config = quant_config
262
self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
263
lora_config)
vllm/model_executor/models/interfaces.py
new
+130
@@ -0,0 +1,130 @@
1
+from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
2
+ Union, overload, runtime_checkable)
3
+
4
+from typing_extensions import TypeGuard
5
+
6
+from vllm.config import LoRAConfig, VisionLanguageConfig
7
+from vllm.logger import init_logger
8
+
9
+logger = init_logger(__name__)
10
+
11
+
12
+@runtime_checkable
13
+class SupportsVision(Protocol):
14
+ """The interface required for all vision language models (VLMs)."""
15
+
16
+ supports_vision: ClassVar[Literal[True]]
17
+
18
+ def __init__(self, *, vlm_config: VisionLanguageConfig) -> None:
19
+ ...
20
+
21
+
22
+# We can't use runtime_checkable with ClassVar for issubclass checks
23
+# so we need to treat the class as an instance and use isinstance instead
24
+@runtime_checkable
25
+class _SupportsVisionType(Protocol):
26
+ supports_vision: Literal[True]
27
+
28
+ def __call__(self, *, vlm_config: VisionLanguageConfig) -> None:
29
+ ...
30
+
31
+
32
+@overload
33
+def supports_vision(model: Type[object]) -> TypeGuard[Type[SupportsVision]]:
34
+ ...
35
+
36
+
37
+@overload
38
+def supports_vision(model: object) -> TypeGuard[SupportsVision]:
39
+ ...
40
+
41
+
42
+def supports_vision(
43
+ model: Union[Type[object], object],
44
+) -> Union[TypeGuard[Type[SupportsVision]], TypeGuard[SupportsVision]]:
45
+ if isinstance(model, type):
46
+ return isinstance(model, _SupportsVisionType)
47
+
48
+ return isinstance(model, SupportsVision)
49
+
50
+
51
+@runtime_checkable
52
+class SupportsLoRA(Protocol):
53
+ """The interface required for all models that support LoRA."""
54
+
55
+ supports_lora: ClassVar[Literal[True]]
56
+
57
+ packed_modules_mapping: ClassVar[Dict[str, List[str]]]
58
+ supported_lora_modules: ClassVar[List[str]]
59
+ embedding_modules: ClassVar[Dict[str, str]]
60
+ embedding_padding_modules: ClassVar[List[str]]
61
+
62
+ # lora_config is None when LoRA is not enabled
63
+ def __init__(self, *, lora_config: Optional[LoRAConfig] = None) -> None:
64
+ ...
65
+
66
+
67
+# We can't use runtime_checkable with ClassVar for issubclass checks
68
+# so we need to treat the class as an instance and use isinstance instead
69
+@runtime_checkable
70
+class _SupportsLoRAType(Protocol):
71
+ supports_lora: Literal[True]
72
+
73
+ packed_modules_mapping: Dict[str, List[str]]
74
+ supported_lora_modules: List[str]
75
+ embedding_modules: Dict[str, str]
76
+ embedding_padding_modules: List[str]
77
+
78
+ def __call__(self, *, lora_config: Optional[LoRAConfig] = None) -> None:
79
+ ...
80
+
81
+
82
+@overload
83
+def supports_lora(model: Type[object]) -> TypeGuard[Type[SupportsLoRA]]:
84
+ ...
85
+
86
+
87
+@overload
88
+def supports_lora(model: object) -> TypeGuard[SupportsLoRA]:
89
+ ...
90
+
91
+
92
+def supports_lora(
93
+ model: Union[Type[object], object],
94
+) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
95
+ result = _supports_lora(model)
96
+
97
+ if not result:
98
+ lora_attrs = (
99
+ "packed_modules_mapping",
100
+ "supported_lora_modules",
101
+ "embedding_modules",
102
+ "embedding_padding_modules",
103
+ )
104
+ missing_attrs = tuple(attr for attr in lora_attrs
105
+ if not hasattr(model, attr))
106
+
107
+ if getattr(model, "supports_lora", False):
108
+ if missing_attrs:
109
+ logger.warning(
110
+ "The model (%s) sets `supports_lora=True`, "
111
+ "but is missing LoRA-specific attributes: %s",
112
+ model,
113
+ missing_attrs,
114
+ )
115
+ else:
116
+ if not missing_attrs:
117
+ logger.warning(
118
+ "The model (%s) contains all LoRA-specific attributes, "
119
+ "but does not set `supports_lora=True`.", model)
120
+
121
+ return result
122
+
123
+
124
+def _supports_lora(
125
+ model: Union[Type[object], object],
126
+) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
127
+ if isinstance(model, type):
128
+ return isinstance(model, _SupportsLoRAType)
129
+
130
+ return isinstance(model, SupportsLoRA)
vllm/model_executor/models/llama.py
+8
-1
@@ -49,6 +49,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.sequence import SamplerOutput
50
from vllm.utils import is_hip, print_warning_once
51
52
+from .interfaces import SupportsLoRA
53
+
54
55
class LlamaMLP(nn.Module):
56
@@ -296,7 +298,9 @@ class LlamaModel(nn.Module):
298
return hidden_states
299
300
299
-class LlamaForCausalLM(nn.Module):
301
+class LlamaForCausalLM(nn.Module, SupportsLoRA):
302
+ supports_lora = True
303
+
304
packed_modules_mapping = {
305
"qkv_proj": [
306
"q_proj",
@@ -336,7 +340,10 @@ class LlamaForCausalLM(nn.Module):
340
lora_config: Optional[LoRAConfig] = None,
341
) -> None:
342
super().__init__()
343
+
344
self.config = config
345
+ self.lora_config = lora_config
346
+
347
self.model = LlamaModel(config,
348
cache_config,
349
quant_config,
vllm/model_executor/models/llava.py
+12
-10
@@ -20,7 +20,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.multimodal.image import get_dummy_image_data
21
from vllm.sequence import SamplerOutput
22
23
-from .vlm_base import VisionLanguageModelBase
23
+from .interfaces import SupportsVision
24
25
_KEYS_TO_MODIFY_MAPPING = {
26
"language_model.lm_head": "lm_head",
@@ -86,18 +86,21 @@ LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
86
@MULTIMODAL_REGISTRY.register_image_feature_input()
87
@MULTIMODAL_REGISTRY.register_image_pixel_input()
88
@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
89
-class LlavaForConditionalGeneration(VisionLanguageModelBase):
89
+class LlavaForConditionalGeneration(nn.Module, SupportsVision):
90
+
91
+ supports_vision = True
92
93
def __init__(self,
94
config: LlavaConfig,
93
- vision_language_config: VisionLanguageConfig,
95
+ vlm_config: VisionLanguageConfig,
96
cache_config: Optional[CacheConfig] = None,
97
quant_config: Optional[QuantizationConfig] = None) -> None:
96
- super().__init__(vision_language_config)
98
+ super().__init__()
99
100
self.config = config
101
+ self.vlm_config = vlm_config
102
100
- if self.vision_language_config.image_input_type == (
103
+ if self.vlm_config.image_input_type == (
104
VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
105
self.vision_tower = CLIPVisionModel(config.vision_config)
106
else:
@@ -122,11 +125,10 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
125
self.sampler = Sampler()
126
127
def _validate_image_data(self, data: torch.Tensor) -> torch.Tensor:
125
- if list(data.shape[1:]) != list(
126
- self.vision_language_config.image_input_shape[1:]):
128
+ if list(data.shape[1:]) != list(self.vlm_config.image_input_shape[1:]):
129
raise ValueError(
130
f"The expected image tensor shape is batch dimension plus "
129
- f"{self.vision_language_config.image_input_shape[1:]}. "
131
+ f"{self.vlm_config.image_input_shape[1:]}. "
132
f"You supplied {data.shape}. "
133
f"If you are using vLLM's entrypoint, make sure your "
134
f"supplied image input is consistent with "
@@ -139,7 +141,7 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
141
pixel_values = kwargs.pop("pixel_values", None)
142
image_features = kwargs.pop("image_features", None)
143
142
- expected_input_type = self.vision_language_config.image_input_type
144
+ expected_input_type = self.vlm_config.image_input_type
145
ImageInputType = VisionLanguageConfig.ImageInputType
146
147
if expected_input_type == ImageInputType.PIXEL_VALUES:
@@ -273,7 +275,7 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
275
276
inputs_embeds = merge_vision_embeddings(
277
input_ids, inputs_embeds, vision_embeddings,
276
- self.vision_language_config.image_token_id)
278
+ self.vlm_config.image_token_id)
279
280
input_ids = None
281
else:
vllm/model_executor/models/llava_next.py
+11
-9
@@ -25,8 +25,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
25
from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
26
from vllm.sequence import SamplerOutput, SequenceData
27
28
+from .interfaces import SupportsVision
29
from .llava import LlavaMultiModalProjector, merge_vision_embeddings
29
-from .vlm_base import VisionLanguageModelBase
30
31
logger = init_logger(__name__)
32
@@ -106,19 +106,21 @@ def _image_pixel_processor(
106
107
@MULTIMODAL_REGISTRY.register_image_pixel_input(_image_pixel_processor)
108
@MULTIMODAL_REGISTRY.register_dummy_data(_get_dummy_image_data)
109
-class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
109
+class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
110
+
111
+ supports_vision = True
112
113
def __init__(self,
114
config: LlavaNextConfig,
113
- vision_language_config: VisionLanguageConfig,
115
+ vlm_config: VisionLanguageConfig,
116
cache_config: Optional[CacheConfig] = None,
117
quant_config: Optional[QuantizationConfig] = None) -> None:
116
- super().__init__(vision_language_config)
118
+ super().__init__()
119
118
- # Update the type annotation from that of its superclass
120
self.config = config
121
+ self.vlm_config = vlm_config
122
121
- if self.vision_language_config.image_input_type == (
123
+ if self.vlm_config.image_input_type == (
124
VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
125
self.vision_tower = CLIPVisionModel(config=config.vision_config)
126
else:
@@ -146,7 +148,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
148
torch.empty(config.text_config.hidden_size))
149
150
def _validate_image_pixels(self, data: torch.Tensor) -> torch.Tensor:
149
- _, num_channels, _, _ = self.vision_language_config.image_input_shape
151
+ _, num_channels, _, _ = self.vlm_config.image_input_shape
152
153
# Note that this is different from that of vLLM vision_language_config
154
# since the image is resized by the HuggingFace preprocessor
@@ -177,7 +179,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
179
image_sizes = kwargs.pop("image_sizes", None)
180
image_features = kwargs.pop("image_features", None)
181
180
- expected_input_type = self.vision_language_config.image_input_type
182
+ expected_input_type = self.vlm_config.image_input_type
183
ImageInputType = VisionLanguageConfig.ImageInputType
184
185
if expected_input_type == ImageInputType.PIXEL_VALUES:
@@ -386,7 +388,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
388
389
inputs_embeds = merge_vision_embeddings(
390
input_ids, inputs_embeds, vision_embeddings,
389
- self.vision_language_config.image_token_id)
391
+ self.vlm_config.image_token_id)
392
393
input_ids = None
394
else:
vllm/model_executor/models/minicpm.py
+10
-2
@@ -26,6 +26,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
26
27
import torch
28
from torch import nn
29
+from transformers import PretrainedConfig
30
31
from vllm.attention import Attention, AttentionMetadata
32
from vllm.config import CacheConfig, LoRAConfig
@@ -51,6 +52,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
52
from vllm.model_executor.utils import set_weight_attrs
53
from vllm.sequence import SamplerOutput
54
55
+from .interfaces import SupportsLoRA
56
+
57
58
class MiniCPMMoE(nn.Module):
59
"""A tensor-parallel MoE implementation that shards each expert
@@ -388,7 +391,9 @@ class MiniCPMModel(nn.Module):
391
return hidden_states
392
393
391
-class MiniCPMForCausalLM(nn.Module):
394
+class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
395
+ supports_lora = True
396
+
397
packed_modules_mapping = {
398
"qkv_proj": [
399
"q_proj",
@@ -418,13 +423,16 @@ class MiniCPMForCausalLM(nn.Module):
423
424
def __init__(
425
self,
421
- config,
426
+ config: PretrainedConfig,
427
cache_config: Optional[CacheConfig] = None,
428
quant_config: Optional[QuantizationConfig] = None,
429
lora_config: Optional[LoRAConfig] = None,
430
) -> None:
431
super().__init__()
432
+
433
self.config = config
434
+ self.lora_config = lora_config
435
+
436
self.num_experts = getattr(self.config, "num_experts", 0)
437
self.quant_config = quant_config
438
self.model = MiniCPMModel(config,
vllm/model_executor/models/mixtral.py
+8
-1
@@ -54,6 +54,8 @@ from vllm.model_executor.utils import set_weight_attrs
54
from vllm.sequence import SamplerOutput
55
from vllm.utils import print_warning_once
56
57
+from .interfaces import SupportsLoRA
58
+
59
60
class MixtralMoE(nn.Module):
61
"""A tensor-parallel MoE implementation for Mixtral that shards each expert
@@ -472,7 +474,9 @@ class MixtralModel(nn.Module):
474
return hidden_states
475
476
475
-class MixtralForCausalLM(nn.Module):
477
+class MixtralForCausalLM(nn.Module, SupportsLoRA):
478
+ supports_lora = True
479
+
480
fall_back_to_pt_during_load = False
481
482
packed_modules_mapping = {
@@ -504,7 +508,10 @@ class MixtralForCausalLM(nn.Module):
508
lora_config: Optional[LoRAConfig] = None,
509
) -> None:
510
super().__init__()
511
+
512
self.config = config
513
+ self.lora_config = lora_config
514
+
515
self.model = MixtralModel(config,
516
cache_config,
517
quant_config,
vllm/model_executor/models/phi.py
+14
-8
@@ -39,7 +39,7 @@ from typing import Iterable, List, Optional, Tuple
39
40
import torch
41
from torch import nn
42
-from transformers import PretrainedConfig
42
+from transformers import PhiConfig
43
44
from vllm.attention import Attention, AttentionMetadata
45
from vllm.config import CacheConfig, LoRAConfig
@@ -59,11 +59,13 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59
from vllm.model_executor.sampling_metadata import SamplingMetadata
60
from vllm.sequence import SamplerOutput
61
62
+from .interfaces import SupportsLoRA
63
+
64
65
class PhiAttention(nn.Module):
66
67
def __init__(self,
66
- config: PretrainedConfig,
68
+ config: PhiConfig,
69
cache_config: Optional[CacheConfig] = None,
70
quant_config: Optional[QuantizationConfig] = None):
71
super().__init__()
@@ -131,7 +133,7 @@ class PhiAttention(nn.Module):
133
class PhiMLP(nn.Module):
134
135
def __init__(self,
134
- config: PretrainedConfig,
136
+ config: PhiConfig,
137
quant_config: Optional[QuantizationConfig] = None):
138
super().__init__()
139
@@ -160,7 +162,7 @@ class PhiMLP(nn.Module):
162
class PhiLayer(nn.Module):
163
164
def __init__(self,
163
- config: PretrainedConfig,
165
+ config: PhiConfig,
166
cache_config: Optional[CacheConfig] = None,
167
quant_config: Optional[QuantizationConfig] = None):
168
super().__init__()
@@ -192,7 +194,7 @@ class PhiLayer(nn.Module):
194
class PhiModel(nn.Module):
195
196
def __init__(self,
195
- config: PretrainedConfig,
197
+ config: PhiConfig,
198
cache_config: Optional[CacheConfig] = None,
199
quant_config: Optional[QuantizationConfig] = None):
200
super().__init__()
@@ -229,7 +231,9 @@ class PhiModel(nn.Module):
231
return hidden_states
232
233
232
-class PhiForCausalLM(nn.Module):
234
+class PhiForCausalLM(nn.Module, SupportsLoRA):
235
+ supports_lora = True
236
+
237
packed_modules_mapping = {
238
"qkv_proj": [
239
"q_proj",
@@ -250,14 +254,16 @@ class PhiForCausalLM(nn.Module):
254
255
def __init__(
256
self,
253
- config: PretrainedConfig,
257
+ config: PhiConfig,
258
cache_config: Optional[CacheConfig] = None,
259
quant_config: Optional[QuantizationConfig] = None,
260
lora_config: Optional[LoRAConfig] = None,
261
):
258
- del lora_config # Unused.
262
super().__init__()
263
+
264
self.config = config
265
+ self.lora_config = lora_config
266
+
267
self.quant_config = quant_config
268
269
self.model = PhiModel(config, cache_config, quant_config)
vllm/model_executor/models/qwen2.py
+8
-2
@@ -48,6 +48,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.sequence import SamplerOutput
49
from vllm.utils import print_warning_once
50
51
+from .interfaces import SupportsLoRA
52
+
53
54
class Qwen2MLP(nn.Module):
55
@@ -263,7 +265,9 @@ class Qwen2Model(nn.Module):
265
return hidden_states
266
267
266
-class Qwen2ForCausalLM(nn.Module):
268
+class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
269
+ supports_lora = True
270
+
271
packed_modules_mapping = {
272
"qkv_proj": [
273
"q_proj",
@@ -293,7 +297,6 @@ class Qwen2ForCausalLM(nn.Module):
297
quant_config: Optional[QuantizationConfig] = None,
298
lora_config: Optional[LoRAConfig] = None,
299
) -> None:
296
- del lora_config
300
# TODO (@robertgshaw2): see if this can be moved out
301
if (cache_config.sliding_window is not None
302
and hasattr(config, "max_window_layers")):
@@ -307,7 +310,10 @@ class Qwen2ForCausalLM(nn.Module):
310
))
311
312
super().__init__()
313
+
314
self.config = config
315
+ self.lora_config = lora_config
316
+
317
self.quant_config = quant_config
318
self.model = Qwen2Model(config, cache_config, quant_config)
319
vllm/model_executor/models/vlm_base.py
deleted
-12
@@ -1,12 +0,0 @@
1
-from torch import nn
2
-
3
-from vllm.config import VisionLanguageConfig
4
-
5
-
6
-class VisionLanguageModelBase(nn.Module):
7
- """Base class for all vision language models (VLMs)."""
8
-
9
- def __init__(self, vision_language_config: VisionLanguageConfig) -> None:
10
- super().__init__()
11
-
12
- self.vision_language_config = vision_language_config
vllm/model_executor/models/xverse.py
+9
-2
@@ -45,6 +45,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.sequence import SamplerOutput
47
48
+from .interfaces import SupportsLoRA
49
+
50
51
class XverseMLP(nn.Module):
52
@@ -266,7 +268,9 @@ class XverseModel(nn.Module):
268
return hidden_states
269
270
269
-class XverseForCausalLM(nn.Module):
271
+class XverseForCausalLM(nn.Module, SupportsLoRA):
272
+ supports_lora = True
273
+
274
packed_modules_mapping = {
275
"qkv_proj": [
276
"q_proj",
@@ -299,10 +303,13 @@ class XverseForCausalLM(nn.Module):
303
config: PretrainedConfig,
304
cache_config: Optional[CacheConfig] = None,
305
quant_config: Optional[QuantizationConfig] = None,
302
- lora_config=None,
306
+ lora_config: Optional[LoRAConfig] = None,
307
) -> None:
308
super().__init__()
309
+
310
self.config = config
311
+ self.lora_config = lora_config
312
+
313
self.quant_config = quant_config
314
self.model = XverseModel(config, cache_config, quant_config)
315
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
vllm/worker/model_runner.py
+3
-8
@@ -22,6 +22,7 @@ from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
22
from vllm.model_executor import SamplingMetadata
23
from vllm.model_executor.model_loader import get_model
24
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
25
+from vllm.model_executor.models.interfaces import supports_lora
26
from vllm.multimodal import MULTIMODAL_REGISTRY
27
from vllm.sampling_params import SamplingParams
28
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
@@ -225,14 +226,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
226
self.model_memory_usage / float(2**30))
227
228
if self.lora_config:
228
- assert hasattr(self.model, "supported_lora_modules"
229
- ) and self.model.supported_lora_modules, (
230
- "Model does not support LoRA")
231
- assert hasattr(
232
- self.model,
233
- "embedding_modules"), "Model does not have embedding_modules"
234
- assert hasattr(self.model, "embedding_padding_modules"
235
- ), "Model does not have embedding_padding_modules"
229
+ assert supports_lora(self.model), "Model does not support LoRA"
230
+
231
self.lora_manager = LRUCacheWorkerLoRAManager(
232
self.scheduler_config.max_num_seqs,
233
self.scheduler_config.max_num_batched_tokens,