@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96354d6a29
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] Add base class for LoRA-supported models (#5018)

Cyrus Leung committed Jun 27, 2024 at 16:03 UTC 96354d6a2967a63eb5c0e56a2da2ead512ff1067
20 files changed +270 -75
docs/source/models/lora.rst
+3
@@ -4,6 +4,9 @@ Using LoRA adapters
4 ===================
5
6 This document shows you how to use `LoRA adapters <https://arxiv.org/abs/2106.09685>`_ with vLLM on top of a base model.
7 +
8 +LoRA adapters can be used with any vLLM model that implements :class:`~vllm.model_executor.models.interfaces.SupportsLoRA`.
9 +
10 Adapters can be efficiently served on a per request basis with minimal overhead. First we download the adapter(s) and save
11 them locally with
12
vllm/lora/lora.py
+2 -1
@@ -2,6 +2,7 @@ from typing import List, Optional
2 from typing import Sequence as GenericSequence
3
4 import torch
5 +import torch.types
6
7 from vllm.utils import is_pin_memory_available
8
@@ -64,7 +65,7 @@ class LoRALayerWeights:
65 output_dim: int,
66 rank: int,
67 dtype: torch.dtype,
67 - device: torch.device,
68 + device: torch.types.Device,
69 embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
70 pin_memory = str(device) == "cpu" and is_pin_memory_available()
71 lora_a = torch.zeros([input_dim, rank],
vllm/lora/models.py
+3 -3
@@ -18,6 +18,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
18 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19 from vllm.lora.utils import (from_layer, from_layer_logits_processor,
20 parse_fine_tuned_lora_name, replace_submodule)
21 +from vllm.model_executor.models.interfaces import SupportsLoRA
22 from vllm.utils import LRUCache, is_pin_memory_available
23
24 logger = init_logger(__name__)
@@ -363,7 +364,7 @@ class LoRAModelManager:
364
365 def __init__(
366 self,
366 - model: nn.Module,
367 + model: SupportsLoRA,
368 max_num_seqs: int,
369 max_num_batched_tokens: int,
370 vocab_size: int,
@@ -411,7 +412,7 @@ class LoRAModelManager:
412 # embeddings_indices
413 self.indices_len: List[Optional[int]] = [None] * 4
414
414 - self.model: nn.Module = model
415 + self.model = model
416 if hasattr(self.model, "supported_lora_modules"):
417 self.supported_lora_modules = copy.deepcopy(
418 self.model.supported_lora_modules)
@@ -428,7 +429,6 @@ class LoRAModelManager:
429 self._active_loras: Dict[int, None] = {}
430 self._last_mapping: Optional[LoRAMapping] = None
431 self._create_lora_modules()
431 - self.model.lora_manager = self
432
433 @property
434 def capacity(self) -> int:
vllm/model_executor/model_loader/loader.py
+13 -7
@@ -32,7 +32,8 @@ from vllm.model_executor.model_loader.weight_utils import (
32 filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
33 get_quant_config, initialize_dummy_weights, np_cache_weights_iterator,
34 pt_weights_iterator, safetensors_weights_iterator)
35 -from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
35 +from vllm.model_executor.models.interfaces import (supports_lora,
36 + supports_vision)
37 from vllm.model_executor.utils import set_weight_attrs
38 from vllm.utils import is_tpu
39
@@ -64,12 +65,15 @@ def _get_quantization_config(
65
66
67 def _get_model_initialization_kwargs(
67 - model_class: Type[nn.Module], lora_config: Optional[LoRAConfig],
68 - vision_language_config: Optional[VisionLanguageConfig]
68 + model_class: Type[nn.Module],
69 + lora_config: Optional[LoRAConfig],
70 + vlm_config: Optional[VisionLanguageConfig],
71 ) -> Dict[str, Any]:
72 """Get extra kwargs for model initialization."""
73 extra_kwargs: Dict[str, Any] = {}
72 - if hasattr(model_class, "supported_lora_modules"):
74 +
75 + if supports_lora(model_class):
76 + # lora_config=None is used to disable LoRA
77 extra_kwargs["lora_config"] = lora_config
78 elif lora_config:
79 raise ValueError(
@@ -77,13 +81,15 @@ def _get_model_initialization_kwargs(
81 "but LoRA is enabled. Support for this model may "
82 "be added in the future. If this is important to you, "
83 "please open an issue on github.")
80 - elif issubclass(model_class, VisionLanguageModelBase):
81 - if vision_language_config is None:
84 +
85 + if supports_vision(model_class):
86 + if vlm_config is None:
87 raise ValueError("Provide `image_input_type` and other vision "
88 "related configurations through LLM entrypoint "
89 "or engine arguments.")
90
86 - extra_kwargs["vision_language_config"] = vision_language_config
91 + extra_kwargs["vlm_config"] = vlm_config
92 +
93 return extra_kwargs
94
95
vllm/model_executor/models/baichuan.py
+9 -2
@@ -45,6 +45,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.sequence import SamplerOutput
47
48 +from .interfaces import SupportsLoRA
49 +
50
51 def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
52 closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
@@ -292,7 +294,9 @@ class BaiChuanModel(nn.Module):
294 return hidden_states
295
296
295 -class BaiChuanBaseForCausalLM(nn.Module):
297 +class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
298 + supports_lora = True
299 +
300 packed_modules_mapping = {
301 "W_pack": ["W_pack"],
302 "gate_up_proj": [
@@ -312,14 +316,17 @@ class BaiChuanBaseForCausalLM(nn.Module):
316
317 def __init__(
318 self,
315 - config,
319 + config: PretrainedConfig,
320 position_embedding: str,
321 cache_config: Optional[CacheConfig] = None,
322 quant_config: Optional[QuantizationConfig] = None,
323 lora_config: Optional[LoRAConfig] = None,
324 ):
325 super().__init__()
326 +
327 self.config = config
328 + self.lora_config = lora_config
329 +
330 self.quant_config = quant_config
331 self.model = BaiChuanModel(config, position_embedding, cache_config,
332 quant_config)
vllm/model_executor/models/chatglm.py
+9 -2
@@ -28,6 +28,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.sequence import SamplerOutput
29 from vllm.transformers_utils.configs import ChatGLMConfig
30
31 +from .interfaces import SupportsLoRA
32 +
33
34 class GLMAttention(nn.Module):
35
@@ -322,7 +324,9 @@ class ChatGLMModel(nn.Module):
324 return hidden_states
325
326
325 -class ChatGLMForCausalLM(nn.Module):
327 +class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
328 + supports_lora = True
329 +
330 packed_modules_mapping = {
331 "query_key_value": ["query_key_value"],
332 "dense_h_to_4h": ["dense_h_to_4h"]
@@ -345,7 +349,10 @@ class ChatGLMForCausalLM(nn.Module):
349 lora_config: Optional[LoRAConfig] = None,
350 ):
351 super().__init__()
348 - self.config: ChatGLMConfig = config
352 +
353 + self.config = config
354 + self.lora_config = lora_config
355 +
356 self.quant_config = quant_config
357 self.max_position_embeddings = getattr(config, "max_sequence_length",
358 8192)
vllm/model_executor/models/decilm.py
+2 -2
@@ -26,7 +26,7 @@
26 from typing import Iterable, Optional, Tuple
27
28 import torch
29 -from transformers import PretrainedConfig
29 +from transformers import LlamaConfig
30
31 from vllm.config import CacheConfig, LoRAConfig
32 from vllm.model_executor.layers.quantization.base_config import (
@@ -55,7 +55,7 @@ class DeciLMForCausalLM(LlamaForCausalLM):
55
56 def __init__(
57 self,
58 - config: Optional[PretrainedConfig] = None,
58 + config: LlamaConfig,
59 cache_config: Optional[CacheConfig] = None,
60 quant_config: Optional[QuantizationConfig] = None,
61 lora_config: Optional[LoRAConfig] = None,
vllm/model_executor/models/gemma.py
+8 -2
@@ -41,6 +41,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import SamplerOutput
43
44 +from .interfaces import SupportsLoRA
45 +
46 logger = init_logger(__name__)
47
48
@@ -288,7 +290,9 @@ class GemmaModel(nn.Module):
290 return hidden_states
291
292
291 -class GemmaForCausalLM(nn.Module):
293 +class GemmaForCausalLM(nn.Module, SupportsLoRA):
294 + supports_lora = True
295 +
296 packed_modules_mapping = {
297 "qkv_proj": [
298 "q_proj",
@@ -319,9 +323,11 @@ class GemmaForCausalLM(nn.Module):
323 quant_config: Optional[QuantizationConfig] = None,
324 lora_config: Optional[LoRAConfig] = None,
325 ) -> None:
322 - del lora_config # Unused.
326 super().__init__()
327 +
328 self.config = config
329 + self.lora_config = lora_config
330 +
331 self.quant_config = quant_config
332 self.model = GemmaModel(config, cache_config, quant_config)
333 self.logits_processor = LogitsProcessor(config.vocab_size)
vllm/model_executor/models/gpt_bigcode.py
+8 -1
@@ -41,6 +41,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 from vllm.sequence import SamplerOutput
43
44 +from .interfaces import SupportsLoRA
45 +
46
47 class GPTBigCodeAttention(nn.Module):
48
@@ -230,7 +232,9 @@ class GPTBigCodeModel(nn.Module):
232 return hidden_states
233
234
233 -class GPTBigCodeForCausalLM(nn.Module):
235 +class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
236 + supports_lora = True
237 +
238 packed_modules_mapping = {"c_attn": ["c_attn"]}
239
240 supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]
@@ -250,7 +254,10 @@ class GPTBigCodeForCausalLM(nn.Module):
254 lora_config: Optional[LoRAConfig] = None,
255 ):
256 super().__init__()
257 +
258 self.config = config
259 + self.lora_config = lora_config
260 +
261 self.quant_config = quant_config
262 self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
263 lora_config)
vllm/model_executor/models/interfaces.py new
+130
@@ -0,0 +1,130 @@
1 +from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
2 + Union, overload, runtime_checkable)
3 +
4 +from typing_extensions import TypeGuard
5 +
6 +from vllm.config import LoRAConfig, VisionLanguageConfig
7 +from vllm.logger import init_logger
8 +
9 +logger = init_logger(__name__)
10 +
11 +
12 +@runtime_checkable
13 +class SupportsVision(Protocol):
14 + """The interface required for all vision language models (VLMs)."""
15 +
16 + supports_vision: ClassVar[Literal[True]]
17 +
18 + def __init__(self, *, vlm_config: VisionLanguageConfig) -> None:
19 + ...
20 +
21 +
22 +# We can't use runtime_checkable with ClassVar for issubclass checks
23 +# so we need to treat the class as an instance and use isinstance instead
24 +@runtime_checkable
25 +class _SupportsVisionType(Protocol):
26 + supports_vision: Literal[True]
27 +
28 + def __call__(self, *, vlm_config: VisionLanguageConfig) -> None:
29 + ...
30 +
31 +
32 +@overload
33 +def supports_vision(model: Type[object]) -> TypeGuard[Type[SupportsVision]]:
34 + ...
35 +
36 +
37 +@overload
38 +def supports_vision(model: object) -> TypeGuard[SupportsVision]:
39 + ...
40 +
41 +
42 +def supports_vision(
43 + model: Union[Type[object], object],
44 +) -> Union[TypeGuard[Type[SupportsVision]], TypeGuard[SupportsVision]]:
45 + if isinstance(model, type):
46 + return isinstance(model, _SupportsVisionType)
47 +
48 + return isinstance(model, SupportsVision)
49 +
50 +
51 +@runtime_checkable
52 +class SupportsLoRA(Protocol):
53 + """The interface required for all models that support LoRA."""
54 +
55 + supports_lora: ClassVar[Literal[True]]
56 +
57 + packed_modules_mapping: ClassVar[Dict[str, List[str]]]
58 + supported_lora_modules: ClassVar[List[str]]
59 + embedding_modules: ClassVar[Dict[str, str]]
60 + embedding_padding_modules: ClassVar[List[str]]
61 +
62 + # lora_config is None when LoRA is not enabled
63 + def __init__(self, *, lora_config: Optional[LoRAConfig] = None) -> None:
64 + ...
65 +
66 +
67 +# We can't use runtime_checkable with ClassVar for issubclass checks
68 +# so we need to treat the class as an instance and use isinstance instead
69 +@runtime_checkable
70 +class _SupportsLoRAType(Protocol):
71 + supports_lora: Literal[True]
72 +
73 + packed_modules_mapping: Dict[str, List[str]]
74 + supported_lora_modules: List[str]
75 + embedding_modules: Dict[str, str]
76 + embedding_padding_modules: List[str]
77 +
78 + def __call__(self, *, lora_config: Optional[LoRAConfig] = None) -> None:
79 + ...
80 +
81 +
82 +@overload
83 +def supports_lora(model: Type[object]) -> TypeGuard[Type[SupportsLoRA]]:
84 + ...
85 +
86 +
87 +@overload
88 +def supports_lora(model: object) -> TypeGuard[SupportsLoRA]:
89 + ...
90 +
91 +
92 +def supports_lora(
93 + model: Union[Type[object], object],
94 +) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
95 + result = _supports_lora(model)
96 +
97 + if not result:
98 + lora_attrs = (
99 + "packed_modules_mapping",
100 + "supported_lora_modules",
101 + "embedding_modules",
102 + "embedding_padding_modules",
103 + )
104 + missing_attrs = tuple(attr for attr in lora_attrs
105 + if not hasattr(model, attr))
106 +
107 + if getattr(model, "supports_lora", False):
108 + if missing_attrs:
109 + logger.warning(
110 + "The model (%s) sets `supports_lora=True`, "
111 + "but is missing LoRA-specific attributes: %s",
112 + model,
113 + missing_attrs,
114 + )
115 + else:
116 + if not missing_attrs:
117 + logger.warning(
118 + "The model (%s) contains all LoRA-specific attributes, "
119 + "but does not set `supports_lora=True`.", model)
120 +
121 + return result
122 +
123 +
124 +def _supports_lora(
125 + model: Union[Type[object], object],
126 +) -> Union[TypeGuard[Type[SupportsLoRA]], TypeGuard[SupportsLoRA]]:
127 + if isinstance(model, type):
128 + return isinstance(model, _SupportsLoRAType)
129 +
130 + return isinstance(model, SupportsLoRA)
vllm/model_executor/models/llama.py
+8 -1
@@ -49,6 +49,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.sequence import SamplerOutput
50 from vllm.utils import is_hip, print_warning_once
51
52 +from .interfaces import SupportsLoRA
53 +
54
55 class LlamaMLP(nn.Module):
56
@@ -296,7 +298,9 @@ class LlamaModel(nn.Module):
298 return hidden_states
299
300
299 -class LlamaForCausalLM(nn.Module):
301 +class LlamaForCausalLM(nn.Module, SupportsLoRA):
302 + supports_lora = True
303 +
304 packed_modules_mapping = {
305 "qkv_proj": [
306 "q_proj",
@@ -336,7 +340,10 @@ class LlamaForCausalLM(nn.Module):
340 lora_config: Optional[LoRAConfig] = None,
341 ) -> None:
342 super().__init__()
343 +
344 self.config = config
345 + self.lora_config = lora_config
346 +
347 self.model = LlamaModel(config,
348 cache_config,
349 quant_config,
vllm/model_executor/models/llava.py
+12 -10
@@ -20,7 +20,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
20 from vllm.multimodal.image import get_dummy_image_data
21 from vllm.sequence import SamplerOutput
22
23 -from .vlm_base import VisionLanguageModelBase
23 +from .interfaces import SupportsVision
24
25 _KEYS_TO_MODIFY_MAPPING = {
26 "language_model.lm_head": "lm_head",
@@ -86,18 +86,21 @@ LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
86 @MULTIMODAL_REGISTRY.register_image_feature_input()
87 @MULTIMODAL_REGISTRY.register_image_pixel_input()
88 @MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
89 -class LlavaForConditionalGeneration(VisionLanguageModelBase):
89 +class LlavaForConditionalGeneration(nn.Module, SupportsVision):
90 +
91 + supports_vision = True
92
93 def __init__(self,
94 config: LlavaConfig,
93 - vision_language_config: VisionLanguageConfig,
95 + vlm_config: VisionLanguageConfig,
96 cache_config: Optional[CacheConfig] = None,
97 quant_config: Optional[QuantizationConfig] = None) -> None:
96 - super().__init__(vision_language_config)
98 + super().__init__()
99
100 self.config = config
101 + self.vlm_config = vlm_config
102
100 - if self.vision_language_config.image_input_type == (
103 + if self.vlm_config.image_input_type == (
104 VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
105 self.vision_tower = CLIPVisionModel(config.vision_config)
106 else:
@@ -122,11 +125,10 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
125 self.sampler = Sampler()
126
127 def _validate_image_data(self, data: torch.Tensor) -> torch.Tensor:
125 - if list(data.shape[1:]) != list(
126 - self.vision_language_config.image_input_shape[1:]):
128 + if list(data.shape[1:]) != list(self.vlm_config.image_input_shape[1:]):
129 raise ValueError(
130 f"The expected image tensor shape is batch dimension plus "
129 - f"{self.vision_language_config.image_input_shape[1:]}. "
131 + f"{self.vlm_config.image_input_shape[1:]}. "
132 f"You supplied {data.shape}. "
133 f"If you are using vLLM's entrypoint, make sure your "
134 f"supplied image input is consistent with "
@@ -139,7 +141,7 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
141 pixel_values = kwargs.pop("pixel_values", None)
142 image_features = kwargs.pop("image_features", None)
143
142 - expected_input_type = self.vision_language_config.image_input_type
144 + expected_input_type = self.vlm_config.image_input_type
145 ImageInputType = VisionLanguageConfig.ImageInputType
146
147 if expected_input_type == ImageInputType.PIXEL_VALUES:
@@ -273,7 +275,7 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
275
276 inputs_embeds = merge_vision_embeddings(
277 input_ids, inputs_embeds, vision_embeddings,
276 - self.vision_language_config.image_token_id)
278 + self.vlm_config.image_token_id)
279
280 input_ids = None
281 else:
vllm/model_executor/models/llava_next.py
+11 -9
@@ -25,8 +25,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
25 from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
26 from vllm.sequence import SamplerOutput, SequenceData
27
28 +from .interfaces import SupportsVision
29 from .llava import LlavaMultiModalProjector, merge_vision_embeddings
29 -from .vlm_base import VisionLanguageModelBase
30
31 logger = init_logger(__name__)
32
@@ -106,19 +106,21 @@ def _image_pixel_processor(
106
107 @MULTIMODAL_REGISTRY.register_image_pixel_input(_image_pixel_processor)
108 @MULTIMODAL_REGISTRY.register_dummy_data(_get_dummy_image_data)
109 -class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
109 +class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
110 +
111 + supports_vision = True
112
113 def __init__(self,
114 config: LlavaNextConfig,
113 - vision_language_config: VisionLanguageConfig,
115 + vlm_config: VisionLanguageConfig,
116 cache_config: Optional[CacheConfig] = None,
117 quant_config: Optional[QuantizationConfig] = None) -> None:
116 - super().__init__(vision_language_config)
118 + super().__init__()
119
118 - # Update the type annotation from that of its superclass
120 self.config = config
121 + self.vlm_config = vlm_config
122
121 - if self.vision_language_config.image_input_type == (
123 + if self.vlm_config.image_input_type == (
124 VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
125 self.vision_tower = CLIPVisionModel(config=config.vision_config)
126 else:
@@ -146,7 +148,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
148 torch.empty(config.text_config.hidden_size))
149
150 def _validate_image_pixels(self, data: torch.Tensor) -> torch.Tensor:
149 - _, num_channels, _, _ = self.vision_language_config.image_input_shape
151 + _, num_channels, _, _ = self.vlm_config.image_input_shape
152
153 # Note that this is different from that of vLLM vision_language_config
154 # since the image is resized by the HuggingFace preprocessor
@@ -177,7 +179,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
179 image_sizes = kwargs.pop("image_sizes", None)
180 image_features = kwargs.pop("image_features", None)
181
180 - expected_input_type = self.vision_language_config.image_input_type
182 + expected_input_type = self.vlm_config.image_input_type
183 ImageInputType = VisionLanguageConfig.ImageInputType
184
185 if expected_input_type == ImageInputType.PIXEL_VALUES:
@@ -386,7 +388,7 @@ class LlavaNextForConditionalGeneration(VisionLanguageModelBase):
388
389 inputs_embeds = merge_vision_embeddings(
390 input_ids, inputs_embeds, vision_embeddings,
389 - self.vision_language_config.image_token_id)
391 + self.vlm_config.image_token_id)
392
393 input_ids = None
394 else:
vllm/model_executor/models/minicpm.py
+10 -2
@@ -26,6 +26,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
26
27 import torch
28 from torch import nn
29 +from transformers import PretrainedConfig
30
31 from vllm.attention import Attention, AttentionMetadata
32 from vllm.config import CacheConfig, LoRAConfig
@@ -51,6 +52,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
52 from vllm.model_executor.utils import set_weight_attrs
53 from vllm.sequence import SamplerOutput
54
55 +from .interfaces import SupportsLoRA
56 +
57
58 class MiniCPMMoE(nn.Module):
59 """A tensor-parallel MoE implementation that shards each expert
@@ -388,7 +391,9 @@ class MiniCPMModel(nn.Module):
391 return hidden_states
392
393
391 -class MiniCPMForCausalLM(nn.Module):
394 +class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
395 + supports_lora = True
396 +
397 packed_modules_mapping = {
398 "qkv_proj": [
399 "q_proj",
@@ -418,13 +423,16 @@ class MiniCPMForCausalLM(nn.Module):
423
424 def __init__(
425 self,
421 - config,
426 + config: PretrainedConfig,
427 cache_config: Optional[CacheConfig] = None,
428 quant_config: Optional[QuantizationConfig] = None,
429 lora_config: Optional[LoRAConfig] = None,
430 ) -> None:
431 super().__init__()
432 +
433 self.config = config
434 + self.lora_config = lora_config
435 +
436 self.num_experts = getattr(self.config, "num_experts", 0)
437 self.quant_config = quant_config
438 self.model = MiniCPMModel(config,
vllm/model_executor/models/mixtral.py
+8 -1
@@ -54,6 +54,8 @@ from vllm.model_executor.utils import set_weight_attrs
54 from vllm.sequence import SamplerOutput
55 from vllm.utils import print_warning_once
56
57 +from .interfaces import SupportsLoRA
58 +
59
60 class MixtralMoE(nn.Module):
61 """A tensor-parallel MoE implementation for Mixtral that shards each expert
@@ -472,7 +474,9 @@ class MixtralModel(nn.Module):
474 return hidden_states
475
476
475 -class MixtralForCausalLM(nn.Module):
477 +class MixtralForCausalLM(nn.Module, SupportsLoRA):
478 + supports_lora = True
479 +
480 fall_back_to_pt_during_load = False
481
482 packed_modules_mapping = {
@@ -504,7 +508,10 @@ class MixtralForCausalLM(nn.Module):
508 lora_config: Optional[LoRAConfig] = None,
509 ) -> None:
510 super().__init__()
511 +
512 self.config = config
513 + self.lora_config = lora_config
514 +
515 self.model = MixtralModel(config,
516 cache_config,
517 quant_config,
vllm/model_executor/models/phi.py
+14 -8
@@ -39,7 +39,7 @@ from typing import Iterable, List, Optional, Tuple
39
40 import torch
41 from torch import nn
42 -from transformers import PretrainedConfig
42 +from transformers import PhiConfig
43
44 from vllm.attention import Attention, AttentionMetadata
45 from vllm.config import CacheConfig, LoRAConfig
@@ -59,11 +59,13 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59 from vllm.model_executor.sampling_metadata import SamplingMetadata
60 from vllm.sequence import SamplerOutput
61
62 +from .interfaces import SupportsLoRA
63 +
64
65 class PhiAttention(nn.Module):
66
67 def __init__(self,
66 - config: PretrainedConfig,
68 + config: PhiConfig,
69 cache_config: Optional[CacheConfig] = None,
70 quant_config: Optional[QuantizationConfig] = None):
71 super().__init__()
@@ -131,7 +133,7 @@ class PhiAttention(nn.Module):
133 class PhiMLP(nn.Module):
134
135 def __init__(self,
134 - config: PretrainedConfig,
136 + config: PhiConfig,
137 quant_config: Optional[QuantizationConfig] = None):
138 super().__init__()
139
@@ -160,7 +162,7 @@ class PhiMLP(nn.Module):
162 class PhiLayer(nn.Module):
163
164 def __init__(self,
163 - config: PretrainedConfig,
165 + config: PhiConfig,
166 cache_config: Optional[CacheConfig] = None,
167 quant_config: Optional[QuantizationConfig] = None):
168 super().__init__()
@@ -192,7 +194,7 @@ class PhiLayer(nn.Module):
194 class PhiModel(nn.Module):
195
196 def __init__(self,
195 - config: PretrainedConfig,
197 + config: PhiConfig,
198 cache_config: Optional[CacheConfig] = None,
199 quant_config: Optional[QuantizationConfig] = None):
200 super().__init__()
@@ -229,7 +231,9 @@ class PhiModel(nn.Module):
231 return hidden_states
232
233
232 -class PhiForCausalLM(nn.Module):
234 +class PhiForCausalLM(nn.Module, SupportsLoRA):
235 + supports_lora = True
236 +
237 packed_modules_mapping = {
238 "qkv_proj": [
239 "q_proj",
@@ -250,14 +254,16 @@ class PhiForCausalLM(nn.Module):
254
255 def __init__(
256 self,
253 - config: PretrainedConfig,
257 + config: PhiConfig,
258 cache_config: Optional[CacheConfig] = None,
259 quant_config: Optional[QuantizationConfig] = None,
260 lora_config: Optional[LoRAConfig] = None,
261 ):
258 - del lora_config # Unused.
262 super().__init__()
263 +
264 self.config = config
265 + self.lora_config = lora_config
266 +
267 self.quant_config = quant_config
268
269 self.model = PhiModel(config, cache_config, quant_config)
vllm/model_executor/models/qwen2.py
+8 -2
@@ -48,6 +48,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.sequence import SamplerOutput
49 from vllm.utils import print_warning_once
50
51 +from .interfaces import SupportsLoRA
52 +
53
54 class Qwen2MLP(nn.Module):
55
@@ -263,7 +265,9 @@ class Qwen2Model(nn.Module):
265 return hidden_states
266
267
266 -class Qwen2ForCausalLM(nn.Module):
268 +class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
269 + supports_lora = True
270 +
271 packed_modules_mapping = {
272 "qkv_proj": [
273 "q_proj",
@@ -293,7 +297,6 @@ class Qwen2ForCausalLM(nn.Module):
297 quant_config: Optional[QuantizationConfig] = None,
298 lora_config: Optional[LoRAConfig] = None,
299 ) -> None:
296 - del lora_config
300 # TODO (@robertgshaw2): see if this can be moved out
301 if (cache_config.sliding_window is not None
302 and hasattr(config, "max_window_layers")):
@@ -307,7 +310,10 @@ class Qwen2ForCausalLM(nn.Module):
310 ))
311
312 super().__init__()
313 +
314 self.config = config
315 + self.lora_config = lora_config
316 +
317 self.quant_config = quant_config
318 self.model = Qwen2Model(config, cache_config, quant_config)
319
vllm/model_executor/models/vlm_base.py deleted
-12
@@ -1,12 +0,0 @@
1 -from torch import nn
2 -
3 -from vllm.config import VisionLanguageConfig
4 -
5 -
6 -class VisionLanguageModelBase(nn.Module):
7 - """Base class for all vision language models (VLMs)."""
8 -
9 - def __init__(self, vision_language_config: VisionLanguageConfig) -> None:
10 - super().__init__()
11 -
12 - self.vision_language_config = vision_language_config
vllm/model_executor/models/xverse.py
+9 -2
@@ -45,6 +45,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.sequence import SamplerOutput
47
48 +from .interfaces import SupportsLoRA
49 +
50
51 class XverseMLP(nn.Module):
52
@@ -266,7 +268,9 @@ class XverseModel(nn.Module):
268 return hidden_states
269
270
269 -class XverseForCausalLM(nn.Module):
271 +class XverseForCausalLM(nn.Module, SupportsLoRA):
272 + supports_lora = True
273 +
274 packed_modules_mapping = {
275 "qkv_proj": [
276 "q_proj",
@@ -299,10 +303,13 @@ class XverseForCausalLM(nn.Module):
303 config: PretrainedConfig,
304 cache_config: Optional[CacheConfig] = None,
305 quant_config: Optional[QuantizationConfig] = None,
302 - lora_config=None,
306 + lora_config: Optional[LoRAConfig] = None,
307 ) -> None:
308 super().__init__()
309 +
310 self.config = config
311 + self.lora_config = lora_config
312 +
313 self.quant_config = quant_config
314 self.model = XverseModel(config, cache_config, quant_config)
315 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
vllm/worker/model_runner.py
+3 -8
@@ -22,6 +22,7 @@ from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
22 from vllm.model_executor import SamplingMetadata
23 from vllm.model_executor.model_loader import get_model
24 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
25 +from vllm.model_executor.models.interfaces import supports_lora
26 from vllm.multimodal import MULTIMODAL_REGISTRY
27 from vllm.sampling_params import SamplingParams
28 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
@@ -225,14 +226,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
226 self.model_memory_usage / float(2**30))
227
228 if self.lora_config:
228 - assert hasattr(self.model, "supported_lora_modules"
229 - ) and self.model.supported_lora_modules, (
230 - "Model does not support LoRA")
231 - assert hasattr(
232 - self.model,
233 - "embedding_modules"), "Model does not have embedding_modules"
234 - assert hasattr(self.model, "embedding_padding_modules"
235 - ), "Model does not have embedding_padding_modules"
229 + assert supports_lora(self.model), "Model does not support LoRA"
230 +
231 self.lora_manager = LRUCacheWorkerLoRAManager(
232 self.scheduler_config.max_num_seqs,
233 self.scheduler_config.max_num_batched_tokens,