A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model][Bugfix] Implicit model flags and reenable Phi-3-Vision (#5896)
Cyrus Leung committed
Jun 28, 2024 at 00:08 UTC
98cf2ed678580326ffc39c987304c61cb0ce4981
14 files changed
+26
-32
vllm/model_executor/models/baichuan.py
-2
@@ -295,8 +295,6 @@ class BaiChuanModel(nn.Module):
295
296
297
class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
298
- supports_lora = True
299
-
298
packed_modules_mapping = {
299
"W_pack": ["W_pack"],
300
"gate_up_proj": [
vllm/model_executor/models/chatglm.py
-2
@@ -325,8 +325,6 @@ class ChatGLMModel(nn.Module):
325
326
327
class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
328
- supports_lora = True
329
-
328
packed_modules_mapping = {
329
"query_key_value": ["query_key_value"],
330
"dense_h_to_4h": ["dense_h_to_4h"]
vllm/model_executor/models/gemma.py
-2
@@ -291,8 +291,6 @@ class GemmaModel(nn.Module):
291
292
293
class GemmaForCausalLM(nn.Module, SupportsLoRA):
294
- supports_lora = True
295
-
294
packed_modules_mapping = {
295
"qkv_proj": [
296
"q_proj",
vllm/model_executor/models/gpt_bigcode.py
-2
@@ -233,8 +233,6 @@ class GPTBigCodeModel(nn.Module):
233
234
235
class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
236
- supports_lora = True
237
-
236
packed_modules_mapping = {"c_attn": ["c_attn"]}
237
238
supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]
vllm/model_executor/models/interfaces.py
+16
-2
@@ -13,7 +13,14 @@ logger = init_logger(__name__)
13
class SupportsVision(Protocol):
14
"""The interface required for all vision language models (VLMs)."""
15
16
- supports_vision: ClassVar[Literal[True]]
16
+ supports_vision: ClassVar[Literal[True]] = True
17
+ """
18
+ A flag that indicates this model supports vision inputs.
19
+
20
+ Note:
21
+ There is no need to redefine this flag if this class is in the
22
+ MRO of your model class.
23
+ """
24
25
def __init__(self, *, vlm_config: VisionLanguageConfig) -> None:
26
...
@@ -52,7 +59,14 @@ def supports_vision(
59
class SupportsLoRA(Protocol):
60
"""The interface required for all models that support LoRA."""
61
55
- supports_lora: ClassVar[Literal[True]]
62
+ supports_lora: ClassVar[Literal[True]] = True
63
+ """
64
+ A flag that indicates this model supports LoRA.
65
+
66
+ Note:
67
+ There is no need to redefine this flag if this class is in the
68
+ MRO of your model class.
69
+ """
70
71
packed_modules_mapping: ClassVar[Dict[str, List[str]]]
72
supported_lora_modules: ClassVar[List[str]]
vllm/model_executor/models/llama.py
-2
@@ -299,8 +299,6 @@ class LlamaModel(nn.Module):
299
300
301
class LlamaForCausalLM(nn.Module, SupportsLoRA):
302
- supports_lora = True
303
-
302
packed_modules_mapping = {
303
"qkv_proj": [
304
"q_proj",
vllm/model_executor/models/llava.py
-2
@@ -88,8 +88,6 @@ LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
88
@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
89
class LlavaForConditionalGeneration(nn.Module, SupportsVision):
90
91
- supports_vision = True
92
-
91
def __init__(self,
92
config: LlavaConfig,
93
vlm_config: VisionLanguageConfig,
vllm/model_executor/models/llava_next.py
-2
@@ -108,8 +108,6 @@ def _image_pixel_processor(
108
@MULTIMODAL_REGISTRY.register_dummy_data(_get_dummy_image_data)
109
class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
110
111
- supports_vision = True
112
-
111
def __init__(self,
112
config: LlavaNextConfig,
113
vlm_config: VisionLanguageConfig,
vllm/model_executor/models/minicpm.py
-2
@@ -392,8 +392,6 @@ class MiniCPMModel(nn.Module):
392
393
394
class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
395
- supports_lora = True
396
-
395
packed_modules_mapping = {
396
"qkv_proj": [
397
"q_proj",
vllm/model_executor/models/mixtral.py
-2
@@ -475,8 +475,6 @@ class MixtralModel(nn.Module):
475
476
477
class MixtralForCausalLM(nn.Module, SupportsLoRA):
478
- supports_lora = True
479
-
478
fall_back_to_pt_during_load = False
479
480
packed_modules_mapping = {
vllm/model_executor/models/phi.py
-2
@@ -232,8 +232,6 @@ class PhiModel(nn.Module):
232
233
234
class PhiForCausalLM(nn.Module, SupportsLoRA):
235
- supports_lora = True
236
-
235
packed_modules_mapping = {
236
"qkv_proj": [
237
"q_proj",
vllm/model_executor/models/phi3v.py
+10
-6
@@ -32,12 +32,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
32
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33
from vllm.model_executor.models.clip import CLIPVisionModel
34
from vllm.model_executor.models.llama import LlamaModel
35
-from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
from vllm.multimodal import MULTIMODAL_REGISTRY
37
from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
38
from vllm.sequence import SamplerOutput
39
40
+from .interfaces import SupportsVision
41
+
42
logger = init_logger(__name__)
43
44
_KEYS_TO_MODIFY_MAPPING = {
@@ -317,18 +318,21 @@ def _image_processor(
318
319
@MULTIMODAL_REGISTRY.register_image_pixel_input(_image_processor)
320
@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
320
-class Phi3VForCausalLM(VisionLanguageModelBase):
321
+class Phi3VForCausalLM(nn.Module, SupportsVision):
322
323
def __init__(self,
324
config: PretrainedConfig,
324
- vision_language_config: VisionLanguageConfig,
325
+ vlm_config: VisionLanguageConfig,
326
cache_config: Optional[CacheConfig] = None,
327
quant_config: Optional[QuantizationConfig] = None) -> None:
327
- super().__init__(vision_language_config)
328
+ super().__init__()
329
+
330
self.config = config
331
+ self.vlm_config = vlm_config
332
+
333
self.model = LlamaModel(config, cache_config, quant_config)
334
self.vision_embed_tokens = Phi3HDImageEmbedding(
331
- vision_language_config, config, self.model.embed_tokens)
335
+ vlm_config, config, self.model.embed_tokens)
336
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
337
self.logits_processor = LogitsProcessor(config.vocab_size)
338
self.sampler = Sampler()
@@ -338,7 +342,7 @@ class Phi3VForCausalLM(VisionLanguageModelBase):
342
pixel_values = kwargs.pop("pixel_values", None)
343
image_sizes = kwargs.pop("image_sizes", None)
344
341
- expected_input_type = self.vision_language_config.image_input_type
345
+ expected_input_type = self.vlm_config.image_input_type
346
ImageInputType = VisionLanguageConfig.ImageInputType
347
348
if expected_input_type != ImageInputType.PIXEL_VALUES:
vllm/model_executor/models/qwen2.py
-2
@@ -266,8 +266,6 @@ class Qwen2Model(nn.Module):
266
267
268
class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
269
- supports_lora = True
270
-
269
packed_modules_mapping = {
270
"qkv_proj": [
271
"q_proj",
vllm/model_executor/models/xverse.py
-2
@@ -269,8 +269,6 @@ class XverseModel(nn.Module):
269
270
271
class XverseForCausalLM(nn.Module, SupportsLoRA):
272
- supports_lora = True
273
-
272
packed_modules_mapping = {
273
"qkv_proj": [
274
"q_proj",