@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 98cf2ed678
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model][Bugfix] Implicit model flags and reenable Phi-3-Vision (#5896)

Cyrus Leung committed Jun 28, 2024 at 00:08 UTC 98cf2ed678580326ffc39c987304c61cb0ce4981
14 files changed +26 -32
vllm/model_executor/models/baichuan.py
-2
@@ -295,8 +295,6 @@ class BaiChuanModel(nn.Module):
295
296
297 class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA):
298 - supports_lora = True
299 -
298 packed_modules_mapping = {
299 "W_pack": ["W_pack"],
300 "gate_up_proj": [
vllm/model_executor/models/chatglm.py
-2
@@ -325,8 +325,6 @@ class ChatGLMModel(nn.Module):
325
326
327 class ChatGLMForCausalLM(nn.Module, SupportsLoRA):
328 - supports_lora = True
329 -
328 packed_modules_mapping = {
329 "query_key_value": ["query_key_value"],
330 "dense_h_to_4h": ["dense_h_to_4h"]
vllm/model_executor/models/gemma.py
-2
@@ -291,8 +291,6 @@ class GemmaModel(nn.Module):
291
292
293 class GemmaForCausalLM(nn.Module, SupportsLoRA):
294 - supports_lora = True
295 -
294 packed_modules_mapping = {
295 "qkv_proj": [
296 "q_proj",
vllm/model_executor/models/gpt_bigcode.py
-2
@@ -233,8 +233,6 @@ class GPTBigCodeModel(nn.Module):
233
234
235 class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA):
236 - supports_lora = True
237 -
236 packed_modules_mapping = {"c_attn": ["c_attn"]}
237
238 supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]
vllm/model_executor/models/interfaces.py
+16 -2
@@ -13,7 +13,14 @@ logger = init_logger(__name__)
13 class SupportsVision(Protocol):
14 """The interface required for all vision language models (VLMs)."""
15
16 - supports_vision: ClassVar[Literal[True]]
16 + supports_vision: ClassVar[Literal[True]] = True
17 + """
18 + A flag that indicates this model supports vision inputs.
19 +
20 + Note:
21 + There is no need to redefine this flag if this class is in the
22 + MRO of your model class.
23 + """
24
25 def __init__(self, *, vlm_config: VisionLanguageConfig) -> None:
26 ...
@@ -52,7 +59,14 @@ def supports_vision(
59 class SupportsLoRA(Protocol):
60 """The interface required for all models that support LoRA."""
61
55 - supports_lora: ClassVar[Literal[True]]
62 + supports_lora: ClassVar[Literal[True]] = True
63 + """
64 + A flag that indicates this model supports LoRA.
65 +
66 + Note:
67 + There is no need to redefine this flag if this class is in the
68 + MRO of your model class.
69 + """
70
71 packed_modules_mapping: ClassVar[Dict[str, List[str]]]
72 supported_lora_modules: ClassVar[List[str]]
vllm/model_executor/models/llama.py
-2
@@ -299,8 +299,6 @@ class LlamaModel(nn.Module):
299
300
301 class LlamaForCausalLM(nn.Module, SupportsLoRA):
302 - supports_lora = True
303 -
302 packed_modules_mapping = {
303 "qkv_proj": [
304 "q_proj",
vllm/model_executor/models/llava.py
-2
@@ -88,8 +88,6 @@ LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
88 @MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
89 class LlavaForConditionalGeneration(nn.Module, SupportsVision):
90
91 - supports_vision = True
92 -
91 def __init__(self,
92 config: LlavaConfig,
93 vlm_config: VisionLanguageConfig,
vllm/model_executor/models/llava_next.py
-2
@@ -108,8 +108,6 @@ def _image_pixel_processor(
108 @MULTIMODAL_REGISTRY.register_dummy_data(_get_dummy_image_data)
109 class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
110
111 - supports_vision = True
112 -
111 def __init__(self,
112 config: LlavaNextConfig,
113 vlm_config: VisionLanguageConfig,
vllm/model_executor/models/minicpm.py
-2
@@ -392,8 +392,6 @@ class MiniCPMModel(nn.Module):
392
393
394 class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
395 - supports_lora = True
396 -
395 packed_modules_mapping = {
396 "qkv_proj": [
397 "q_proj",
vllm/model_executor/models/mixtral.py
-2
@@ -475,8 +475,6 @@ class MixtralModel(nn.Module):
475
476
477 class MixtralForCausalLM(nn.Module, SupportsLoRA):
478 - supports_lora = True
479 -
478 fall_back_to_pt_during_load = False
479
480 packed_modules_mapping = {
vllm/model_executor/models/phi.py
-2
@@ -232,8 +232,6 @@ class PhiModel(nn.Module):
232
233
234 class PhiForCausalLM(nn.Module, SupportsLoRA):
235 - supports_lora = True
236 -
235 packed_modules_mapping = {
236 "qkv_proj": [
237 "q_proj",
vllm/model_executor/models/phi3v.py
+10 -6
@@ -32,12 +32,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
32 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
33 from vllm.model_executor.models.clip import CLIPVisionModel
34 from vllm.model_executor.models.llama import LlamaModel
35 -from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
36 from vllm.multimodal import MULTIMODAL_REGISTRY
37 from vllm.multimodal.image import ImagePixelData, get_dummy_image_data
38 from vllm.sequence import SamplerOutput
39
40 +from .interfaces import SupportsVision
41 +
42 logger = init_logger(__name__)
43
44 _KEYS_TO_MODIFY_MAPPING = {
@@ -317,18 +318,21 @@ def _image_processor(
318
319 @MULTIMODAL_REGISTRY.register_image_pixel_input(_image_processor)
320 @MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
320 -class Phi3VForCausalLM(VisionLanguageModelBase):
321 +class Phi3VForCausalLM(nn.Module, SupportsVision):
322
323 def __init__(self,
324 config: PretrainedConfig,
324 - vision_language_config: VisionLanguageConfig,
325 + vlm_config: VisionLanguageConfig,
326 cache_config: Optional[CacheConfig] = None,
327 quant_config: Optional[QuantizationConfig] = None) -> None:
327 - super().__init__(vision_language_config)
328 + super().__init__()
329 +
330 self.config = config
331 + self.vlm_config = vlm_config
332 +
333 self.model = LlamaModel(config, cache_config, quant_config)
334 self.vision_embed_tokens = Phi3HDImageEmbedding(
331 - vision_language_config, config, self.model.embed_tokens)
335 + vlm_config, config, self.model.embed_tokens)
336 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
337 self.logits_processor = LogitsProcessor(config.vocab_size)
338 self.sampler = Sampler()
@@ -338,7 +342,7 @@ class Phi3VForCausalLM(VisionLanguageModelBase):
342 pixel_values = kwargs.pop("pixel_values", None)
343 image_sizes = kwargs.pop("image_sizes", None)
344
341 - expected_input_type = self.vision_language_config.image_input_type
345 + expected_input_type = self.vlm_config.image_input_type
346 ImageInputType = VisionLanguageConfig.ImageInputType
347
348 if expected_input_type != ImageInputType.PIXEL_VALUES:
vllm/model_executor/models/qwen2.py
-2
@@ -266,8 +266,6 @@ class Qwen2Model(nn.Module):
266
267
268 class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
269 - supports_lora = True
270 -
269 packed_modules_mapping = {
270 "qkv_proj": [
271 "q_proj",
vllm/model_executor/models/xverse.py
-2
@@ -269,8 +269,6 @@ class XverseModel(nn.Module):
269
270
271 class XverseForCausalLM(nn.Module, SupportsLoRA):
272 - supports_lora = True
273 -
272 packed_modules_mapping = {
273 "qkv_proj": [
274 "q_proj",