@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f26efbb3a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] Support SigLIP encoder and alternative decoders for LLaVA models (#7153)

Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Cyrus Leung committed Aug 6, 2024 at 16:55 UTC 1f26efbb3a5e6dad0b98421dd697167c42a50629
14 files changed +455 -269
requirements-test.txt
+3
@@ -20,6 +20,9 @@ sentence-transformers # required for embedding
20 compressed-tensors==0.4.0 # required for compressed-tensors
21 timm # required for internvl test
22
23 +# TODO: Add this after fully implementing llava(mantis)
24 +# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
25 +
26 # Benchmarking
27 aiohttp
28
tests/models/test_llava.py
+31 -5
@@ -1,10 +1,11 @@
1 from typing import List, Optional, Tuple, Type
2
3 import pytest
4 -from transformers import AutoTokenizer
4 +from transformers import AutoConfig, AutoTokenizer
5
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
8 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
9
10 from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
11 from .utils import check_logprobs_close
@@ -18,9 +19,11 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19 "USER: <image>\nWhat is the season?\nASSISTANT:",
20 })
21
21 -IMAGE_TOKEN_ID = 32000
22 -
23 -models = ["llava-hf/llava-1.5-7b-hf"]
22 +models = [
23 + "llava-hf/llava-1.5-7b-hf",
24 + # TODO: Get this model to produce meaningful output in vLLM
25 + # "TIGER-Lab/Mantis-8B-siglip-llama3",
26 +]
27
28
29 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -29,12 +32,15 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
32 """Sanitize vllm output to be comparable with hf output."""
33 output_ids, output_str, out_logprobs = vllm_output
34
35 + config = AutoConfig.from_pretrained(model)
36 + image_token_id = config.image_token_index
37 +
38 tokenizer = AutoTokenizer.from_pretrained(model)
39 eos_token_id = tokenizer.eos_token_id
40
41 hf_output_ids = [
42 token_id for idx, token_id in enumerate(output_ids)
37 - if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
43 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
44 ]
45
46 assert output_str[0] == " "
@@ -67,6 +73,17 @@ def run_test(
73 Note, the text input is also adjusted to abide by vllm contract.
74 The text output is sanitized to be able to compare with hf.
75 """
76 + # NOTE: For local use; this isn't tested in CI yet (see TODO above)
77 + if model.startswith("TIGER-Lab/Mantis"):
78 + from mantis.models.mllava import MLlavaProcessor
79 +
80 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
81 + mantis_processor = MLlavaProcessor.from_pretrained(
82 + model, torch_dtype=torch_dtype)
83 + assert isinstance(mantis_processor, MLlavaProcessor)
84 + else:
85 + mantis_processor = None
86 +
87 images = [asset.pil_image for asset in image_assets]
88
89 inputs_per_image = [(
@@ -94,6 +111,15 @@ def run_test(
111 ]
112
113 with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
114 + if mantis_processor is not None:
115 +
116 + def process(*args, **kwargs):
117 + output = mantis_processor(*args, **kwargs)
118 + output["pixel_values"] = output["pixel_values"].to(torch_dtype)
119 + return output
120 +
121 + hf_model.processor = process
122 +
123 hf_outputs_per_image = [
124 hf_model.generate_greedy_logprobs_limit(prompts,
125 max_tokens,
tests/models/test_llava_next.py
+5 -4
@@ -1,7 +1,7 @@
1 from typing import List, Optional, Tuple, Type, overload
2
3 import pytest
4 -from transformers import AutoTokenizer
4 +from transformers import AutoConfig, AutoTokenizer
5
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
@@ -23,8 +23,6 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
23 f"{_PREFACE} USER: <image>\nWhat is the season? ASSISTANT:",
24 })
25
26 -IMAGE_TOKEN_ID = 32000
27 -
26 models = ["llava-hf/llava-v1.6-vicuna-7b-hf"]
27
28
@@ -34,12 +32,15 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
32 """Sanitize vllm output to be comparable with hf output."""
33 output_ids, output_str, out_logprobs = vllm_output
34
35 + config = AutoConfig.from_pretrained(model)
36 + image_token_id = config.image_token_index
37 +
38 tokenizer = AutoTokenizer.from_pretrained(model)
39 eos_token_id = tokenizer.eos_token_id
40
41 hf_output_ids = [
42 token_id for idx, token_id in enumerate(output_ids)
42 - if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
43 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
44 ]
45
46 assert output_str[0] == " "
tests/models/test_paligemma.py
+5 -4
@@ -2,7 +2,7 @@ import os
2 from typing import List, Optional, Tuple, Type
3
4 import pytest
5 -from transformers import AutoTokenizer
5 +from transformers import AutoConfig, AutoTokenizer
6
7 from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
@@ -20,8 +20,6 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20 "What is in the picture?",
21 })
22
23 -IMAGE_TOKEN_ID = 257152
24 -
23 models = ["google/paligemma-3b-mix-224"]
24
25 # ROCm Triton FA can run into compilation issues with these models due to,
@@ -37,12 +35,15 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
35 """Sanitize vllm output to be comparable with hf output."""
36 output_ids, output_str, out_logprobs = vllm_output
37
38 + config = AutoConfig.from_pretrained(model)
39 + image_token_id = config.image_token_index
40 +
41 tokenizer = AutoTokenizer.from_pretrained(model)
42 eos_token_id = tokenizer.eos_token_id
43
44 hf_output_ids = [
45 token_id for idx, token_id in enumerate(output_ids)
45 - if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
46 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
47 ]
48
49 hf_output_str = output_str
tests/models/test_registry.py
+1 -1
@@ -6,4 +6,4 @@ from vllm.model_executor.models import _MODELS, ModelRegistry
6 @pytest.mark.parametrize("model_cls", _MODELS)
7 def test_registry_imports(model_cls):
8 # Ensure all model classes can be imported successfully
9 - ModelRegistry.load_model_cls(model_cls)
9 + ModelRegistry.resolve_model_cls([model_cls])
vllm/model_executor/model_loader/loader.py
+28 -10
@@ -16,7 +16,7 @@ import numpy as np
16 import torch
17 from huggingface_hub import HfApi, hf_hub_download
18 from torch import nn
19 -from transformers import AutoModelForCausalLM
19 +from transformers import AutoModelForCausalLM, PretrainedConfig
20
21 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
22 LoRAConfig, ModelConfig, MultiModalConfig,
@@ -143,6 +143,22 @@ def _get_model_initialization_kwargs(
143 return extra_kwargs
144
145
146 +def build_model(model_class: Type[nn.Module], hf_config: PretrainedConfig,
147 + cache_config: Optional[CacheConfig],
148 + quant_config: Optional[QuantizationConfig], *,
149 + lora_config: Optional[LoRAConfig],
150 + multimodal_config: Optional[MultiModalConfig],
151 + scheduler_config: Optional[SchedulerConfig]) -> nn.Module:
152 + extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
153 + multimodal_config,
154 + scheduler_config)
155 +
156 + return model_class(config=hf_config,
157 + cache_config=cache_config,
158 + quant_config=quant_config,
159 + **extra_kwargs)
160 +
161 +
162 def _initialize_model(
163 model_config: ModelConfig,
164 load_config: LoadConfig,
@@ -151,15 +167,17 @@ def _initialize_model(
167 cache_config: CacheConfig,
168 scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
169 """Initialize a model with the given configurations."""
154 - model_class = get_model_architecture(model_config)[0]
155 - quant_config = _get_quantization_config(model_config, load_config)
156 -
157 - return model_class(config=model_config.hf_config,
158 - cache_config=cache_config,
159 - quant_config=quant_config,
160 - **_get_model_initialization_kwargs(
161 - model_class, lora_config, multimodal_config,
162 - scheduler_config))
170 + model_class, _ = get_model_architecture(model_config)
171 +
172 + return build_model(
173 + model_class,
174 + model_config.hf_config,
175 + quant_config=_get_quantization_config(model_config, load_config),
176 + lora_config=lora_config,
177 + multimodal_config=multimodal_config,
178 + cache_config=cache_config,
179 + scheduler_config=scheduler_config,
180 + )
181
182
183 class BaseModelLoader(ABC):
vllm/model_executor/model_loader/utils.py
+1 -7
@@ -28,13 +28,7 @@ def get_model_architecture(
28 and "MixtralForCausalLM" in architectures):
29 architectures = ["QuantMixtralForCausalLM"]
30
31 - for arch in architectures:
32 - model_cls = ModelRegistry.load_model_cls(arch)
33 - if model_cls is not None:
34 - return (model_cls, arch)
35 - raise ValueError(
36 - f"Model architectures {architectures} are not supported for now. "
37 - f"Supported architectures: {ModelRegistry.get_supported_archs()}")
31 + return ModelRegistry.resolve_model_cls(architectures)
32
33
34 def get_architecture_class_name(model_config: ModelConfig) -> str:
vllm/model_executor/models/__init__.py
+14 -2
@@ -1,6 +1,6 @@
1 import functools
2 import importlib
3 -from typing import Dict, List, Optional, Type
3 +from typing import Dict, List, Optional, Tuple, Type
4
5 import torch.nn as nn
6
@@ -126,7 +126,7 @@ class ModelRegistry:
126 return getattr(module, model_cls_name, None)
127
128 @staticmethod
129 - def load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
129 + def _try_load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
130 if model_arch in _OOT_MODELS:
131 return _OOT_MODELS[model_arch]
132 if model_arch not in _MODELS:
@@ -143,6 +143,18 @@ class ModelRegistry:
143
144 return ModelRegistry._get_model(model_arch)
145
146 + @staticmethod
147 + def resolve_model_cls(
148 + architectures: List[str]) -> Tuple[Type[nn.Module], str]:
149 + for arch in architectures:
150 + model_cls = ModelRegistry._try_load_model_cls(arch)
151 + if model_cls is not None:
152 + return (model_cls, arch)
153 +
154 + raise ValueError(
155 + f"Model architectures {architectures} are not supported for now. "
156 + f"Supported architectures: {ModelRegistry.get_supported_archs()}")
157 +
158 @staticmethod
159 def get_supported_archs() -> List[str]:
160 return list(_MODELS.keys())
vllm/model_executor/models/clip.py
+22 -2
@@ -1,6 +1,6 @@
1 """Minimal implementation of CLIPVisionModel intended to be only used
2 within a vision language model."""
3 -from typing import Optional
3 +from typing import Iterable, Optional, Tuple
4
5 import torch
6 import torch.nn as nn
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
15 RowParallelLinear)
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18 from vllm.multimodal.image import (cached_get_tokenizer,
19 repeat_and_pad_image_tokens)
20 from vllm.sequence import SequenceData
@@ -32,7 +33,7 @@ def get_clip_num_patches(*, image_size: int, patch_size: int) -> int:
33
34 def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
35 return get_clip_num_patches(image_size=hf_config.image_size,
35 - patch_size=hf_config.patch_size)
36 + patch_size=hf_config.patch_size) + 1
37
38
39 def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
@@ -291,3 +292,22 @@ class CLIPVisionModel(nn.Module):
292 @property
293 def device(self):
294 return next(self.parameters()).device
295 +
296 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
297 + params_dict = dict(self.named_parameters())
298 + layer_count = len(self.vision_model.encoder.layers)
299 +
300 + for name, loaded_weight in weights:
301 + # post_layernorm is not needed in CLIPVisionModel
302 + if "vision_model.post_layernorm" in name:
303 + continue
304 + # omit layers when num_hidden_layers_override is set
305 + if "vision_model.encoder.layers." in name:
306 + layer_idx = int(name.split(".")[3])
307 + if layer_idx >= layer_count:
308 + continue
309 +
310 + param = params_dict[name]
311 + weight_loader = getattr(param, "weight_loader",
312 + default_weight_loader)
313 + weight_loader(param, loaded_weight)
vllm/model_executor/models/internvl.py
+7 -17
@@ -18,7 +18,6 @@ from vllm.config import CacheConfig, MultiModalConfig
18 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
19 from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 -from vllm.model_executor.models import ModelRegistry
21 from vllm.model_executor.models.intern_vit import InternVisionModel
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -29,7 +28,8 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
28 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
29 get_clip_num_patches)
30 from .interfaces import SupportsVision
32 -from .utils import merge_vision_embeddings
31 +from .utils import (filter_weights, init_vllm_registered_model,
32 + merge_vision_embeddings)
33
34 IMG_START = '<img>'
35 IMG_END = '</img>'
@@ -283,10 +283,8 @@ class InternVLChatModel(nn.Module, SupportsVision):
283 self.vision_model = InternVisionModel(
284 config.vision_config, num_hidden_layers_override=num_hidden_layers)
285
286 - llm_class = ModelRegistry.load_model_cls(
287 - config.text_config.architectures[0])
288 - self.language_model = llm_class(config.text_config, cache_config,
289 - quant_config)
286 + self.language_model = init_vllm_registered_model(
287 + config.text_config, cache_config, quant_config)
288
289 vit_hidden_size = config.vision_config.hidden_size
290 llm_hidden_size = config.text_config.hidden_size
@@ -415,24 +413,16 @@ class InternVLChatModel(nn.Module, SupportsVision):
413 ) -> Optional[SamplerOutput]:
414 return self.language_model.sample(logits, sampling_metadata)
415
418 - def _filter_weights(self, weights: Iterable[Tuple[str, torch.Tensor]],
419 - prefix: str):
420 - for name, loaded_weight in weights:
421 - name = name.split(".")
422 - if prefix == name.pop(0):
423 - name = ".".join(name)
424 - yield name, loaded_weight
425 -
416 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
417 # prepare weight iterators for components
418 vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
419
420 # load vision encoder
431 - vit_weights = self._filter_weights(vit_weights, "vision_model")
421 + vit_weights = filter_weights(vit_weights, "vision_model")
422 self.vision_model.load_weights(vit_weights)
423
424 # load mlp projector
435 - mlp_weights = self._filter_weights(mlp_weights, "mlp1")
425 + mlp_weights = filter_weights(mlp_weights, "mlp1")
426 mlp_params_dict = dict(self.mlp1.named_parameters())
427 for name, loaded_weight in mlp_weights:
428 param = mlp_params_dict[name]
@@ -441,5 +431,5 @@ class InternVLChatModel(nn.Module, SupportsVision):
431 weight_loader(param, loaded_weight)
432
433 # load llm backbone
444 - llm_weights = self._filter_weights(llm_weights, "language_model")
434 + llm_weights = filter_weights(llm_weights, "language_model")
435 self.language_model.load_weights(llm_weights)
vllm/model_executor/models/llava.py
+113 -98
@@ -1,34 +1,30 @@
1 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1 +import itertools
2 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
3
4 import torch
5 import torch.nn as nn
5 -from transformers import CLIPVisionConfig, LlavaConfig
6 +from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
7
8 from vllm.attention import AttentionMetadata
9 from vllm.config import CacheConfig, MultiModalConfig
10 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11 from vllm.model_executor.layers.activation import get_act_fn
11 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 -from vllm.model_executor.layers.sampler import Sampler
15 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
14 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 -from vllm.model_executor.models.clip import CLIPVisionModel
18 -from vllm.model_executor.models.llama import LlamaModel
15 from vllm.model_executor.sampling_metadata import SamplingMetadata
16 from vllm.multimodal import MULTIMODAL_REGISTRY
17 from vllm.sequence import IntermediateTensors, SamplerOutput
18
23 -from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
24 - get_max_clip_image_tokens, input_processor_for_clip)
19 +from .clip import (CLIPVisionModel, dummy_image_for_clip,
20 + dummy_seq_data_for_clip, get_max_clip_image_tokens,
21 + input_processor_for_clip)
22 from .interfaces import SupportsVision
26 -from .utils import merge_vision_embeddings
27 -
28 -_KEYS_TO_MODIFY_MAPPING = {
29 - "language_model.lm_head": "lm_head",
30 - "language_model.model": "language_model",
31 -}
23 +from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24 + dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
25 + input_processor_for_siglip)
26 +from .utils import (filter_weights, init_vllm_registered_model,
27 + merge_vision_embeddings)
28
29
30 # TODO(xwjiang): Run benchmark and decide if TP.
@@ -67,25 +63,48 @@ def get_max_llava_image_tokens(ctx: InputContext):
63 vision_config = hf_config.vision_config
64
65 if isinstance(vision_config, CLIPVisionConfig):
70 - return get_max_clip_image_tokens(vision_config)
71 -
72 - msg = f"Unsupported vision config: {type(vision_config)}"
73 - raise NotImplementedError(msg)
66 + num_image_tokens = get_max_clip_image_tokens(vision_config)
67 + elif isinstance(vision_config, SiglipVisionConfig):
68 + num_image_tokens = get_max_siglip_image_tokens(vision_config)
69 + else:
70 + msg = f"Unsupported vision config: {type(vision_config)}"
71 + raise NotImplementedError(msg)
72 +
73 + strategy = hf_config.vision_feature_select_strategy
74 + if strategy == "default":
75 + return num_image_tokens - 1
76 + elif strategy == "full":
77 + return num_image_tokens
78 + else:
79 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
80
81
82 def dummy_data_for_llava(ctx: InputContext, seq_len: int):
83 hf_config = ctx.get_hf_config(LlavaConfig)
84 vision_config = hf_config.vision_config
85
86 + image_feature_size = get_max_llava_image_tokens(ctx)
87 +
88 if isinstance(vision_config, CLIPVisionConfig):
89 seq_data = dummy_seq_data_for_clip(
90 vision_config,
91 seq_len,
92 image_token_id=hf_config.image_token_index,
93 + image_feature_size_override=image_feature_size,
94 )
95
96 mm_data = dummy_image_for_clip(vision_config)
97 return seq_data, mm_data
98 + elif isinstance(vision_config, SiglipVisionConfig):
99 + seq_data = dummy_seq_data_for_siglip(
100 + vision_config,
101 + seq_len,
102 + image_token_id=hf_config.image_token_index,
103 + image_feature_size_override=image_feature_size,
104 + )
105 +
106 + mm_data = dummy_image_for_siglip(vision_config)
107 + return seq_data, mm_data
108
109 msg = f"Unsupported vision config: {type(vision_config)}"
110 raise NotImplementedError(msg)
@@ -100,12 +119,49 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
119 hf_config = ctx.get_hf_config(LlavaConfig)
120 vision_config = hf_config.vision_config
121
122 + image_feature_size = get_max_llava_image_tokens(ctx)
123 +
124 if isinstance(vision_config, CLIPVisionConfig):
125 return input_processor_for_clip(
126 model_config,
127 vision_config,
128 llm_inputs,
129 image_token_id=hf_config.image_token_index,
130 + image_feature_size_override=image_feature_size,
131 + )
132 + elif isinstance(vision_config, SiglipVisionConfig):
133 + return input_processor_for_siglip(
134 + model_config,
135 + vision_config,
136 + llm_inputs,
137 + image_token_id=hf_config.image_token_index,
138 + image_feature_size_override=image_feature_size,
139 + )
140 +
141 + msg = f"Unsupported vision config: {type(vision_config)}"
142 + raise NotImplementedError(msg)
143 +
144 +
145 +def _init_vision_tower(hf_config: LlavaConfig):
146 + vision_config = hf_config.vision_config
147 +
148 + # Initialize the vision tower only up to the required feature layer
149 + vision_feature_layer = hf_config.vision_feature_layer
150 + if vision_feature_layer < 0:
151 + num_hidden_layers = hf_config.vision_config.num_hidden_layers \
152 + + vision_feature_layer + 1
153 + else:
154 + num_hidden_layers = vision_feature_layer + 1
155 +
156 + if isinstance(vision_config, CLIPVisionConfig):
157 + return CLIPVisionModel(
158 + vision_config,
159 + num_hidden_layers_override=num_hidden_layers,
160 + )
161 + elif isinstance(vision_config, SiglipVisionConfig):
162 + return SiglipVisionModel(
163 + vision_config,
164 + num_hidden_layers_override=num_hidden_layers,
165 )
166
167 msg = f"Unsupported vision config: {type(vision_config)}"
@@ -128,36 +184,15 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
184 self.config = config
185 self.multimodal_config = multimodal_config
186
131 - # Initialize the vision tower only up to the required feature layer
132 - vision_feature_layer = config.vision_feature_layer
133 - if vision_feature_layer < 0:
134 - num_hidden_layers = config.vision_config.num_hidden_layers \
135 - + vision_feature_layer + 1
136 - else:
137 - num_hidden_layers = vision_feature_layer + 1
138 -
187 # TODO: Optionally initializes this for supporting embeddings.
140 - self.vision_tower = CLIPVisionModel(
141 - config.vision_config, num_hidden_layers_override=num_hidden_layers)
188 + self.vision_tower = _init_vision_tower(config)
189 self.multi_modal_projector = LlavaMultiModalProjector(
190 vision_hidden_size=config.vision_config.hidden_size,
191 text_hidden_size=config.text_config.hidden_size,
192 projector_hidden_act=config.projector_hidden_act)
193
147 - self.quant_config = quant_config
148 - self.language_model = LlamaModel(config.text_config, cache_config,
149 - quant_config)
150 - self.unpadded_vocab_size = config.text_config.vocab_size
151 - self.lm_head = ParallelLMHead(
152 - self.unpadded_vocab_size,
153 - config.text_config.hidden_size,
154 - org_num_embeddings=self.language_model.org_vocab_size,
155 - quant_config=quant_config)
156 - logit_scale = getattr(config, "logit_scale", 1.0)
157 - self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
158 - config.text_config.vocab_size,
159 - logit_scale)
160 - self.sampler = Sampler()
194 + self.language_model = init_vllm_registered_model(
195 + config.text_config, cache_config, quant_config)
196
197 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
198 h = w = self.config.vision_config.image_size
@@ -198,8 +233,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
233
234 raise ValueError(f"Unexpected select feature strategy: {strategy}")
235
201 - def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
202 - pixel_values: torch.Tensor) -> torch.Tensor:
236 + def _image_pixels_to_features(
237 + self,
238 + vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
239 + pixel_values: torch.Tensor,
240 + ) -> torch.Tensor:
241
242 # NOTE: we skip the step to select the vision feature layer since
243 # this is already done inside the vision tower
@@ -272,7 +310,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
310
311 if image_input is not None:
312 vision_embeddings = self._process_image_input(image_input)
275 - inputs_embeds = self.language_model.get_input_embeddings(input_ids)
313 + inputs_embeds = self.language_model.model.get_input_embeddings(
314 + input_ids)
315
316 inputs_embeds = merge_vision_embeddings(
317 input_ids, inputs_embeds, vision_embeddings,
@@ -282,68 +321,44 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
321 else:
322 inputs_embeds = None
323
285 - hidden_states = self.language_model(input_ids,
286 - positions,
287 - kv_caches,
288 - attn_metadata,
289 - None,
290 - inputs_embeds=inputs_embeds)
324 + hidden_states = self.language_model.model(input_ids,
325 + positions,
326 + kv_caches,
327 + attn_metadata,
328 + None,
329 + inputs_embeds=inputs_embeds)
330
331 return hidden_states
332
333 def compute_logits(self, hidden_states: torch.Tensor,
334 sampling_metadata: SamplingMetadata) -> torch.Tensor:
296 - logits = self.logits_processor(self.lm_head, hidden_states,
297 - sampling_metadata)
298 - return logits
335 + return self.language_model.compute_logits(hidden_states,
336 + sampling_metadata)
337
338 def sample(
339 self,
340 logits: torch.Tensor,
341 sampling_metadata: SamplingMetadata,
342 ) -> Optional[SamplerOutput]:
305 - next_tokens = self.sampler(logits, sampling_metadata)
306 - return next_tokens
343 + return self.language_model.sample(logits, sampling_metadata)
344
345 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
309 - # only doing this for language model part for now.
310 - stacked_params_mapping = [
311 - # (param_name, shard_name, shard_id)
312 - ("qkv_proj", "q_proj", "q"),
313 - ("qkv_proj", "k_proj", "k"),
314 - ("qkv_proj", "v_proj", "v"),
315 - ("gate_up_proj", "gate_proj", 0),
316 - ("gate_up_proj", "up_proj", 1),
317 - ]
318 - params_dict = dict(self.named_parameters())
319 - for name, loaded_weight in weights:
320 - if "rotary_emb.inv_freq" in name:
321 - continue
322 - # post_layernorm is not needed in CLIPVisionModel
323 - if "vision_model.post_layernorm" in name:
324 - continue
325 - for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
326 - if key_to_modify in name:
327 - name = name.replace(key_to_modify, new_key)
328 - use_default_weight_loading = False
329 - if "vision" in name:
330 - if self.vision_tower is not None:
331 - # We only do sharding for language model and
332 - # not vision model for now.
333 - use_default_weight_loading = True
334 - else:
335 - for (param_name, weight_name,
336 - shard_id) in stacked_params_mapping:
337 - if weight_name not in name:
338 - continue
339 - param = params_dict[name.replace(weight_name, param_name)]
340 - weight_loader = param.weight_loader
341 - weight_loader(param, loaded_weight, shard_id)
342 - break
343 - else:
344 - use_default_weight_loading = True
345 - if use_default_weight_loading and name in params_dict:
346 - param = params_dict[name]
347 - weight_loader = getattr(param, "weight_loader",
348 - default_weight_loader)
349 - weight_loader(param, loaded_weight)
346 + # prepare weight iterators for components
347 + vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
348 +
349 + # load vision encoder
350 + vit_weights = filter_weights(vit_weights, "vision_tower")
351 + self.vision_tower.load_weights(vit_weights)
352 +
353 + # load mlp projector
354 + mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
355 + mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
356 + for name, loaded_weight in mlp_weights:
357 + param = mlp_params_dict[name]
358 + weight_loader = getattr(param, "weight_loader",
359 + default_weight_loader)
360 + weight_loader(param, loaded_weight)
361 +
362 + # load llm backbone
363 + llm_weights = filter_weights(llm_weights, "language_model")
364 + self.language_model.load_weights(llm_weights)
vllm/model_executor/models/llava_next.py
+135 -104
@@ -1,9 +1,10 @@
1 +import itertools
2 from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
3
4 import torch
5 import torch.nn as nn
6 from PIL import Image
6 -from transformers import CLIPVisionConfig, LlavaNextConfig
7 +from transformers import CLIPVisionConfig, LlavaNextConfig, SiglipVisionConfig
8 from transformers.models.llava_next.modeling_llava_next import (
9 get_anyres_image_grid_shape, unpad_image)
10 from typing_extensions import NotRequired
@@ -12,23 +13,23 @@ from vllm.attention import AttentionMetadata
13 from vllm.config import CacheConfig, MultiModalConfig
14 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15 from vllm.logger import init_logger
15 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 from vllm.model_executor.layers.quantization.base_config import (
17 QuantizationConfig)
18 -from vllm.model_executor.layers.sampler import Sampler
19 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 -from vllm.model_executor.models.clip import CLIPVisionModel
22 -from vllm.model_executor.models.llama import LlamaModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
21 from vllm.sequence import IntermediateTensors, SamplerOutput
22
27 -from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
23 +from .clip import (CLIPVisionModel, dummy_image_for_clip,
24 + dummy_seq_data_for_clip, get_clip_image_feature_size,
25 get_clip_patch_grid_length, input_processor_for_clip)
26 from .interfaces import SupportsVision
27 from .llava import LlavaMultiModalProjector
31 -from .utils import merge_vision_embeddings
28 +from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29 + dummy_seq_data_for_siglip, get_siglip_image_feature_size,
30 + get_siglip_patch_grid_length, input_processor_for_siglip)
31 +from .utils import (filter_weights, init_vllm_registered_model,
32 + merge_vision_embeddings)
33
34 logger = init_logger(__name__)
35
@@ -104,30 +105,42 @@ def get_llava_next_image_feature_size(
105 image_size=vision_config.image_size,
106 patch_size=vision_config.patch_size,
107 )
107 - base_feature_size = num_patches * num_patches
108 -
109 - num_patch_height, num_patch_width = get_anyres_image_grid_shape(
110 - image_size=(input_height, input_width),
111 - grid_pinpoints=hf_config.image_grid_pinpoints,
112 - patch_size=vision_config.image_size,
108 + base_feature_size = get_clip_image_feature_size(vision_config)
109 + elif isinstance(vision_config, SiglipVisionConfig):
110 + num_patches = get_siglip_patch_grid_length(
111 + image_size=vision_config.image_size,
112 + patch_size=vision_config.patch_size,
113 )
114 + base_feature_size = get_siglip_image_feature_size(vision_config)
115 + else:
116 + msg = f"Unsupported vision config: {type(vision_config)}"
117 + raise NotImplementedError(msg)
118 +
119 + strategy = hf_config.vision_feature_select_strategy
120 + if strategy == "default":
121 + base_feature_size -= 1
122 + elif strategy == "full":
123 + pass
124 + else:
125 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
126
115 - (
116 - unpadded_feature_size,
117 - newline_feature_size,
118 - ) = _get_llava_next_num_unpadded_features(input_height, input_width,
119 - num_patches,
120 - num_patch_height,
121 - num_patch_width)
127 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
128 + image_size=(input_height, input_width),
129 + grid_pinpoints=hf_config.image_grid_pinpoints,
130 + patch_size=vision_config.image_size,
131 + )
132
123 - return unpadded_feature_size + newline_feature_size + base_feature_size
133 + (
134 + unpadded_feature_size,
135 + newline_feature_size,
136 + ) = _get_llava_next_num_unpadded_features(input_height, input_width,
137 + num_patches, num_patch_height,
138 + num_patch_width)
139
125 - msg = f"Unsupported vision config: {type(vision_config)}"
126 - raise NotImplementedError(msg)
140 + return unpadded_feature_size + newline_feature_size + base_feature_size
141
142
143 def get_max_llava_next_image_tokens(ctx: InputContext):
130 -
144 return get_llava_next_image_feature_size(
145 ctx.get_hf_config(LlavaNextConfig),
146 input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
@@ -155,6 +168,21 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
168 image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
169 )
170
171 + return seq_data, mm_data
172 + elif isinstance(vision_config, SiglipVisionConfig):
173 + seq_data = dummy_seq_data_for_siglip(
174 + vision_config,
175 + seq_len,
176 + image_token_id=hf_config.image_token_index,
177 + image_feature_size_override=image_feature_size,
178 + )
179 +
180 + mm_data = dummy_image_for_siglip(
181 + vision_config,
182 + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
183 + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
184 + )
185 +
186 return seq_data, mm_data
187
188 msg = f"Unsupported vision config: {type(vision_config)}"
@@ -194,6 +222,40 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
222 image_token_id=hf_config.image_token_index,
223 image_feature_size_override=image_feature_size,
224 )
225 + elif isinstance(vision_config, SiglipVisionConfig):
226 + return input_processor_for_siglip(
227 + model_config,
228 + vision_config,
229 + llm_inputs,
230 + image_token_id=hf_config.image_token_index,
231 + image_feature_size_override=image_feature_size,
232 + )
233 +
234 + msg = f"Unsupported vision config: {type(vision_config)}"
235 + raise NotImplementedError(msg)
236 +
237 +
238 +def _init_vision_tower(hf_config: LlavaNextConfig):
239 + vision_config = hf_config.vision_config
240 +
241 + # Initialize the vision tower only up to the required feature layer
242 + vision_feature_layer = hf_config.vision_feature_layer
243 + if vision_feature_layer < 0:
244 + num_hidden_layers = hf_config.vision_config.num_hidden_layers \
245 + + vision_feature_layer + 1
246 + else:
247 + num_hidden_layers = vision_feature_layer + 1
248 +
249 + if isinstance(vision_config, CLIPVisionConfig):
250 + return CLIPVisionModel(
251 + vision_config,
252 + num_hidden_layers_override=num_hidden_layers,
253 + )
254 + elif isinstance(vision_config, SiglipVisionConfig):
255 + return SiglipVisionModel(
256 + vision_config,
257 + num_hidden_layers_override=num_hidden_layers,
258 + )
259
260 msg = f"Unsupported vision config: {type(vision_config)}"
261 raise NotImplementedError(msg)
@@ -215,36 +277,15 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
277 self.config = config
278 self.multimodal_config = multimodal_config
279
218 - # Initialize the vision tower only up to the required feature layer
219 - vision_feature_layer = config.vision_feature_layer
220 - if vision_feature_layer < 0:
221 - num_hidden_layers = config.vision_config.num_hidden_layers \
222 - + vision_feature_layer + 1
223 - else:
224 - num_hidden_layers = vision_feature_layer + 1
225 -
280 # TODO: Optionally initializes this for supporting embeddings.
227 - self.vision_tower = CLIPVisionModel(
228 - config.vision_config, num_hidden_layers_override=num_hidden_layers)
281 + self.vision_tower = _init_vision_tower(config)
282 self.multi_modal_projector = LlavaMultiModalProjector(
283 vision_hidden_size=config.vision_config.hidden_size,
284 text_hidden_size=config.text_config.hidden_size,
285 projector_hidden_act=config.projector_hidden_act)
286
234 - self.quant_config = quant_config
235 - self.language_model = LlamaModel(config.text_config, cache_config,
236 - quant_config)
237 - self.unpadded_vocab_size = config.text_config.vocab_size
238 - self.lm_head = ParallelLMHead(
239 - self.unpadded_vocab_size,
240 - config.text_config.hidden_size,
241 - org_num_embeddings=self.language_model.org_vocab_size,
242 - quant_config=quant_config)
243 - logit_scale = getattr(config, "logit_scale", 1.0)
244 - self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
245 - config.text_config.vocab_size,
246 - logit_scale)
247 - self.sampler = Sampler()
287 + self.language_model = init_vllm_registered_model(
288 + config.text_config, cache_config, quant_config)
289
290 self.image_newline = nn.Parameter(
291 torch.empty(config.text_config.hidden_size))
@@ -310,8 +351,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
351
352 raise ValueError(f"Unexpected select feature strategy: {strategy}")
353
313 - def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
314 - pixel_values: torch.Tensor) -> torch.Tensor:
354 + def _image_pixels_to_features(
355 + self,
356 + vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
357 + pixel_values: torch.Tensor,
358 + ) -> torch.Tensor:
359
360 # NOTE: we skip the step to select the vision feature layer since
361 # this is already done inside the vision tower
@@ -496,7 +540,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
540
541 if image_input is not None:
542 vision_embeddings = self._process_image_input(image_input)
499 - inputs_embeds = self.language_model.get_input_embeddings(input_ids)
543 + inputs_embeds = self.language_model.model.get_input_embeddings(
544 + input_ids)
545
546 inputs_embeds = merge_vision_embeddings(
547 input_ids, inputs_embeds, vision_embeddings,
@@ -506,68 +551,54 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
551 else:
552 inputs_embeds = None
553
509 - hidden_states = self.language_model(input_ids,
510 - positions,
511 - kv_caches,
512 - attn_metadata,
513 - None,
514 - inputs_embeds=inputs_embeds)
554 + hidden_states = self.language_model.model(input_ids,
555 + positions,
556 + kv_caches,
557 + attn_metadata,
558 + None,
559 + inputs_embeds=inputs_embeds)
560
561 return hidden_states
562
563 def compute_logits(self, hidden_states: torch.Tensor,
564 sampling_metadata: SamplingMetadata) -> torch.Tensor:
520 - logits = self.logits_processor(self.lm_head, hidden_states,
521 - sampling_metadata)
522 - return logits
565 + return self.language_model.compute_logits(hidden_states,
566 + sampling_metadata)
567
568 def sample(
569 self,
570 logits: torch.Tensor,
571 sampling_metadata: SamplingMetadata,
572 ) -> Optional[SamplerOutput]:
529 - next_tokens = self.sampler(logits, sampling_metadata)
530 - return next_tokens
573 + return self.language_model.sample(logits, sampling_metadata)
574
575 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
533 - # only doing this for language model part for now.
534 - stacked_params_mapping = [
535 - # (param_name, shard_name, shard_id)
536 - ("qkv_proj", "q_proj", "q"),
537 - ("qkv_proj", "k_proj", "k"),
538 - ("qkv_proj", "v_proj", "v"),
539 - ("gate_up_proj", "gate_proj", 0),
540 - ("gate_up_proj", "up_proj", 1),
541 - ]
542 - params_dict = dict(self.named_parameters())
543 - for name, loaded_weight in weights:
544 - if "rotary_emb.inv_freq" in name:
545 - continue
546 - # post_layernorm is not needed in CLIPVisionModel
547 - if "vision_model.post_layernorm" in name:
548 - continue
549 - for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
550 - if key_to_modify in name:
551 - name = name.replace(key_to_modify, new_key)
552 - use_default_weight_loading = False
553 - if "vision" in name:
554 - if self.vision_tower is not None:
555 - # We only do sharding for language model and
556 - # not vision model for now.
557 - use_default_weight_loading = True
558 - else:
559 - for (param_name, weight_name,
560 - shard_id) in stacked_params_mapping:
561 - if weight_name not in name:
562 - continue
563 - param = params_dict[name.replace(weight_name, param_name)]
564 - weight_loader = param.weight_loader
565 - weight_loader(param, loaded_weight, shard_id)
566 - break
567 - else:
568 - use_default_weight_loading = True
569 - if use_default_weight_loading and name in params_dict:
570 - param = params_dict[name]
571 - weight_loader = getattr(param, "weight_loader",
572 - default_weight_loader)
573 - weight_loader(param, loaded_weight)
576 + # prepare weight iterators for components
577 + vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
578 + weights, 4)
579 +
580 + # load vision encoder
581 + vit_weights = filter_weights(vit_weights, "vision_tower")
582 + self.vision_tower.load_weights(vit_weights)
583 +
584 + # load mlp projector
585 + mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
586 + mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
587 + for name, loaded_weight in mlp_weights:
588 + param = mlp_params_dict[name]
589 + weight_loader = getattr(param, "weight_loader",
590 + default_weight_loader)
591 + weight_loader(param, loaded_weight)
592 +
593 + # load newline
594 + newline_weights = filter_weights(newline_weights, "image_newline")
595 + for name, loaded_weight in newline_weights:
596 + assert name == ""
597 + param = self.image_newline
598 + weight_loader = getattr(param, "weight_loader",
599 + default_weight_loader)
600 + weight_loader(param, loaded_weight)
601 +
602 + # load llm backbone
603 + llm_weights = filter_weights(llm_weights, "language_model")
604 + self.language_model.load_weights(llm_weights)
vllm/model_executor/models/siglip.py
+38 -11
@@ -2,12 +2,12 @@
2 within a vision language model."""
3
4 import math
5 -from typing import Optional, Tuple
5 +from typing import Iterable, Optional, Tuple
6
7 import torch
8 from PIL import Image
9 from torch import nn
10 -from transformers import SiglipConfig, SiglipVisionConfig
10 +from transformers import SiglipVisionConfig
11 from transformers.models.siglip.modeling_siglip import SiglipAttention
12 from vllm_flash_attn import flash_attn_func
13 from xformers.ops import memory_efficient_attention
@@ -22,13 +22,15 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 VocabParallelEmbedding)
25 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26 from vllm.multimodal.image import (cached_get_tokenizer,
27 repeat_and_pad_image_tokens)
28 from vllm.sequence import SequenceData
29
30
31 def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
31 - assert image_size % patch_size == 0
32 + # Since interpolation is applied, the image size need not be divisible
33 + # assert image_size % patch_size == 0
34 return image_size // patch_size
35
36
@@ -454,7 +456,7 @@ class SiglipEncoderLayer(nn.Module):
456
457 def __init__(
458 self,
457 - config: SiglipConfig,
459 + config: SiglipVisionConfig,
460 quant_config: Optional[QuantizationConfig] = None,
461 ):
462 super().__init__()
@@ -474,7 +476,7 @@ class SiglipEncoderLayer(nn.Module):
476 def forward(
477 self,
478 hidden_states: torch.Tensor,
477 - ) -> Tuple[torch.Tensor]:
479 + ) -> Tuple[torch.Tensor, None]:
480 residual = hidden_states
481
482 hidden_states = self.layer_norm1(hidden_states)
@@ -493,22 +495,27 @@ class SiglipEncoder(nn.Module):
495
496 def __init__(
497 self,
496 - config: SiglipConfig,
498 + config: SiglipVisionConfig,
499 quant_config: Optional[QuantizationConfig] = None,
500 + num_hidden_layers_override: Optional[int] = None,
501 ):
502 super().__init__()
503 self.config = config
504 +
505 + if num_hidden_layers_override is None:
506 + num_hidden_layers = config.num_hidden_layers
507 + else:
508 + num_hidden_layers = num_hidden_layers_override
509 +
510 self.layers = nn.ModuleList([
502 - SiglipEncoderLayer(
503 - config,
504 - quant_config=quant_config,
505 - ) for _ in range(config.num_hidden_layers)
511 + SiglipEncoderLayer(config, quant_config=quant_config)
512 + for _ in range(num_hidden_layers)
513 ])
514
515 def forward(
516 self,
517 inputs_embeds: torch.Tensor,
511 - ) -> Tuple:
518 + ) -> torch.Tensor:
519 hidden_states = inputs_embeds
520 for encoder_layer in self.layers:
521 hidden_states, _ = encoder_layer(hidden_states)
@@ -553,6 +560,7 @@ class SiglipVisionTransformer(nn.Module):
560 self,
561 config: SiglipVisionConfig,
562 quant_config: Optional[QuantizationConfig] = None,
563 + num_hidden_layers_override: Optional[int] = None,
564 ):
565 super().__init__()
566 self.config = config
@@ -562,6 +570,7 @@ class SiglipVisionTransformer(nn.Module):
570 self.encoder = SiglipEncoder(
571 config,
572 quant_config=quant_config,
573 + num_hidden_layers_override=num_hidden_layers_override,
574 )
575 self.post_layernorm = nn.LayerNorm(embed_dim,
576 eps=config.layer_norm_eps)
@@ -600,11 +609,13 @@ class SiglipVisionModel(nn.Module):
609 self,
610 config: SiglipVisionConfig,
611 quant_config: Optional[QuantizationConfig] = None,
612 + num_hidden_layers_override: Optional[int] = None,
613 ):
614 super().__init__()
615 self.vision_model = SiglipVisionTransformer(
616 config,
617 quant_config,
618 + num_hidden_layers_override=num_hidden_layers_override,
619 )
620
621 def get_input_embeddings(self) -> nn.Module:
@@ -619,3 +630,19 @@ class SiglipVisionModel(nn.Module):
630 pixel_values=pixel_values,
631 interpolate_pos_encoding=interpolate_pos_encoding,
632 )
633 +
634 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
635 + params_dict = dict(self.named_parameters())
636 + layer_count = len(self.vision_model.encoder.layers)
637 +
638 + for name, loaded_weight in weights:
639 + # omit layers when num_hidden_layers_override is set
640 + if "vision_model.encoder.layers." in name:
641 + layer_idx = int(name.split(".")[3])
642 + if layer_idx >= layer_count:
643 + continue
644 +
645 + param = params_dict[name]
646 + weight_loader = getattr(param, "weight_loader",
647 + default_weight_loader)
648 + weight_loader(param, loaded_weight)
vllm/model_executor/models/utils.py
+52 -4
@@ -1,22 +1,70 @@
1 -from typing import Dict, List, Protocol, Tuple
1 +from typing import Dict, Iterable, List, Optional, Protocol, Tuple
2
3 import torch
4 +import torch.nn as nn
5 from torch.func import functional_call
6 +from transformers import PretrainedConfig
7
8 +from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
9 + SchedulerConfig)
10 +from vllm.model_executor.layers.quantization import QuantizationConfig
11 +from vllm.model_executor.model_loader.loader import build_model
12 +from vllm.model_executor.models import ModelRegistry
13 from vllm.multimodal import BatchedTensors
14 from vllm.utils import is_pin_memory_available
15
16
17 +def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]], prefix: str):
18 + """
19 + Helper function to load weights for inner vLLM models.
20 +
21 + See also:
22 + :ref:`init_vllm_registered_model`
23 + """
24 + for name, loaded_weight in weights:
25 + name = name.split(".")
26 + if prefix == name.pop(0):
27 + name = ".".join(name)
28 + yield name, loaded_weight
29 +
30 +
31 +def init_vllm_registered_model(
32 + hf_config: PretrainedConfig,
33 + cache_config: Optional[CacheConfig],
34 + quant_config: Optional[QuantizationConfig],
35 + *,
36 + lora_config: Optional[LoRAConfig] = None,
37 + multimodal_config: Optional[MultiModalConfig] = None,
38 + scheduler_config: Optional[SchedulerConfig] = None,
39 +) -> nn.Module:
40 + """
41 + Helper function to initialize an inner model registered to vLLM,
42 + based on the arguments passed to the outer vLLM model.
43 + """
44 + model_class, _ = ModelRegistry.resolve_model_cls(hf_config.architectures)
45 +
46 + return build_model(
47 + model_class,
48 + hf_config,
49 + cache_config,
50 + quant_config,
51 + lora_config=lora_config,
52 + multimodal_config=multimodal_config,
53 + scheduler_config=scheduler_config,
54 + )
55 +
56 +
57 def merge_vision_embeddings(input_ids: torch.Tensor,
58 inputs_embeds: torch.Tensor,
59 vision_embeddings: BatchedTensors,
60 image_token_id: int) -> torch.Tensor:
61 """
15 - Merge `vision_embeddings` into `inputs_embeds` by overwriting the positions
16 - in `inputs_embeds` corresponding to placeholder image tokens in `input_ids`.
62 + Merge ``vision_embeddings`` into ``inputs_embeds`` by overwriting the
63 + positions in ``inputs_embeds`` corresponding to placeholder image tokens in
64 + ``input_ids``.
65
66 Note:
19 - This updates `inputs_embeds` in place.
67 + This updates ``inputs_embeds`` in place.
68 """
69 mask = (input_ids == image_token_id)
70 num_expected_tokens = mask.sum()