A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] Support SigLIP encoder and alternative decoders for LLaVA models (#7153)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Cyrus Leung committed
Aug 6, 2024 at 16:55 UTC
1f26efbb3a5e6dad0b98421dd697167c42a50629
14 files changed
+455
-269
requirements-test.txt
+3
@@ -20,6 +20,9 @@ sentence-transformers # required for embedding
20
compressed-tensors==0.4.0 # required for compressed-tensors
21
timm # required for internvl test
22
23
+# TODO: Add this after fully implementing llava(mantis)
24
+# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
25
+
26
# Benchmarking
27
aiohttp
28
tests/models/test_llava.py
+31
-5
@@ -1,10 +1,11 @@
1
from typing import List, Optional, Tuple, Type
2
3
import pytest
4
-from transformers import AutoTokenizer
4
+from transformers import AutoConfig, AutoTokenizer
5
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
8
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
9
10
from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
11
from .utils import check_logprobs_close
@@ -18,9 +19,11 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19
"USER: <image>\nWhat is the season?\nASSISTANT:",
20
})
21
21
-IMAGE_TOKEN_ID = 32000
22
-
23
-models = ["llava-hf/llava-1.5-7b-hf"]
22
+models = [
23
+ "llava-hf/llava-1.5-7b-hf",
24
+ # TODO: Get this model to produce meaningful output in vLLM
25
+ # "TIGER-Lab/Mantis-8B-siglip-llama3",
26
+]
27
28
29
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -29,12 +32,15 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
32
"""Sanitize vllm output to be comparable with hf output."""
33
output_ids, output_str, out_logprobs = vllm_output
34
35
+ config = AutoConfig.from_pretrained(model)
36
+ image_token_id = config.image_token_index
37
+
38
tokenizer = AutoTokenizer.from_pretrained(model)
39
eos_token_id = tokenizer.eos_token_id
40
41
hf_output_ids = [
42
token_id for idx, token_id in enumerate(output_ids)
37
- if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
43
+ if token_id != image_token_id or output_ids[idx - 1] != image_token_id
44
]
45
46
assert output_str[0] == " "
@@ -67,6 +73,17 @@ def run_test(
73
Note, the text input is also adjusted to abide by vllm contract.
74
The text output is sanitized to be able to compare with hf.
75
"""
76
+ # NOTE: For local use; this isn't tested in CI yet (see TODO above)
77
+ if model.startswith("TIGER-Lab/Mantis"):
78
+ from mantis.models.mllava import MLlavaProcessor
79
+
80
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
81
+ mantis_processor = MLlavaProcessor.from_pretrained(
82
+ model, torch_dtype=torch_dtype)
83
+ assert isinstance(mantis_processor, MLlavaProcessor)
84
+ else:
85
+ mantis_processor = None
86
+
87
images = [asset.pil_image for asset in image_assets]
88
89
inputs_per_image = [(
@@ -94,6 +111,15 @@ def run_test(
111
]
112
113
with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
114
+ if mantis_processor is not None:
115
+
116
+ def process(*args, **kwargs):
117
+ output = mantis_processor(*args, **kwargs)
118
+ output["pixel_values"] = output["pixel_values"].to(torch_dtype)
119
+ return output
120
+
121
+ hf_model.processor = process
122
+
123
hf_outputs_per_image = [
124
hf_model.generate_greedy_logprobs_limit(prompts,
125
max_tokens,
tests/models/test_llava_next.py
+5
-4
@@ -1,7 +1,7 @@
1
from typing import List, Optional, Tuple, Type, overload
2
3
import pytest
4
-from transformers import AutoTokenizer
4
+from transformers import AutoConfig, AutoTokenizer
5
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
@@ -23,8 +23,6 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
23
f"{_PREFACE} USER: <image>\nWhat is the season? ASSISTANT:",
24
})
25
26
-IMAGE_TOKEN_ID = 32000
27
-
26
models = ["llava-hf/llava-v1.6-vicuna-7b-hf"]
27
28
@@ -34,12 +32,15 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
32
"""Sanitize vllm output to be comparable with hf output."""
33
output_ids, output_str, out_logprobs = vllm_output
34
35
+ config = AutoConfig.from_pretrained(model)
36
+ image_token_id = config.image_token_index
37
+
38
tokenizer = AutoTokenizer.from_pretrained(model)
39
eos_token_id = tokenizer.eos_token_id
40
41
hf_output_ids = [
42
token_id for idx, token_id in enumerate(output_ids)
42
- if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
43
+ if token_id != image_token_id or output_ids[idx - 1] != image_token_id
44
]
45
46
assert output_str[0] == " "
tests/models/test_paligemma.py
+5
-4
@@ -2,7 +2,7 @@ import os
2
from typing import List, Optional, Tuple, Type
3
4
import pytest
5
-from transformers import AutoTokenizer
5
+from transformers import AutoConfig, AutoTokenizer
6
7
from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
@@ -20,8 +20,6 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20
"What is in the picture?",
21
})
22
23
-IMAGE_TOKEN_ID = 257152
24
-
23
models = ["google/paligemma-3b-mix-224"]
24
25
# ROCm Triton FA can run into compilation issues with these models due to,
@@ -37,12 +35,15 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
35
"""Sanitize vllm output to be comparable with hf output."""
36
output_ids, output_str, out_logprobs = vllm_output
37
38
+ config = AutoConfig.from_pretrained(model)
39
+ image_token_id = config.image_token_index
40
+
41
tokenizer = AutoTokenizer.from_pretrained(model)
42
eos_token_id = tokenizer.eos_token_id
43
44
hf_output_ids = [
45
token_id for idx, token_id in enumerate(output_ids)
45
- if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
46
+ if token_id != image_token_id or output_ids[idx - 1] != image_token_id
47
]
48
49
hf_output_str = output_str
tests/models/test_registry.py
+1
-1
@@ -6,4 +6,4 @@ from vllm.model_executor.models import _MODELS, ModelRegistry
6
@pytest.mark.parametrize("model_cls", _MODELS)
7
def test_registry_imports(model_cls):
8
# Ensure all model classes can be imported successfully
9
- ModelRegistry.load_model_cls(model_cls)
9
+ ModelRegistry.resolve_model_cls([model_cls])
vllm/model_executor/model_loader/loader.py
+28
-10
@@ -16,7 +16,7 @@ import numpy as np
16
import torch
17
from huggingface_hub import HfApi, hf_hub_download
18
from torch import nn
19
-from transformers import AutoModelForCausalLM
19
+from transformers import AutoModelForCausalLM, PretrainedConfig
20
21
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
22
LoRAConfig, ModelConfig, MultiModalConfig,
@@ -143,6 +143,22 @@ def _get_model_initialization_kwargs(
143
return extra_kwargs
144
145
146
+def build_model(model_class: Type[nn.Module], hf_config: PretrainedConfig,
147
+ cache_config: Optional[CacheConfig],
148
+ quant_config: Optional[QuantizationConfig], *,
149
+ lora_config: Optional[LoRAConfig],
150
+ multimodal_config: Optional[MultiModalConfig],
151
+ scheduler_config: Optional[SchedulerConfig]) -> nn.Module:
152
+ extra_kwargs = _get_model_initialization_kwargs(model_class, lora_config,
153
+ multimodal_config,
154
+ scheduler_config)
155
+
156
+ return model_class(config=hf_config,
157
+ cache_config=cache_config,
158
+ quant_config=quant_config,
159
+ **extra_kwargs)
160
+
161
+
162
def _initialize_model(
163
model_config: ModelConfig,
164
load_config: LoadConfig,
@@ -151,15 +167,17 @@ def _initialize_model(
167
cache_config: CacheConfig,
168
scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
169
"""Initialize a model with the given configurations."""
154
- model_class = get_model_architecture(model_config)[0]
155
- quant_config = _get_quantization_config(model_config, load_config)
156
-
157
- return model_class(config=model_config.hf_config,
158
- cache_config=cache_config,
159
- quant_config=quant_config,
160
- **_get_model_initialization_kwargs(
161
- model_class, lora_config, multimodal_config,
162
- scheduler_config))
170
+ model_class, _ = get_model_architecture(model_config)
171
+
172
+ return build_model(
173
+ model_class,
174
+ model_config.hf_config,
175
+ quant_config=_get_quantization_config(model_config, load_config),
176
+ lora_config=lora_config,
177
+ multimodal_config=multimodal_config,
178
+ cache_config=cache_config,
179
+ scheduler_config=scheduler_config,
180
+ )
181
182
183
class BaseModelLoader(ABC):
vllm/model_executor/model_loader/utils.py
+1
-7
@@ -28,13 +28,7 @@ def get_model_architecture(
28
and "MixtralForCausalLM" in architectures):
29
architectures = ["QuantMixtralForCausalLM"]
30
31
- for arch in architectures:
32
- model_cls = ModelRegistry.load_model_cls(arch)
33
- if model_cls is not None:
34
- return (model_cls, arch)
35
- raise ValueError(
36
- f"Model architectures {architectures} are not supported for now. "
37
- f"Supported architectures: {ModelRegistry.get_supported_archs()}")
31
+ return ModelRegistry.resolve_model_cls(architectures)
32
33
34
def get_architecture_class_name(model_config: ModelConfig) -> str:
vllm/model_executor/models/__init__.py
+14
-2
@@ -1,6 +1,6 @@
1
import functools
2
import importlib
3
-from typing import Dict, List, Optional, Type
3
+from typing import Dict, List, Optional, Tuple, Type
4
5
import torch.nn as nn
6
@@ -126,7 +126,7 @@ class ModelRegistry:
126
return getattr(module, model_cls_name, None)
127
128
@staticmethod
129
- def load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
129
+ def _try_load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
130
if model_arch in _OOT_MODELS:
131
return _OOT_MODELS[model_arch]
132
if model_arch not in _MODELS:
@@ -143,6 +143,18 @@ class ModelRegistry:
143
144
return ModelRegistry._get_model(model_arch)
145
146
+ @staticmethod
147
+ def resolve_model_cls(
148
+ architectures: List[str]) -> Tuple[Type[nn.Module], str]:
149
+ for arch in architectures:
150
+ model_cls = ModelRegistry._try_load_model_cls(arch)
151
+ if model_cls is not None:
152
+ return (model_cls, arch)
153
+
154
+ raise ValueError(
155
+ f"Model architectures {architectures} are not supported for now. "
156
+ f"Supported architectures: {ModelRegistry.get_supported_archs()}")
157
+
158
@staticmethod
159
def get_supported_archs() -> List[str]:
160
return list(_MODELS.keys())
vllm/model_executor/models/clip.py
+22
-2
@@ -1,6 +1,6 @@
1
"""Minimal implementation of CLIPVisionModel intended to be only used
2
within a vision language model."""
3
-from typing import Optional
3
+from typing import Iterable, Optional, Tuple
4
5
import torch
6
import torch.nn as nn
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
15
RowParallelLinear)
16
from vllm.model_executor.layers.quantization import QuantizationConfig
17
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
from vllm.multimodal.image import (cached_get_tokenizer,
19
repeat_and_pad_image_tokens)
20
from vllm.sequence import SequenceData
@@ -32,7 +33,7 @@ def get_clip_num_patches(*, image_size: int, patch_size: int) -> int:
33
34
def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
35
return get_clip_num_patches(image_size=hf_config.image_size,
35
- patch_size=hf_config.patch_size)
36
+ patch_size=hf_config.patch_size) + 1
37
38
39
def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
@@ -291,3 +292,22 @@ class CLIPVisionModel(nn.Module):
292
@property
293
def device(self):
294
return next(self.parameters()).device
295
+
296
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
297
+ params_dict = dict(self.named_parameters())
298
+ layer_count = len(self.vision_model.encoder.layers)
299
+
300
+ for name, loaded_weight in weights:
301
+ # post_layernorm is not needed in CLIPVisionModel
302
+ if "vision_model.post_layernorm" in name:
303
+ continue
304
+ # omit layers when num_hidden_layers_override is set
305
+ if "vision_model.encoder.layers." in name:
306
+ layer_idx = int(name.split(".")[3])
307
+ if layer_idx >= layer_count:
308
+ continue
309
+
310
+ param = params_dict[name]
311
+ weight_loader = getattr(param, "weight_loader",
312
+ default_weight_loader)
313
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/internvl.py
+7
-17
@@ -18,7 +18,6 @@ from vllm.config import CacheConfig, MultiModalConfig
18
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
19
from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
-from vllm.model_executor.models import ModelRegistry
21
from vllm.model_executor.models.intern_vit import InternVisionModel
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -29,7 +28,8 @@ from vllm.sequence import IntermediateTensors, SamplerOutput
28
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
29
get_clip_num_patches)
30
from .interfaces import SupportsVision
32
-from .utils import merge_vision_embeddings
31
+from .utils import (filter_weights, init_vllm_registered_model,
32
+ merge_vision_embeddings)
33
34
IMG_START = '<img>'
35
IMG_END = '</img>'
@@ -283,10 +283,8 @@ class InternVLChatModel(nn.Module, SupportsVision):
283
self.vision_model = InternVisionModel(
284
config.vision_config, num_hidden_layers_override=num_hidden_layers)
285
286
- llm_class = ModelRegistry.load_model_cls(
287
- config.text_config.architectures[0])
288
- self.language_model = llm_class(config.text_config, cache_config,
289
- quant_config)
286
+ self.language_model = init_vllm_registered_model(
287
+ config.text_config, cache_config, quant_config)
288
289
vit_hidden_size = config.vision_config.hidden_size
290
llm_hidden_size = config.text_config.hidden_size
@@ -415,24 +413,16 @@ class InternVLChatModel(nn.Module, SupportsVision):
413
) -> Optional[SamplerOutput]:
414
return self.language_model.sample(logits, sampling_metadata)
415
418
- def _filter_weights(self, weights: Iterable[Tuple[str, torch.Tensor]],
419
- prefix: str):
420
- for name, loaded_weight in weights:
421
- name = name.split(".")
422
- if prefix == name.pop(0):
423
- name = ".".join(name)
424
- yield name, loaded_weight
425
-
416
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
417
# prepare weight iterators for components
418
vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
419
420
# load vision encoder
431
- vit_weights = self._filter_weights(vit_weights, "vision_model")
421
+ vit_weights = filter_weights(vit_weights, "vision_model")
422
self.vision_model.load_weights(vit_weights)
423
424
# load mlp projector
435
- mlp_weights = self._filter_weights(mlp_weights, "mlp1")
425
+ mlp_weights = filter_weights(mlp_weights, "mlp1")
426
mlp_params_dict = dict(self.mlp1.named_parameters())
427
for name, loaded_weight in mlp_weights:
428
param = mlp_params_dict[name]
@@ -441,5 +431,5 @@ class InternVLChatModel(nn.Module, SupportsVision):
431
weight_loader(param, loaded_weight)
432
433
# load llm backbone
444
- llm_weights = self._filter_weights(llm_weights, "language_model")
434
+ llm_weights = filter_weights(llm_weights, "language_model")
435
self.language_model.load_weights(llm_weights)
vllm/model_executor/models/llava.py
+113
-98
@@ -1,34 +1,30 @@
1
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1
+import itertools
2
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
3
4
import torch
5
import torch.nn as nn
5
-from transformers import CLIPVisionConfig, LlavaConfig
6
+from transformers import CLIPVisionConfig, LlavaConfig, SiglipVisionConfig
7
8
from vllm.attention import AttentionMetadata
9
from vllm.config import CacheConfig, MultiModalConfig
10
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11
from vllm.model_executor.layers.activation import get_act_fn
11
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
-from vllm.model_executor.layers.sampler import Sampler
15
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
14
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
-from vllm.model_executor.models.clip import CLIPVisionModel
18
-from vllm.model_executor.models.llama import LlamaModel
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
16
from vllm.multimodal import MULTIMODAL_REGISTRY
17
from vllm.sequence import IntermediateTensors, SamplerOutput
18
23
-from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
24
- get_max_clip_image_tokens, input_processor_for_clip)
19
+from .clip import (CLIPVisionModel, dummy_image_for_clip,
20
+ dummy_seq_data_for_clip, get_max_clip_image_tokens,
21
+ input_processor_for_clip)
22
from .interfaces import SupportsVision
26
-from .utils import merge_vision_embeddings
27
-
28
-_KEYS_TO_MODIFY_MAPPING = {
29
- "language_model.lm_head": "lm_head",
30
- "language_model.model": "language_model",
31
-}
23
+from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24
+ dummy_seq_data_for_siglip, get_max_siglip_image_tokens,
25
+ input_processor_for_siglip)
26
+from .utils import (filter_weights, init_vllm_registered_model,
27
+ merge_vision_embeddings)
28
29
30
# TODO(xwjiang): Run benchmark and decide if TP.
@@ -67,25 +63,48 @@ def get_max_llava_image_tokens(ctx: InputContext):
63
vision_config = hf_config.vision_config
64
65
if isinstance(vision_config, CLIPVisionConfig):
70
- return get_max_clip_image_tokens(vision_config)
71
-
72
- msg = f"Unsupported vision config: {type(vision_config)}"
73
- raise NotImplementedError(msg)
66
+ num_image_tokens = get_max_clip_image_tokens(vision_config)
67
+ elif isinstance(vision_config, SiglipVisionConfig):
68
+ num_image_tokens = get_max_siglip_image_tokens(vision_config)
69
+ else:
70
+ msg = f"Unsupported vision config: {type(vision_config)}"
71
+ raise NotImplementedError(msg)
72
+
73
+ strategy = hf_config.vision_feature_select_strategy
74
+ if strategy == "default":
75
+ return num_image_tokens - 1
76
+ elif strategy == "full":
77
+ return num_image_tokens
78
+ else:
79
+ raise ValueError(f"Unexpected select feature strategy: {strategy}")
80
81
82
def dummy_data_for_llava(ctx: InputContext, seq_len: int):
83
hf_config = ctx.get_hf_config(LlavaConfig)
84
vision_config = hf_config.vision_config
85
86
+ image_feature_size = get_max_llava_image_tokens(ctx)
87
+
88
if isinstance(vision_config, CLIPVisionConfig):
89
seq_data = dummy_seq_data_for_clip(
90
vision_config,
91
seq_len,
92
image_token_id=hf_config.image_token_index,
93
+ image_feature_size_override=image_feature_size,
94
)
95
96
mm_data = dummy_image_for_clip(vision_config)
97
return seq_data, mm_data
98
+ elif isinstance(vision_config, SiglipVisionConfig):
99
+ seq_data = dummy_seq_data_for_siglip(
100
+ vision_config,
101
+ seq_len,
102
+ image_token_id=hf_config.image_token_index,
103
+ image_feature_size_override=image_feature_size,
104
+ )
105
+
106
+ mm_data = dummy_image_for_siglip(vision_config)
107
+ return seq_data, mm_data
108
109
msg = f"Unsupported vision config: {type(vision_config)}"
110
raise NotImplementedError(msg)
@@ -100,12 +119,49 @@ def input_processor_for_llava(ctx: InputContext, llm_inputs: LLMInputs):
119
hf_config = ctx.get_hf_config(LlavaConfig)
120
vision_config = hf_config.vision_config
121
122
+ image_feature_size = get_max_llava_image_tokens(ctx)
123
+
124
if isinstance(vision_config, CLIPVisionConfig):
125
return input_processor_for_clip(
126
model_config,
127
vision_config,
128
llm_inputs,
129
image_token_id=hf_config.image_token_index,
130
+ image_feature_size_override=image_feature_size,
131
+ )
132
+ elif isinstance(vision_config, SiglipVisionConfig):
133
+ return input_processor_for_siglip(
134
+ model_config,
135
+ vision_config,
136
+ llm_inputs,
137
+ image_token_id=hf_config.image_token_index,
138
+ image_feature_size_override=image_feature_size,
139
+ )
140
+
141
+ msg = f"Unsupported vision config: {type(vision_config)}"
142
+ raise NotImplementedError(msg)
143
+
144
+
145
+def _init_vision_tower(hf_config: LlavaConfig):
146
+ vision_config = hf_config.vision_config
147
+
148
+ # Initialize the vision tower only up to the required feature layer
149
+ vision_feature_layer = hf_config.vision_feature_layer
150
+ if vision_feature_layer < 0:
151
+ num_hidden_layers = hf_config.vision_config.num_hidden_layers \
152
+ + vision_feature_layer + 1
153
+ else:
154
+ num_hidden_layers = vision_feature_layer + 1
155
+
156
+ if isinstance(vision_config, CLIPVisionConfig):
157
+ return CLIPVisionModel(
158
+ vision_config,
159
+ num_hidden_layers_override=num_hidden_layers,
160
+ )
161
+ elif isinstance(vision_config, SiglipVisionConfig):
162
+ return SiglipVisionModel(
163
+ vision_config,
164
+ num_hidden_layers_override=num_hidden_layers,
165
)
166
167
msg = f"Unsupported vision config: {type(vision_config)}"
@@ -128,36 +184,15 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
184
self.config = config
185
self.multimodal_config = multimodal_config
186
131
- # Initialize the vision tower only up to the required feature layer
132
- vision_feature_layer = config.vision_feature_layer
133
- if vision_feature_layer < 0:
134
- num_hidden_layers = config.vision_config.num_hidden_layers \
135
- + vision_feature_layer + 1
136
- else:
137
- num_hidden_layers = vision_feature_layer + 1
138
-
187
# TODO: Optionally initializes this for supporting embeddings.
140
- self.vision_tower = CLIPVisionModel(
141
- config.vision_config, num_hidden_layers_override=num_hidden_layers)
188
+ self.vision_tower = _init_vision_tower(config)
189
self.multi_modal_projector = LlavaMultiModalProjector(
190
vision_hidden_size=config.vision_config.hidden_size,
191
text_hidden_size=config.text_config.hidden_size,
192
projector_hidden_act=config.projector_hidden_act)
193
147
- self.quant_config = quant_config
148
- self.language_model = LlamaModel(config.text_config, cache_config,
149
- quant_config)
150
- self.unpadded_vocab_size = config.text_config.vocab_size
151
- self.lm_head = ParallelLMHead(
152
- self.unpadded_vocab_size,
153
- config.text_config.hidden_size,
154
- org_num_embeddings=self.language_model.org_vocab_size,
155
- quant_config=quant_config)
156
- logit_scale = getattr(config, "logit_scale", 1.0)
157
- self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
158
- config.text_config.vocab_size,
159
- logit_scale)
160
- self.sampler = Sampler()
194
+ self.language_model = init_vllm_registered_model(
195
+ config.text_config, cache_config, quant_config)
196
197
def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
198
h = w = self.config.vision_config.image_size
@@ -198,8 +233,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
233
234
raise ValueError(f"Unexpected select feature strategy: {strategy}")
235
201
- def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
202
- pixel_values: torch.Tensor) -> torch.Tensor:
236
+ def _image_pixels_to_features(
237
+ self,
238
+ vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
239
+ pixel_values: torch.Tensor,
240
+ ) -> torch.Tensor:
241
242
# NOTE: we skip the step to select the vision feature layer since
243
# this is already done inside the vision tower
@@ -272,7 +310,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
310
311
if image_input is not None:
312
vision_embeddings = self._process_image_input(image_input)
275
- inputs_embeds = self.language_model.get_input_embeddings(input_ids)
313
+ inputs_embeds = self.language_model.model.get_input_embeddings(
314
+ input_ids)
315
316
inputs_embeds = merge_vision_embeddings(
317
input_ids, inputs_embeds, vision_embeddings,
@@ -282,68 +321,44 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
321
else:
322
inputs_embeds = None
323
285
- hidden_states = self.language_model(input_ids,
286
- positions,
287
- kv_caches,
288
- attn_metadata,
289
- None,
290
- inputs_embeds=inputs_embeds)
324
+ hidden_states = self.language_model.model(input_ids,
325
+ positions,
326
+ kv_caches,
327
+ attn_metadata,
328
+ None,
329
+ inputs_embeds=inputs_embeds)
330
331
return hidden_states
332
333
def compute_logits(self, hidden_states: torch.Tensor,
334
sampling_metadata: SamplingMetadata) -> torch.Tensor:
296
- logits = self.logits_processor(self.lm_head, hidden_states,
297
- sampling_metadata)
298
- return logits
335
+ return self.language_model.compute_logits(hidden_states,
336
+ sampling_metadata)
337
338
def sample(
339
self,
340
logits: torch.Tensor,
341
sampling_metadata: SamplingMetadata,
342
) -> Optional[SamplerOutput]:
305
- next_tokens = self.sampler(logits, sampling_metadata)
306
- return next_tokens
343
+ return self.language_model.sample(logits, sampling_metadata)
344
345
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
309
- # only doing this for language model part for now.
310
- stacked_params_mapping = [
311
- # (param_name, shard_name, shard_id)
312
- ("qkv_proj", "q_proj", "q"),
313
- ("qkv_proj", "k_proj", "k"),
314
- ("qkv_proj", "v_proj", "v"),
315
- ("gate_up_proj", "gate_proj", 0),
316
- ("gate_up_proj", "up_proj", 1),
317
- ]
318
- params_dict = dict(self.named_parameters())
319
- for name, loaded_weight in weights:
320
- if "rotary_emb.inv_freq" in name:
321
- continue
322
- # post_layernorm is not needed in CLIPVisionModel
323
- if "vision_model.post_layernorm" in name:
324
- continue
325
- for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
326
- if key_to_modify in name:
327
- name = name.replace(key_to_modify, new_key)
328
- use_default_weight_loading = False
329
- if "vision" in name:
330
- if self.vision_tower is not None:
331
- # We only do sharding for language model and
332
- # not vision model for now.
333
- use_default_weight_loading = True
334
- else:
335
- for (param_name, weight_name,
336
- shard_id) in stacked_params_mapping:
337
- if weight_name not in name:
338
- continue
339
- param = params_dict[name.replace(weight_name, param_name)]
340
- weight_loader = param.weight_loader
341
- weight_loader(param, loaded_weight, shard_id)
342
- break
343
- else:
344
- use_default_weight_loading = True
345
- if use_default_weight_loading and name in params_dict:
346
- param = params_dict[name]
347
- weight_loader = getattr(param, "weight_loader",
348
- default_weight_loader)
349
- weight_loader(param, loaded_weight)
346
+ # prepare weight iterators for components
347
+ vit_weights, mlp_weights, llm_weights = itertools.tee(weights, 3)
348
+
349
+ # load vision encoder
350
+ vit_weights = filter_weights(vit_weights, "vision_tower")
351
+ self.vision_tower.load_weights(vit_weights)
352
+
353
+ # load mlp projector
354
+ mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
355
+ mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
356
+ for name, loaded_weight in mlp_weights:
357
+ param = mlp_params_dict[name]
358
+ weight_loader = getattr(param, "weight_loader",
359
+ default_weight_loader)
360
+ weight_loader(param, loaded_weight)
361
+
362
+ # load llm backbone
363
+ llm_weights = filter_weights(llm_weights, "language_model")
364
+ self.language_model.load_weights(llm_weights)
vllm/model_executor/models/llava_next.py
+135
-104
@@ -1,9 +1,10 @@
1
+import itertools
2
from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
3
4
import torch
5
import torch.nn as nn
6
from PIL import Image
6
-from transformers import CLIPVisionConfig, LlavaNextConfig
7
+from transformers import CLIPVisionConfig, LlavaNextConfig, SiglipVisionConfig
8
from transformers.models.llava_next.modeling_llava_next import (
9
get_anyres_image_grid_shape, unpad_image)
10
from typing_extensions import NotRequired
@@ -12,23 +13,23 @@ from vllm.attention import AttentionMetadata
13
from vllm.config import CacheConfig, MultiModalConfig
14
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15
from vllm.logger import init_logger
15
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
from vllm.model_executor.layers.quantization.base_config import (
17
QuantizationConfig)
18
-from vllm.model_executor.layers.sampler import Sampler
19
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
18
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
-from vllm.model_executor.models.clip import CLIPVisionModel
22
-from vllm.model_executor.models.llama import LlamaModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
from vllm.sequence import IntermediateTensors, SamplerOutput
22
27
-from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
23
+from .clip import (CLIPVisionModel, dummy_image_for_clip,
24
+ dummy_seq_data_for_clip, get_clip_image_feature_size,
25
get_clip_patch_grid_length, input_processor_for_clip)
26
from .interfaces import SupportsVision
27
from .llava import LlavaMultiModalProjector
31
-from .utils import merge_vision_embeddings
28
+from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
29
+ dummy_seq_data_for_siglip, get_siglip_image_feature_size,
30
+ get_siglip_patch_grid_length, input_processor_for_siglip)
31
+from .utils import (filter_weights, init_vllm_registered_model,
32
+ merge_vision_embeddings)
33
34
logger = init_logger(__name__)
35
@@ -104,30 +105,42 @@ def get_llava_next_image_feature_size(
105
image_size=vision_config.image_size,
106
patch_size=vision_config.patch_size,
107
)
107
- base_feature_size = num_patches * num_patches
108
-
109
- num_patch_height, num_patch_width = get_anyres_image_grid_shape(
110
- image_size=(input_height, input_width),
111
- grid_pinpoints=hf_config.image_grid_pinpoints,
112
- patch_size=vision_config.image_size,
108
+ base_feature_size = get_clip_image_feature_size(vision_config)
109
+ elif isinstance(vision_config, SiglipVisionConfig):
110
+ num_patches = get_siglip_patch_grid_length(
111
+ image_size=vision_config.image_size,
112
+ patch_size=vision_config.patch_size,
113
)
114
+ base_feature_size = get_siglip_image_feature_size(vision_config)
115
+ else:
116
+ msg = f"Unsupported vision config: {type(vision_config)}"
117
+ raise NotImplementedError(msg)
118
+
119
+ strategy = hf_config.vision_feature_select_strategy
120
+ if strategy == "default":
121
+ base_feature_size -= 1
122
+ elif strategy == "full":
123
+ pass
124
+ else:
125
+ raise ValueError(f"Unexpected select feature strategy: {strategy}")
126
115
- (
116
- unpadded_feature_size,
117
- newline_feature_size,
118
- ) = _get_llava_next_num_unpadded_features(input_height, input_width,
119
- num_patches,
120
- num_patch_height,
121
- num_patch_width)
127
+ num_patch_height, num_patch_width = get_anyres_image_grid_shape(
128
+ image_size=(input_height, input_width),
129
+ grid_pinpoints=hf_config.image_grid_pinpoints,
130
+ patch_size=vision_config.image_size,
131
+ )
132
123
- return unpadded_feature_size + newline_feature_size + base_feature_size
133
+ (
134
+ unpadded_feature_size,
135
+ newline_feature_size,
136
+ ) = _get_llava_next_num_unpadded_features(input_height, input_width,
137
+ num_patches, num_patch_height,
138
+ num_patch_width)
139
125
- msg = f"Unsupported vision config: {type(vision_config)}"
126
- raise NotImplementedError(msg)
140
+ return unpadded_feature_size + newline_feature_size + base_feature_size
141
142
143
def get_max_llava_next_image_tokens(ctx: InputContext):
130
-
144
return get_llava_next_image_feature_size(
145
ctx.get_hf_config(LlavaNextConfig),
146
input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
@@ -155,6 +168,21 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
168
image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
169
)
170
171
+ return seq_data, mm_data
172
+ elif isinstance(vision_config, SiglipVisionConfig):
173
+ seq_data = dummy_seq_data_for_siglip(
174
+ vision_config,
175
+ seq_len,
176
+ image_token_id=hf_config.image_token_index,
177
+ image_feature_size_override=image_feature_size,
178
+ )
179
+
180
+ mm_data = dummy_image_for_siglip(
181
+ vision_config,
182
+ image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
183
+ image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
184
+ )
185
+
186
return seq_data, mm_data
187
188
msg = f"Unsupported vision config: {type(vision_config)}"
@@ -194,6 +222,40 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
222
image_token_id=hf_config.image_token_index,
223
image_feature_size_override=image_feature_size,
224
)
225
+ elif isinstance(vision_config, SiglipVisionConfig):
226
+ return input_processor_for_siglip(
227
+ model_config,
228
+ vision_config,
229
+ llm_inputs,
230
+ image_token_id=hf_config.image_token_index,
231
+ image_feature_size_override=image_feature_size,
232
+ )
233
+
234
+ msg = f"Unsupported vision config: {type(vision_config)}"
235
+ raise NotImplementedError(msg)
236
+
237
+
238
+def _init_vision_tower(hf_config: LlavaNextConfig):
239
+ vision_config = hf_config.vision_config
240
+
241
+ # Initialize the vision tower only up to the required feature layer
242
+ vision_feature_layer = hf_config.vision_feature_layer
243
+ if vision_feature_layer < 0:
244
+ num_hidden_layers = hf_config.vision_config.num_hidden_layers \
245
+ + vision_feature_layer + 1
246
+ else:
247
+ num_hidden_layers = vision_feature_layer + 1
248
+
249
+ if isinstance(vision_config, CLIPVisionConfig):
250
+ return CLIPVisionModel(
251
+ vision_config,
252
+ num_hidden_layers_override=num_hidden_layers,
253
+ )
254
+ elif isinstance(vision_config, SiglipVisionConfig):
255
+ return SiglipVisionModel(
256
+ vision_config,
257
+ num_hidden_layers_override=num_hidden_layers,
258
+ )
259
260
msg = f"Unsupported vision config: {type(vision_config)}"
261
raise NotImplementedError(msg)
@@ -215,36 +277,15 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
277
self.config = config
278
self.multimodal_config = multimodal_config
279
218
- # Initialize the vision tower only up to the required feature layer
219
- vision_feature_layer = config.vision_feature_layer
220
- if vision_feature_layer < 0:
221
- num_hidden_layers = config.vision_config.num_hidden_layers \
222
- + vision_feature_layer + 1
223
- else:
224
- num_hidden_layers = vision_feature_layer + 1
225
-
280
# TODO: Optionally initializes this for supporting embeddings.
227
- self.vision_tower = CLIPVisionModel(
228
- config.vision_config, num_hidden_layers_override=num_hidden_layers)
281
+ self.vision_tower = _init_vision_tower(config)
282
self.multi_modal_projector = LlavaMultiModalProjector(
283
vision_hidden_size=config.vision_config.hidden_size,
284
text_hidden_size=config.text_config.hidden_size,
285
projector_hidden_act=config.projector_hidden_act)
286
234
- self.quant_config = quant_config
235
- self.language_model = LlamaModel(config.text_config, cache_config,
236
- quant_config)
237
- self.unpadded_vocab_size = config.text_config.vocab_size
238
- self.lm_head = ParallelLMHead(
239
- self.unpadded_vocab_size,
240
- config.text_config.hidden_size,
241
- org_num_embeddings=self.language_model.org_vocab_size,
242
- quant_config=quant_config)
243
- logit_scale = getattr(config, "logit_scale", 1.0)
244
- self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
245
- config.text_config.vocab_size,
246
- logit_scale)
247
- self.sampler = Sampler()
287
+ self.language_model = init_vllm_registered_model(
288
+ config.text_config, cache_config, quant_config)
289
290
self.image_newline = nn.Parameter(
291
torch.empty(config.text_config.hidden_size))
@@ -310,8 +351,11 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
351
352
raise ValueError(f"Unexpected select feature strategy: {strategy}")
353
313
- def _image_pixels_to_features(self, vision_tower: CLIPVisionModel,
314
- pixel_values: torch.Tensor) -> torch.Tensor:
354
+ def _image_pixels_to_features(
355
+ self,
356
+ vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
357
+ pixel_values: torch.Tensor,
358
+ ) -> torch.Tensor:
359
360
# NOTE: we skip the step to select the vision feature layer since
361
# this is already done inside the vision tower
@@ -496,7 +540,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
540
541
if image_input is not None:
542
vision_embeddings = self._process_image_input(image_input)
499
- inputs_embeds = self.language_model.get_input_embeddings(input_ids)
543
+ inputs_embeds = self.language_model.model.get_input_embeddings(
544
+ input_ids)
545
546
inputs_embeds = merge_vision_embeddings(
547
input_ids, inputs_embeds, vision_embeddings,
@@ -506,68 +551,54 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
551
else:
552
inputs_embeds = None
553
509
- hidden_states = self.language_model(input_ids,
510
- positions,
511
- kv_caches,
512
- attn_metadata,
513
- None,
514
- inputs_embeds=inputs_embeds)
554
+ hidden_states = self.language_model.model(input_ids,
555
+ positions,
556
+ kv_caches,
557
+ attn_metadata,
558
+ None,
559
+ inputs_embeds=inputs_embeds)
560
561
return hidden_states
562
563
def compute_logits(self, hidden_states: torch.Tensor,
564
sampling_metadata: SamplingMetadata) -> torch.Tensor:
520
- logits = self.logits_processor(self.lm_head, hidden_states,
521
- sampling_metadata)
522
- return logits
565
+ return self.language_model.compute_logits(hidden_states,
566
+ sampling_metadata)
567
568
def sample(
569
self,
570
logits: torch.Tensor,
571
sampling_metadata: SamplingMetadata,
572
) -> Optional[SamplerOutput]:
529
- next_tokens = self.sampler(logits, sampling_metadata)
530
- return next_tokens
573
+ return self.language_model.sample(logits, sampling_metadata)
574
575
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
533
- # only doing this for language model part for now.
534
- stacked_params_mapping = [
535
- # (param_name, shard_name, shard_id)
536
- ("qkv_proj", "q_proj", "q"),
537
- ("qkv_proj", "k_proj", "k"),
538
- ("qkv_proj", "v_proj", "v"),
539
- ("gate_up_proj", "gate_proj", 0),
540
- ("gate_up_proj", "up_proj", 1),
541
- ]
542
- params_dict = dict(self.named_parameters())
543
- for name, loaded_weight in weights:
544
- if "rotary_emb.inv_freq" in name:
545
- continue
546
- # post_layernorm is not needed in CLIPVisionModel
547
- if "vision_model.post_layernorm" in name:
548
- continue
549
- for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
550
- if key_to_modify in name:
551
- name = name.replace(key_to_modify, new_key)
552
- use_default_weight_loading = False
553
- if "vision" in name:
554
- if self.vision_tower is not None:
555
- # We only do sharding for language model and
556
- # not vision model for now.
557
- use_default_weight_loading = True
558
- else:
559
- for (param_name, weight_name,
560
- shard_id) in stacked_params_mapping:
561
- if weight_name not in name:
562
- continue
563
- param = params_dict[name.replace(weight_name, param_name)]
564
- weight_loader = param.weight_loader
565
- weight_loader(param, loaded_weight, shard_id)
566
- break
567
- else:
568
- use_default_weight_loading = True
569
- if use_default_weight_loading and name in params_dict:
570
- param = params_dict[name]
571
- weight_loader = getattr(param, "weight_loader",
572
- default_weight_loader)
573
- weight_loader(param, loaded_weight)
576
+ # prepare weight iterators for components
577
+ vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
578
+ weights, 4)
579
+
580
+ # load vision encoder
581
+ vit_weights = filter_weights(vit_weights, "vision_tower")
582
+ self.vision_tower.load_weights(vit_weights)
583
+
584
+ # load mlp projector
585
+ mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
586
+ mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
587
+ for name, loaded_weight in mlp_weights:
588
+ param = mlp_params_dict[name]
589
+ weight_loader = getattr(param, "weight_loader",
590
+ default_weight_loader)
591
+ weight_loader(param, loaded_weight)
592
+
593
+ # load newline
594
+ newline_weights = filter_weights(newline_weights, "image_newline")
595
+ for name, loaded_weight in newline_weights:
596
+ assert name == ""
597
+ param = self.image_newline
598
+ weight_loader = getattr(param, "weight_loader",
599
+ default_weight_loader)
600
+ weight_loader(param, loaded_weight)
601
+
602
+ # load llm backbone
603
+ llm_weights = filter_weights(llm_weights, "language_model")
604
+ self.language_model.load_weights(llm_weights)
vllm/model_executor/models/siglip.py
+38
-11
@@ -2,12 +2,12 @@
2
within a vision language model."""
3
4
import math
5
-from typing import Optional, Tuple
5
+from typing import Iterable, Optional, Tuple
6
7
import torch
8
from PIL import Image
9
from torch import nn
10
-from transformers import SiglipConfig, SiglipVisionConfig
10
+from transformers import SiglipVisionConfig
11
from transformers.models.siglip.modeling_siglip import SiglipAttention
12
from vllm_flash_attn import flash_attn_func
13
from xformers.ops import memory_efficient_attention
@@ -22,13 +22,15 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22
from vllm.model_executor.layers.quantization import QuantizationConfig
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
VocabParallelEmbedding)
25
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26
from vllm.multimodal.image import (cached_get_tokenizer,
27
repeat_and_pad_image_tokens)
28
from vllm.sequence import SequenceData
29
30
31
def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
31
- assert image_size % patch_size == 0
32
+ # Since interpolation is applied, the image size need not be divisible
33
+ # assert image_size % patch_size == 0
34
return image_size // patch_size
35
36
@@ -454,7 +456,7 @@ class SiglipEncoderLayer(nn.Module):
456
457
def __init__(
458
self,
457
- config: SiglipConfig,
459
+ config: SiglipVisionConfig,
460
quant_config: Optional[QuantizationConfig] = None,
461
):
462
super().__init__()
@@ -474,7 +476,7 @@ class SiglipEncoderLayer(nn.Module):
476
def forward(
477
self,
478
hidden_states: torch.Tensor,
477
- ) -> Tuple[torch.Tensor]:
479
+ ) -> Tuple[torch.Tensor, None]:
480
residual = hidden_states
481
482
hidden_states = self.layer_norm1(hidden_states)
@@ -493,22 +495,27 @@ class SiglipEncoder(nn.Module):
495
496
def __init__(
497
self,
496
- config: SiglipConfig,
498
+ config: SiglipVisionConfig,
499
quant_config: Optional[QuantizationConfig] = None,
500
+ num_hidden_layers_override: Optional[int] = None,
501
):
502
super().__init__()
503
self.config = config
504
+
505
+ if num_hidden_layers_override is None:
506
+ num_hidden_layers = config.num_hidden_layers
507
+ else:
508
+ num_hidden_layers = num_hidden_layers_override
509
+
510
self.layers = nn.ModuleList([
502
- SiglipEncoderLayer(
503
- config,
504
- quant_config=quant_config,
505
- ) for _ in range(config.num_hidden_layers)
511
+ SiglipEncoderLayer(config, quant_config=quant_config)
512
+ for _ in range(num_hidden_layers)
513
])
514
515
def forward(
516
self,
517
inputs_embeds: torch.Tensor,
511
- ) -> Tuple:
518
+ ) -> torch.Tensor:
519
hidden_states = inputs_embeds
520
for encoder_layer in self.layers:
521
hidden_states, _ = encoder_layer(hidden_states)
@@ -553,6 +560,7 @@ class SiglipVisionTransformer(nn.Module):
560
self,
561
config: SiglipVisionConfig,
562
quant_config: Optional[QuantizationConfig] = None,
563
+ num_hidden_layers_override: Optional[int] = None,
564
):
565
super().__init__()
566
self.config = config
@@ -562,6 +570,7 @@ class SiglipVisionTransformer(nn.Module):
570
self.encoder = SiglipEncoder(
571
config,
572
quant_config=quant_config,
573
+ num_hidden_layers_override=num_hidden_layers_override,
574
)
575
self.post_layernorm = nn.LayerNorm(embed_dim,
576
eps=config.layer_norm_eps)
@@ -600,11 +609,13 @@ class SiglipVisionModel(nn.Module):
609
self,
610
config: SiglipVisionConfig,
611
quant_config: Optional[QuantizationConfig] = None,
612
+ num_hidden_layers_override: Optional[int] = None,
613
):
614
super().__init__()
615
self.vision_model = SiglipVisionTransformer(
616
config,
617
quant_config,
618
+ num_hidden_layers_override=num_hidden_layers_override,
619
)
620
621
def get_input_embeddings(self) -> nn.Module:
@@ -619,3 +630,19 @@ class SiglipVisionModel(nn.Module):
630
pixel_values=pixel_values,
631
interpolate_pos_encoding=interpolate_pos_encoding,
632
)
633
+
634
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
635
+ params_dict = dict(self.named_parameters())
636
+ layer_count = len(self.vision_model.encoder.layers)
637
+
638
+ for name, loaded_weight in weights:
639
+ # omit layers when num_hidden_layers_override is set
640
+ if "vision_model.encoder.layers." in name:
641
+ layer_idx = int(name.split(".")[3])
642
+ if layer_idx >= layer_count:
643
+ continue
644
+
645
+ param = params_dict[name]
646
+ weight_loader = getattr(param, "weight_loader",
647
+ default_weight_loader)
648
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/utils.py
+52
-4
@@ -1,22 +1,70 @@
1
-from typing import Dict, List, Protocol, Tuple
1
+from typing import Dict, Iterable, List, Optional, Protocol, Tuple
2
3
import torch
4
+import torch.nn as nn
5
from torch.func import functional_call
6
+from transformers import PretrainedConfig
7
8
+from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
9
+ SchedulerConfig)
10
+from vllm.model_executor.layers.quantization import QuantizationConfig
11
+from vllm.model_executor.model_loader.loader import build_model
12
+from vllm.model_executor.models import ModelRegistry
13
from vllm.multimodal import BatchedTensors
14
from vllm.utils import is_pin_memory_available
15
16
17
+def filter_weights(weights: Iterable[Tuple[str, torch.Tensor]], prefix: str):
18
+ """
19
+ Helper function to load weights for inner vLLM models.
20
+
21
+ See also:
22
+ :ref:`init_vllm_registered_model`
23
+ """
24
+ for name, loaded_weight in weights:
25
+ name = name.split(".")
26
+ if prefix == name.pop(0):
27
+ name = ".".join(name)
28
+ yield name, loaded_weight
29
+
30
+
31
+def init_vllm_registered_model(
32
+ hf_config: PretrainedConfig,
33
+ cache_config: Optional[CacheConfig],
34
+ quant_config: Optional[QuantizationConfig],
35
+ *,
36
+ lora_config: Optional[LoRAConfig] = None,
37
+ multimodal_config: Optional[MultiModalConfig] = None,
38
+ scheduler_config: Optional[SchedulerConfig] = None,
39
+) -> nn.Module:
40
+ """
41
+ Helper function to initialize an inner model registered to vLLM,
42
+ based on the arguments passed to the outer vLLM model.
43
+ """
44
+ model_class, _ = ModelRegistry.resolve_model_cls(hf_config.architectures)
45
+
46
+ return build_model(
47
+ model_class,
48
+ hf_config,
49
+ cache_config,
50
+ quant_config,
51
+ lora_config=lora_config,
52
+ multimodal_config=multimodal_config,
53
+ scheduler_config=scheduler_config,
54
+ )
55
+
56
+
57
def merge_vision_embeddings(input_ids: torch.Tensor,
58
inputs_embeds: torch.Tensor,
59
vision_embeddings: BatchedTensors,
60
image_token_id: int) -> torch.Tensor:
61
"""
15
- Merge `vision_embeddings` into `inputs_embeds` by overwriting the positions
16
- in `inputs_embeds` corresponding to placeholder image tokens in `input_ids`.
62
+ Merge ``vision_embeddings`` into ``inputs_embeds`` by overwriting the
63
+ positions in ``inputs_embeds`` corresponding to placeholder image tokens in
64
+ ``input_ids``.
65
66
Note:
19
- This updates `inputs_embeds` in place.
67
+ This updates ``inputs_embeds`` in place.
68
"""
69
mask = (input_ids == image_token_id)
70
num_expected_tokens = mask.sum()