@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 875af38e01
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 23:45 UTC

Support Intern-S1 (#21628)

Signed-off-by: Roger Wang <hey@rogerw.me> Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Your Name <you@example.com> Co-authored-by: Roger Wang <hey@rogerw.me> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Lyu Han committed Jul 26, 2025 at 19:14 UTC 875af38e01217f20827f0b4e1353b91c884b9d53
7 files changed +1196
docs/models/supported_models.md
+1
@@ -593,6 +593,7 @@ Specified using `--task generate`.
593 | `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | ✅︎ |
594 | `H2OVLChatModel` | H2OVL | T + I<sup>E+</sup> | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | | ✅︎ | ✅︎ |
595 | `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3`, etc. | ✅︎ | | ✅︎ |
596 +| `InternS1ForConditionalGeneration` | Intern-S1 | T + I<sup>E+</sup> | `internlm/Intern-S1`, etc. | | ✅︎ | ✅︎ |
597 | `InternVLChatModel` | InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + I<sup>E+</sup> + (V<sup>E+</sup>) | `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ | ✅︎ |
598 | `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-8B-Preview` | | | ✅︎ |
599 | `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I<sup>+</sup> | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | | | ✅︎ |
examples/offline_inference/vision_language.py
+32
@@ -468,6 +468,37 @@ def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
468 )
469
470
471 +# Intern-S1
472 +def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
473 + assert modality == "image"
474 +
475 + model_name = "internlm/Intern-S1"
476 +
477 + engine_args = EngineArgs(
478 + model=model_name,
479 + trust_remote_code=True,
480 + max_model_len=8192,
481 + max_num_seqs=2,
482 + limit_mm_per_prompt={modality: 1},
483 + enforce_eager=True,
484 + )
485 +
486 + placeholder = "<IMG_CONTEXT>"
487 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
488 + messages = [
489 + [{"role": "user", "content": f"{placeholder}\n{question}"}]
490 + for question in questions
491 + ]
492 + prompts = tokenizer.apply_chat_template(
493 + messages, tokenize=False, add_generation_prompt=True
494 + )
495 +
496 + return ModelRequestData(
497 + engine_args=engine_args,
498 + prompts=prompts,
499 + )
500 +
501 +
502 # InternVL
503 def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
504 model_name = "OpenGVLab/InternVL3-2B"
@@ -1303,6 +1334,7 @@ model_example_map = {
1334 "h2ovl_chat": run_h2ovl,
1335 "hyperclovax_seed_vision": run_hyperclovax_seed_vision,
1336 "idefics3": run_idefics3,
1337 + "interns1": run_interns1,
1338 "internvl_chat": run_internvl,
1339 "nemotron_vl": run_nemotron_vl,
1340 "keye_vl": run_keye_vl,
examples/offline_inference/vision_language_multi_image.py
+28
@@ -253,6 +253,33 @@ def load_smolvlm(question: str, image_urls: list[str]) -> ModelRequestData:
253 )
254
255
256 +def load_interns1(question: str, image_urls: list[str]) -> ModelRequestData:
257 + model_name = "internlm/Intern-S1"
258 +
259 + engine_args = EngineArgs(
260 + model=model_name,
261 + trust_remote_code=True,
262 + max_model_len=4096,
263 + limit_mm_per_prompt={"image": len(image_urls)},
264 + )
265 +
266 + placeholders = "\n".join(
267 + f"Image-{i}: <IMG_CONTEXT>\n" for i, _ in enumerate(image_urls, start=1)
268 + )
269 + messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
270 +
271 + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
272 + prompt = tokenizer.apply_chat_template(
273 + messages, tokenize=False, add_generation_prompt=True
274 + )
275 +
276 + return ModelRequestData(
277 + engine_args=engine_args,
278 + prompt=prompt,
279 + image_data=[fetch_image(url) for url in image_urls],
280 + )
281 +
282 +
283 def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:
284 model_name = "OpenGVLab/InternVL2-2B"
285
@@ -946,6 +973,7 @@ model_example_map = {
973 "gemma3": load_gemma3,
974 "h2ovl_chat": load_h2ovl,
975 "idefics3": load_idefics3,
976 + "interns1": load_interns1,
977 "internvl_chat": load_internvl,
978 "hyperclovax_seed_vision": load_hyperclovax_seed_vision,
979 "keye_vl": load_keye_vl,
tests/models/registry.py
+2
@@ -381,6 +381,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
381 extras={"2B": "OpenGVLab/InternVL2-2B",
382 "3.0": "OpenGVLab/InternVL3-1B"}, # noqa: E501
383 trust_remote_code=True),
384 + "InternS1ForConditionalGeneration": _HfExamplesInfo("internlm/Intern-S1",
385 + trust_remote_code=True),
386 "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3", # noqa: E501
387 {"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"}), # noqa: E501
388 "KeyeForConditionalGeneration": _HfExamplesInfo("Kwai-Keye/Keye-VL-8B-Preview", # noqa: E501
vllm/model_executor/models/interns1.py new
+711
@@ -0,0 +1,711 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# --------------------------------------------------------
5 +# InternS1
6 +# Copyright (c) 2025 Shanghai AI Lab
7 +# Licensed under The MIT License [see LICENSE for details]
8 +# --------------------------------------------------------
9 +from collections.abc import Iterable, Mapping, Sequence
10 +from typing import Literal, Optional, TypedDict, Union
11 +
12 +import torch
13 +import torch.nn as nn
14 +from transformers import InternVLProcessor, PretrainedConfig
15 +from transformers.activations import ACT2FN
16 +from transformers.models.got_ocr2.image_processing_got_ocr2_fast import (
17 + GotOcr2ImageProcessorFast)
18 +
19 +from vllm.config import VllmConfig
20 +from vllm.model_executor.layers.quantization import QuantizationConfig
21 +from vllm.model_executor.models.interns1_vit import InternS1VisionModel
22 +from vllm.model_executor.models.module_mapping import MultiModelKeys
23 +from vllm.model_executor.sampling_metadata import SamplingMetadata
24 +from vllm.multimodal import MULTIMODAL_REGISTRY
25 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26 + MultiModalKwargs, NestedTensors)
27 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28 + ImageSize, MultiModalDataItems)
29 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 + BaseProcessingInfo, PromptReplacement,
31 + PromptUpdate, PromptUpdateDetails)
32 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
33 +from vllm.sequence import IntermediateTensors
34 +
35 +from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
36 + SupportsMultiModal, SupportsPP)
37 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
38 + init_vllm_registered_model, maybe_prefix,
39 + merge_multimodal_embeddings)
40 +
41 +
42 +class InternS1MultiModalProjector(nn.Module):
43 +
44 + def __init__(self, config):
45 + super().__init__()
46 + self.layer_norm = nn.LayerNorm(config.vision_config.hidden_size *
47 + int(1 / config.downsample_ratio)**2)
48 + self.linear_1 = nn.Linear(
49 + config.vision_config.hidden_size *
50 + int(1 / config.downsample_ratio)**2,
51 + config.text_config.hidden_size)
52 + self.act = ACT2FN[config.projector_hidden_act]
53 + self.linear_2 = nn.Linear(config.text_config.hidden_size,
54 + config.text_config.hidden_size)
55 +
56 + def forward(self, image_features):
57 + hidden_states = self.layer_norm(image_features)
58 + hidden_states = self.linear_1(hidden_states)
59 + hidden_states = self.act(hidden_states)
60 + hidden_states = self.linear_2(hidden_states)
61 + return hidden_states
62 +
63 +
64 +class InternS1ImagePixelInputs(TypedDict):
65 + type: Literal["pixel_values"]
66 + pixel_values: torch.Tensor
67 + """
68 + Shape:
69 + `(batch_size * num_images * (1 + num_patches), num_channels, height, width)`
70 + """
71 +
72 +
73 +class InternS1ImageEmbeddingInputs(TypedDict):
74 + type: Literal["image_embeds"]
75 + data: Union[torch.Tensor, list[torch.Tensor]]
76 + """
77 + A tensor of shape `(num_images, total_image_feature_size, hidden_size)`
78 + or a list of tensors of shape `(total_image_feature_size, hidden_size)`
79 +
80 + `hidden_size` must match the hidden size of language model backbone.
81 + """
82 +
83 +
84 +InternS1ImageInputs = Union[InternS1ImagePixelInputs,
85 + InternS1ImageEmbeddingInputs]
86 +
87 +
88 +class InternS1VideoPixelInputs(TypedDict):
89 + type: Literal["pixel_values_videos"]
90 + pixel_values: torch.Tensor
91 + """
92 + Shape:
93 + `(batch_size * num_video * num_frames, num_channels, height, width)`
94 + """
95 +
96 + num_patches: torch.Tensor
97 + """Shape: `(batch_size * num_images)`"""
98 +
99 +
100 +class InternS1VideoEmbeddingInputs(TypedDict):
101 + type: Literal["video_embeds"]
102 + data: Union[torch.Tensor, list[torch.Tensor]]
103 + """
104 + A tensor of shape `(num_videos, total_video_feature_size, hidden_size)`
105 + or a list of tensors of shape `(total_video_feature_size, hidden_size)`
106 +
107 + `hidden_size` must match the hidden size of language model backbone.
108 + """
109 +
110 +
111 +InternS1VideoInputs = Union[InternS1VideoPixelInputs,
112 + InternS1VideoEmbeddingInputs]
113 +
114 +
115 +def resolve_interns1_min_max_num(
116 + min_dynamic_patch: int,
117 + max_dynamic_patch: int,
118 + dynamic_image_size: bool,
119 + use_thumbnail: bool,
120 +) -> tuple[int, int]:
121 + min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
122 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
123 +
124 + if use_thumbnail and max_dynamic_patch != 1:
125 + max_dynamic_patch += 1
126 +
127 + return min_dynamic_patch, max_dynamic_patch
128 +
129 +
130 +def get_interns1_target_ratios(
131 + min_num: int,
132 + max_num: int,
133 +) -> list[tuple[int, int]]:
134 + target_ratios = {(i, j)
135 + for n in range(min_num, max_num + 1)
136 + for i in range(1, n + 1)
137 + for j in range(1, n + 1) if min_num <= i * j <= max_num}
138 + return sorted(target_ratios, key=lambda x: x[0] * x[1])
139 +
140 +
141 +class InternS1ProcessingInfo(BaseProcessingInfo):
142 + """Basic image-only ProcessingInfo for InternS1-style models."""
143 +
144 + def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
145 + return self.ctx.get_hf_processor(InternVLProcessor, **kwargs)
146 +
147 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
148 + return {"image": None}
149 +
150 + def get_num_image_tokens(
151 + self,
152 + *,
153 + image_width: int,
154 + image_height: int,
155 + processor: Optional['GotOcr2ImageProcessorFast'] = None,
156 + ) -> int:
157 + if processor is None:
158 + processor = self.get_hf_processor().image_processor
159 +
160 + if not isinstance(processor, GotOcr2ImageProcessorFast):
161 + raise ValueError(f'GotOcr2ImageProcessorFast is expected but got '
162 + f'{type(processor)}')
163 + num_image_patches = processor.get_number_of_image_tokens(
164 + image_height, image_width, images_kwargs=dict())
165 + num_image_tokens = self.get_hf_processor(
166 + ).image_seq_length * num_image_patches
167 + return num_image_tokens
168 +
169 + def resolve_target_ratios(self, use_thumbnail: Optional[bool] = None):
170 + image_processor = self.get_hf_processor().image_processor
171 + min_dynamic_patch = image_processor.min_patches
172 + max_dynamic_patch = image_processor.max_patches
173 + # HF format's InternVL processor uses `crop_to_patches` which is
174 + # equivalent to `use_thumbnail` in original format.
175 + use_thumbnail = image_processor.crop_to_patches
176 + dynamic_image_size = True
177 + min_num, max_num = resolve_interns1_min_max_num(
178 + min_dynamic_patch,
179 + max_dynamic_patch,
180 + dynamic_image_size,
181 + use_thumbnail=use_thumbnail)
182 +
183 + return get_interns1_target_ratios(min_num, max_num)
184 +
185 + def get_image_size_with_most_features(self) -> ImageSize:
186 + processor = self.get_hf_processor()
187 +
188 + hf_config = self.ctx.get_hf_config()
189 + base_height, base_width = hf_config.vision_config.image_size
190 + target_ratios = self.resolve_target_ratios()
191 +
192 + largest_feature_size, largest_feature_pinpoint = 0, None
193 + for wr, hr in target_ratios:
194 + width, height = base_width * wr, base_height * hr
195 +
196 + feat_size = self.get_num_image_tokens(
197 + image_width=width,
198 + image_height=height,
199 + processor=processor.image_processor,
200 + )
201 + if feat_size > largest_feature_size:
202 + largest_feature_size = feat_size
203 + largest_feature_pinpoint = ImageSize(width=width,
204 + height=height)
205 +
206 + assert not (largest_feature_size == 0 or largest_feature_pinpoint
207 + is None), ("Cannot have a largest feature size of 0!")
208 +
209 + return largest_feature_pinpoint
210 +
211 + def get_max_image_tokens(self) -> int:
212 + processor = self.get_hf_processor()
213 + target_width, target_height = self.get_image_size_with_most_features()
214 +
215 + return self.get_num_image_tokens(
216 + image_width=target_width,
217 + image_height=target_height,
218 + processor=processor.image_processor,
219 + )
220 +
221 +
222 +class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo]
223 + ):
224 + """Basic image-only DummyInputsBuilder for InternS1-style models."""
225 +
226 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
227 + num_images = mm_counts.get("image", 0)
228 + image_token = self.info.get_hf_processor().image_token
229 +
230 + return image_token * num_images
231 +
232 + def get_dummy_mm_data(
233 + self,
234 + seq_len: int,
235 + mm_counts: Mapping[str, int],
236 + ) -> MultiModalDataDict:
237 + target_width, target_height = \
238 + self.info.get_image_size_with_most_features()
239 + num_images = mm_counts.get("image", 0)
240 +
241 + return {
242 + "image":
243 + self._get_dummy_images(width=target_width,
244 + height=target_height,
245 + num_images=num_images)
246 + }
247 +
248 +
249 +class InternS1MultiModalProcessor(
250 + BaseMultiModalProcessor[InternS1ProcessingInfo]):
251 + """ Basic image-only MultiModalProcessor for InternS1-style models."""
252 +
253 + def _call_hf_processor(
254 + self,
255 + prompt: str,
256 + mm_data: Mapping[str, object],
257 + mm_kwargs: Mapping[str, object],
258 + tok_kwargs: Mapping[str, object],
259 + ) -> Mapping[str, NestedTensors]:
260 + processed_outputs = super()._call_hf_processor(
261 + prompt=prompt,
262 + mm_data=mm_data,
263 + mm_kwargs=mm_kwargs,
264 + tok_kwargs=tok_kwargs,
265 + )
266 +
267 + hf_processor = self.info.get_hf_processor(**mm_kwargs)
268 + image_token_id = hf_processor.image_token_id
269 +
270 + # Since there may be extra tokens in the feature placeholders,
271 + # we need to pass the image token ID to the model to select the
272 + # tokens to merge from the vision encoder outputs
273 + processed_outputs["image_token_id"] = torch.tensor(image_token_id)
274 + images = mm_data.get('images', None)
275 + image_processor = self.info.get_hf_processor().image_processor
276 + if images is not None:
277 + image_inputs = image_processor(images=images)
278 + image_num_patches = image_inputs.pop("num_patches")
279 + if not isinstance(image_num_patches, list):
280 + raise ValueError(
281 + f'num_patches is supposed to be list, but got '
282 + f'{type(image_num_patches)}')
283 + image_num_patches = torch.tensor(image_num_patches)
284 + processed_outputs['image_num_patches'] = image_num_patches
285 +
286 + return processed_outputs
287 +
288 + def _get_mm_fields_config(
289 + self,
290 + hf_inputs: Mapping[str, NestedTensors],
291 + hf_processor_mm_kwargs: Mapping[str, object],
292 + ) -> Mapping[str, MultiModalFieldConfig]:
293 +
294 + image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
295 + num_images = len(image_num_patches)
296 +
297 + return dict(
298 + pixel_values=MultiModalFieldConfig.flat_from_sizes(
299 + "image", image_num_patches),
300 + image_num_patches=MultiModalFieldConfig.batched("image"),
301 + image_embeds=MultiModalFieldConfig.batched("image"),
302 + image_token_id=MultiModalFieldConfig.shared("image", num_images),
303 + )
304 +
305 + def _get_prompt_updates(
306 + self,
307 + mm_items: MultiModalDataItems,
308 + hf_processor_mm_kwargs: Mapping[str, object],
309 + out_mm_kwargs: MultiModalKwargs,
310 + ) -> Sequence[PromptUpdate]:
311 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
312 + img_context_token = hf_processor.image_token
313 + start_image_token = hf_processor.start_image_token
314 + end_image_token = hf_processor.end_image_token
315 +
316 + def get_replacement(item_idx: int):
317 + images = mm_items.get_items(
318 + "image", (ImageEmbeddingItems, ImageProcessorItems))
319 +
320 + if isinstance(images, ImageEmbeddingItems):
321 + feature_size = images.get_feature_size(item_idx)
322 + else:
323 + image_size = images.get_image_size(item_idx)
324 + feature_size = self.info.get_num_image_tokens(
325 + image_width=image_size.width,
326 + image_height=image_size.height,
327 + processor=hf_processor.image_processor,
328 + )
329 +
330 + repl_features = img_context_token * feature_size
331 + repl_full = start_image_token + repl_features + end_image_token
332 + return PromptUpdateDetails.select_text(repl_full,
333 + img_context_token)
334 +
335 + return [
336 + PromptReplacement(
337 + modality="image",
338 + target=img_context_token,
339 + replacement=get_replacement,
340 + )
341 + ]
342 +
343 +
344 +@MULTIMODAL_REGISTRY.register_processor(
345 + InternS1MultiModalProcessor,
346 + info=InternS1ProcessingInfo,
347 + dummy_inputs=InternS1DummyInputsBuilder)
348 +class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal,
349 + SupportsPP, SupportsLoRA):
350 +
351 + # To ensure correct weight loading and mapping.
352 + hf_to_vllm_mapper = WeightsMapper(
353 + orig_to_new_prefix={
354 + "lm_head.": "language_model.lm_head.",
355 + "model.language_model.": "language_model.model.",
356 + "model.vision_tower.": "vision_tower.",
357 + "model.multi_modal_projector.": "multi_modal_projector.",
358 + })
359 +
360 + @classmethod
361 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
362 + # transformers InternVLProcessor uses <IMG_CONTEXT> as the seperator
363 + # refer to https://github.com/huggingface/transformers/blob/f90de364c2484c7c325bbe05befdcf487bd75b63/src/transformers/models/internvl/processing_internvl.py#L116
364 + if modality.startswith("image"):
365 + return '<IMG_CONTEXT>'
366 + if modality.startswith("video"):
367 + return "<video>"
368 +
369 + raise ValueError("Only image or video modality is supported")
370 +
371 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
372 + super().__init__()
373 + config = vllm_config.model_config.hf_config
374 + quant_config = vllm_config.quant_config
375 + multimodal_config = vllm_config.model_config.multimodal_config
376 +
377 + self.config = config
378 + self.multimodal_config = multimodal_config
379 +
380 + image_size = config.vision_config.image_size[0]
381 + patch_size = config.vision_config.patch_size[0]
382 + self.patch_size = patch_size
383 + self.num_image_token = int(
384 + (image_size // patch_size)**2 * (config.downsample_ratio**2))
385 + self.downsample_ratio = config.downsample_ratio
386 +
387 + self.llm_arch_name = config.text_config.architectures[0]
388 + self.vision_tower = self._init_vision_model(
389 + config,
390 + quant_config=quant_config,
391 + prefix=maybe_prefix(prefix, "vision_tower"),
392 + )
393 +
394 + self.language_model = init_vllm_registered_model(
395 + vllm_config=vllm_config,
396 + hf_config=config.text_config,
397 + prefix=maybe_prefix(prefix, "language_model"),
398 + )
399 +
400 + self.multi_modal_projector = self._init_mlp1(config)
401 +
402 + self.img_context_token_id = None
403 + self.video_context_token_id = None
404 +
405 + self.visual_token_mask = None
406 + self.make_empty_intermediate_tensors = (
407 + self.language_model.make_empty_intermediate_tensors)
408 +
409 + def _init_vision_model(
410 + self,
411 + config: PretrainedConfig,
412 + quant_config: Optional[QuantizationConfig],
413 + *,
414 + prefix: str,
415 + ):
416 + num_hidden_layers = config.vision_config.num_hidden_layers
417 + return InternS1VisionModel(
418 + config.vision_config,
419 + quant_config=quant_config,
420 + num_hidden_layers_override=num_hidden_layers,
421 + prefix=prefix,
422 + )
423 +
424 + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
425 + return InternS1MultiModalProjector(config)
426 +
427 + def pixel_shuffle(self, x, scale_factor=0.5):
428 + n, w, h, c = x.size()
429 + # N, W, H, C --> N, W, H * scale, C // scale
430 + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
431 + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
432 + x = x.permute(0, 2, 1, 3).contiguous()
433 + x = x.view(n, int(h * scale_factor), int(w * scale_factor),
434 + int(c / (scale_factor * scale_factor)))
435 + x = x.permute(0, 2, 1, 3).contiguous()
436 + return x
437 +
438 + def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
439 + vit_embeds = self.vision_tower(pixel_values=pixel_values)
440 + vit_embeds = vit_embeds[:, 1:, :]
441 +
442 + h = w = int(vit_embeds.shape[1]**0.5)
443 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
444 + vit_embeds = self.pixel_shuffle(vit_embeds,
445 + scale_factor=self.downsample_ratio)
446 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1,
447 + vit_embeds.shape[-1])
448 +
449 + vit_embeds = self.multi_modal_projector(vit_embeds)
450 + return vit_embeds
451 +
452 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
453 +
454 + h, w = self.config.vision_config.image_size
455 + expected_dims = (3, h, w)
456 +
457 + def _validate_shape(d: torch.Tensor):
458 + actual_dims = tuple(d.shape)
459 +
460 + if actual_dims != expected_dims:
461 + expected_expr = str(expected_dims)
462 + raise ValueError(
463 + "The expected shape of pixel values per image per batch "
464 + f" per patch is {expected_expr}. "
465 + f"You supplied {tuple(d.shape)}.")
466 +
467 + for d in data:
468 + _validate_shape(d)
469 +
470 + return data
471 +
472 + def _parse_and_validate_image_input(
473 + self, **kwargs: object) -> Optional[InternS1ImageInputs]:
474 + pixel_values = kwargs.pop("pixel_values", None)
475 + image_num_patches = kwargs.pop("image_num_patches", None)
476 + image_embeds = kwargs.pop("image_embeds", None)
477 +
478 + if pixel_values is None and image_embeds is None:
479 + return None
480 +
481 + if image_embeds is not None:
482 + if not isinstance(image_embeds, (torch.Tensor, list)):
483 + raise ValueError("Incorrect type of image embeddings. "
484 + f"Got type: {type(image_embeds)}")
485 +
486 + return InternS1ImageEmbeddingInputs(
487 + type="image_embeds",
488 + data=flatten_bn(image_embeds),
489 + )
490 +
491 + image_token_id = kwargs["image_token_id"]
492 + assert isinstance(image_token_id, torch.Tensor)
493 + self.img_context_token_id = image_token_id.flatten().unique().item()
494 +
495 + if pixel_values is not None:
496 + if not isinstance(pixel_values, (torch.Tensor, list)):
497 + raise ValueError("Incorrect type of pixel values. "
498 + f"Got type: {type(pixel_values)}")
499 +
500 + if not isinstance(image_num_patches, (torch.Tensor, list)):
501 + raise ValueError("Incorrect type of image_num_patches. "
502 + f"Got type: {type(image_num_patches)}")
503 +
504 + pixel_values = flatten_bn(pixel_values, concat=True)
505 + image_num_patches = flatten_bn(image_num_patches, concat=True)
506 +
507 + return InternS1ImagePixelInputs(
508 + type="pixel_values",
509 + pixel_values=self._validate_pixel_values(pixel_values),
510 + num_patches=image_num_patches,
511 + )
512 +
513 + raise AssertionError("This line should be unreachable.")
514 +
515 + def _parse_and_validate_video_input(
516 + self, **kwargs: object) -> Optional[InternS1VideoPixelInputs]:
517 + pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
518 + video_num_patches = kwargs.pop("video_num_patches", None)
519 + video_embeds = kwargs.pop("video_embeds", None)
520 +
521 + if pixel_values_flat_video is None and video_embeds is None:
522 + return None
523 +
524 + if video_embeds is not None:
525 + if not isinstance(video_embeds, (torch.Tensor, list)):
526 + raise ValueError("Incorrect type of video embeddings. "
527 + f"Got type: {type(video_embeds)}")
528 +
529 + return InternS1ImageEmbeddingInputs(
530 + type="video_embeds",
531 + data=flatten_bn(video_embeds),
532 + )
533 +
534 + video_token_id = kwargs["video_token_id"]
535 + assert isinstance(video_token_id, torch.Tensor)
536 + self.video_context_token_id = video_token_id.flatten().unique().item()
537 +
538 + if pixel_values_flat_video is not None:
539 + if not isinstance(pixel_values_flat_video, (torch.Tensor, list)):
540 + raise ValueError("Incorrect type of pixel values. "
541 + f"Got type: {type(pixel_values_flat_video)}")
542 +
543 + if not isinstance(video_num_patches, (torch.Tensor, list)):
544 + raise ValueError("Incorrect type of image_num_patches. "
545 + f"Got type: {type(video_num_patches)}")
546 +
547 + pixel_values_flat_video = flatten_bn(pixel_values_flat_video,
548 + concat=True)
549 + video_num_patches = flatten_bn(video_num_patches, concat=True)
550 +
551 + return InternS1VideoPixelInputs(
552 + type="pixel_values_videos",
553 + pixel_values=self._validate_pixel_values(
554 + pixel_values_flat_video),
555 + num_patches=video_num_patches,
556 + )
557 +
558 + raise AssertionError("This line should be unreachable.")
559 +
560 + def _process_image_input(
561 + self,
562 + image_input: Union[InternS1ImageInputs, InternS1VideoPixelInputs],
563 + ) -> tuple[torch.Tensor, ...]:
564 + if image_input["type"] == "image_embeds":
565 + return image_input["data"]
566 +
567 + assert self.vision_tower is not None
568 +
569 + image_embeds = self.extract_feature(image_input["pixel_values"])
570 +
571 + num_patches = image_input["num_patches"]
572 +
573 + # Only one image in the current batch
574 + if len(num_patches) == 1:
575 + return (image_embeds.view(-1,
576 + self.config.text_config.hidden_size), )
577 +
578 + # NOTE: Image embeddings are split into separate tensors for each image
579 + # by the size of each embedding.
580 + feature_size = image_embeds.shape[1]
581 + image_embeds = image_embeds.view(-1,
582 + self.config.text_config.hidden_size)
583 + image_feature_sizes = [
584 + num_patches * feature_size for num_patches in num_patches
585 + ]
586 + return image_embeds.split(image_feature_sizes)
587 +
588 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
589 + modalities = {}
590 +
591 + # Preserve the order of modalities if there are multiple of them
592 + # from the order of kwargs.
593 + for input_key in kwargs:
594 + if input_key in ("pixel_values",
595 + "image_embeds") and "images" not in modalities:
596 + modalities["images"] = self._parse_and_validate_image_input(
597 + **kwargs)
598 + if input_key in ("pixel_values_flat_video",
599 + ) and "videos" not in modalities:
600 + modalities["videos"] = self._parse_and_validate_video_input(
601 + **kwargs)
602 +
603 + return modalities
604 +
605 + def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
606 + self.visual_token_mask = None
607 +
608 + def get_language_model(self) -> torch.nn.Module:
609 + return self.language_model
610 +
611 + def get_multimodal_embeddings(self,
612 + **kwargs: object) -> MultiModalEmbeddings:
613 +
614 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
615 + if not modalities:
616 + return []
617 + return None
618 +
619 + # The result multimodal_embeddings is tuple of tensors, with each
620 + # tensor correspoending to a multimodal data item (image or video).
621 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
622 +
623 + # NOTE: It is important to iterate over the keys in this dictionary
624 + # to preserve the order of the modalities.
625 + for modality in modalities:
626 + if modality == "images":
627 + image_input = modalities["images"]
628 + vision_embeddings = self._process_image_input(image_input)
629 + multimodal_embeddings += vision_embeddings
630 + if modality == "videos":
631 + video_input = modalities["videos"]
632 + video_embeddings = self._process_image_input(video_input)
633 + multimodal_embeddings += video_embeddings
634 +
635 + return multimodal_embeddings
636 +
637 + def get_input_embeddings(
638 + self,
639 + input_ids: torch.Tensor,
640 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
641 + ) -> torch.Tensor:
642 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
643 + if multimodal_embeddings is not None \
644 + and len(multimodal_embeddings) != 0:
645 + context_token_ids = [
646 + token_id for token_id in (self.img_context_token_id,
647 + self.video_context_token_id)
648 + if token_id is not None
649 + ]
650 + assert len(context_token_ids) >= 1
651 + self._set_visual_token_mask(input_ids)
652 + inputs_embeds = merge_multimodal_embeddings(
653 + input_ids,
654 + inputs_embeds,
655 + multimodal_embeddings,
656 + context_token_ids,
657 + )
658 + return inputs_embeds
659 +
660 + def forward(
661 + self,
662 + input_ids: torch.Tensor,
663 + positions: torch.Tensor,
664 + intermediate_tensors: Optional[IntermediateTensors] = None,
665 + inputs_embeds: Optional[torch.Tensor] = None,
666 + **kwargs: object,
667 + ) -> IntermediateTensors:
668 +
669 + if intermediate_tensors is not None:
670 + input_ids = None
671 + inputs_embeds = None
672 +
673 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
674 + # condition is for v0 compatibility.
675 + elif inputs_embeds is None:
676 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
677 + inputs_embeds = self.get_input_embeddings(input_ids,
678 + vision_embeddings)
679 + input_ids = None
680 +
681 + forward_kwargs = {
682 + "input_ids": input_ids,
683 + "positions": positions,
684 + "intermediate_tensors": intermediate_tensors,
685 + "inputs_embeds": inputs_embeds,
686 + }
687 +
688 + hidden_states = self.language_model.model(**forward_kwargs)
689 + return hidden_states
690 +
691 + def compute_logits(
692 + self,
693 + hidden_states: torch.Tensor,
694 + sampling_metadata: SamplingMetadata,
695 + ) -> Optional[torch.Tensor]:
696 + return self.language_model.compute_logits(hidden_states,
697 + sampling_metadata)
698 +
699 + def load_weights(self, weights: Iterable[tuple[str,
700 + torch.Tensor]]) -> set[str]:
701 + loader = AutoWeightsLoader(self)
702 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
703 +
704 + def get_mm_mapping(self) -> MultiModelKeys:
705 + """
706 + Get the module prefix in multimodal models
707 + """
708 + return MultiModelKeys.from_string_field(
709 + language_model="language_model",
710 + connector="multi_modal_projector",
711 + tower_model="vision_tower")
vllm/model_executor/models/interns1_vit.py new
+421
@@ -0,0 +1,421 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py
5 +# --------------------------------------------------------
6 +# InternVL
7 +# Copyright (c) 2023 OpenGVLab
8 +# Licensed under The MIT License [see LICENSE for details]
9 +# --------------------------------------------------------
10 +from collections.abc import Iterable
11 +from typing import Optional
12 +
13 +import torch
14 +import torch.nn as nn
15 +import torch.nn.functional as F
16 +from transformers import PretrainedConfig
17 +from transformers.utils import torch_int
18 +
19 +from vllm.model_executor.layers.activation import get_act_fn
20 +from vllm.model_executor.layers.layernorm import RMSNorm
21 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
22 + RowParallelLinear)
23 +from vllm.model_executor.layers.quantization import QuantizationConfig
24 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25 +
26 +NORM2FN = {
27 + 'rms_norm': RMSNorm,
28 + 'layer_norm': nn.LayerNorm,
29 +}
30 +
31 +
32 +class InternS1VisionPatchEmbeddings(nn.Module):
33 +
34 + def __init__(self, config):
35 + super().__init__()
36 + image_size, patch_size = config.image_size, config.patch_size
37 + num_channels, hidden_size = config.num_channels, config.hidden_size
38 +
39 + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] //
40 + patch_size[0])
41 + patch_shape = (image_size[0] // patch_size[0],
42 + image_size[1] // patch_size[1])
43 + self.image_size = image_size
44 + self.patch_size = patch_size
45 + self.num_channels = num_channels
46 + self.num_patches = num_patches
47 + self.patch_shape = patch_shape
48 +
49 + self.projection = nn.Conv2d(num_channels,
50 + hidden_size,
51 + kernel_size=patch_size,
52 + stride=patch_size)
53 +
54 + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
55 + batch_size, num_channels, height, width = pixel_values.shape
56 + if num_channels != self.num_channels:
57 + raise ValueError(
58 + "Make sure that the channel dimension of the pixel values "
59 + "match with the one set in the configuration.")
60 +
61 + embeddings = self.projection(
62 + pixel_values.to(self.projection.weight.dtype))
63 + patch_height, patch_width = embeddings.shape[2], embeddings.shape[3]
64 + embeddings = embeddings.flatten(2).transpose(1, 2)
65 +
66 + return embeddings, (patch_height, patch_width)
67 +
68 +
69 +class InternS1VisionEmbeddings(nn.Module):
70 +
71 + def __init__(self, config: PretrainedConfig):
72 + super().__init__()
73 + self.config = config
74 + self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
75 + if config.use_mask_token:
76 + self.mask_token = nn.Parameter(
77 + torch.zeros(1, 1, config.hidden_size))
78 + else:
79 + self.mask_token = None
80 + self.patch_embeddings = InternS1VisionPatchEmbeddings(config)
81 + self.patch_size = config.patch_size
82 + self.image_size = (config.image_size if isinstance(
83 + config.image_size, Iterable) else
84 + (config.image_size, config.image_size))
85 + num_patches = self.patch_embeddings.num_patches
86 + if config.use_absolute_position_embeddings:
87 + self.position_embeddings = nn.Parameter(
88 + torch.zeros(1, num_patches + 1, config.hidden_size))
89 + else:
90 + self.position_embeddings = None
91 +
92 + def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int,
93 + width: int) -> torch.Tensor:
94 + """
95 + This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
96 + images. This method is also adapted to support torch.jit tracing.
97 +
98 + Adapted from:
99 + - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
100 + - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
101 + """ # noqa: E501
102 +
103 + num_patches = embeddings.shape[1] - 1
104 + num_positions = self.position_embeddings.shape[1] - 1
105 +
106 + # always interpolate when tracing to ensure the exported model
107 + # works for dynamic input shapes
108 + if not torch.jit.is_tracing(
109 + ) and num_patches == num_positions and height == width:
110 + return self.position_embeddings
111 +
112 + class_pos_embed = self.position_embeddings[:, :1]
113 + patch_pos_embed = self.position_embeddings[:, 1:]
114 +
115 + dim = embeddings.shape[-1]
116 +
117 + new_height = height // self.patch_size[0]
118 + new_width = width // self.patch_size[1]
119 +
120 + sqrt_num_positions = torch_int(num_positions**0.5)
121 + patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions,
122 + sqrt_num_positions, dim)
123 + patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)
124 +
125 + patch_pos_embed = nn.functional.interpolate(
126 + patch_pos_embed,
127 + size=(new_height, new_width),
128 + mode="bicubic",
129 + align_corners=False,
130 + )
131 +
132 + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
133 +
134 + return torch.cat((class_pos_embed, patch_pos_embed), dim=1)
135 +
136 + def forward(
137 + self,
138 + pixel_values: torch.Tensor,
139 + bool_masked_pos: Optional[torch.BoolTensor] = None,
140 + ) -> torch.Tensor:
141 + _, _, height, width = pixel_values.shape
142 + embeddings, (patch_height,
143 + patch_width) = self.patch_embeddings(pixel_values)
144 + batch_size, seq_len, _ = embeddings.size()
145 +
146 + if bool_masked_pos is not None:
147 + mask_tokens = self.mask_token.expand(batch_size, seq_len, -1)
148 + # replace the masked visual tokens by mask_tokens
149 + w = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens)
150 + embeddings = embeddings * (1 - w) + mask_tokens * w
151 +
152 + cls_tokens = self.cls_token.expand(batch_size, -1, -1)
153 + embeddings = torch.cat((cls_tokens, embeddings), dim=1)
154 +
155 + if self.position_embeddings is not None:
156 + embeddings = embeddings + self.interpolate_pos_encoding(
157 + embeddings, height, width)
158 +
159 + return embeddings, (patch_height, patch_width)
160 +
161 +
162 +class InternSdpaAttention(nn.Module):
163 + """Multi-headed attention from 'Attention Is All You Need' paper"""
164 +
165 + def __init__(
166 + self,
167 + config: PretrainedConfig,
168 + *,
169 + num_dummy_heads: int = 0,
170 + ) -> None:
171 + super().__init__()
172 +
173 + self.config = config
174 + self.embed_dim = config.hidden_size
175 + self.num_heads = config.num_attention_heads
176 + self.head_dim = self.embed_dim // self.num_heads
177 + if self.head_dim * self.num_heads != self.embed_dim:
178 + raise ValueError(
179 + f'embed_dim must be divisible by num_heads '
180 + f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
181 + f' {self.num_heads}).')
182 +
183 + # Additional dummy heads are used to enable TP for common GPU counts.
184 + self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim
185 +
186 + self.scale = self.head_dim**-0.5
187 +
188 + self.q_proj = nn.Linear(self.embed_dim,
189 + self.num_heads * self.head_dim,
190 + bias=config.attention_bias)
191 + self.k_proj = nn.Linear(self.embed_dim,
192 + self.num_heads * self.head_dim,
193 + bias=config.attention_bias)
194 + self.v_proj = nn.Linear(self.embed_dim,
195 + self.num_heads * self.head_dim,
196 + bias=config.attention_bias)
197 +
198 + self.qk_normalization = config.use_qk_norm
199 + if self.qk_normalization:
200 + self.q_norm = RMSNorm(self.dummy_dim,
201 + eps=config.layer_norm_eps,
202 + var_hidden_size=self.embed_dim)
203 + self.k_norm = RMSNorm(self.dummy_dim,
204 + eps=config.layer_norm_eps,
205 + var_hidden_size=self.embed_dim)
206 +
207 + self.projection_layer = nn.Linear(self.dummy_dim, self.embed_dim)
208 +
209 + def forward(self, x: torch.Tensor) -> torch.Tensor:
210 + B, N, C = x.shape
211 +
212 + q = self.q_proj(x)
213 + k = self.k_proj(x)
214 + v = self.v_proj(x)
215 +
216 + q = q.view(B, N, self.num_heads, self.head_dim)
217 + k = k.view(B, N, self.num_heads, self.head_dim)
218 + v = v.view(B, N, self.num_heads, self.head_dim)
219 +
220 + if self.qk_normalization:
221 + B_, N_, H_, D_ = q.shape
222 + q = self.q_norm(q.flatten(-2, -1)).view(B_, N_, H_, D_)
223 + k = self.k_norm(k.flatten(-2, -1)).view(B_, N_, H_, D_)
224 + q = q.transpose(1, 2)
225 + k = k.transpose(1, 2)
226 + v = v.transpose(1, 2)
227 +
228 + x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
229 + x = x.transpose(1, 2).reshape(B, N, -1)
230 +
231 + x = self.projection_layer(x)
232 + return x
233 +
234 +
235 +class InternS1VisionMLP(nn.Module):
236 +
237 + def __init__(
238 + self,
239 + config: PretrainedConfig,
240 + quant_config: Optional[QuantizationConfig] = None,
241 + prefix: str = "",
242 + ) -> None:
243 + super().__init__()
244 +
245 + self.config = config
246 + self.activation_fn = get_act_fn(config.hidden_act)
247 + self.fc1 = ColumnParallelLinear(config.hidden_size,
248 + config.intermediate_size,
249 + bias=True,
250 + quant_config=quant_config,
251 + prefix=f"{prefix}.fc1")
252 + self.fc2 = RowParallelLinear(config.intermediate_size,
253 + config.hidden_size,
254 + bias=True,
255 + quant_config=quant_config,
256 + prefix=f"{prefix}.fc2")
257 +
258 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
259 + hidden_states, _ = self.fc1(hidden_states)
260 + hidden_states = self.activation_fn(hidden_states)
261 + hidden_states, _ = self.fc2(hidden_states)
262 +
263 + return hidden_states
264 +
265 +
266 +class InternS1VisionLayer(nn.Module):
267 +
268 + def __init__(
269 + self,
270 + config: PretrainedConfig,
271 + quant_config: Optional[QuantizationConfig] = None,
272 + *,
273 + num_dummy_heads: int = 0,
274 + prefix: str = "",
275 + ) -> None:
276 + super().__init__()
277 +
278 + self.attention = self._init_attn(config,
279 + quant_config,
280 + num_dummy_heads=num_dummy_heads,
281 + prefix=f"{prefix}.attention")
282 +
283 + self.mlp = InternS1VisionMLP(config,
284 + quant_config=quant_config,
285 + prefix=f"{prefix}.mlp")
286 + self.layernorm_before = NORM2FN[config.norm_type](
287 + config.hidden_size, eps=config.layer_norm_eps)
288 + self.layernorm_after = NORM2FN[config.norm_type](
289 + config.hidden_size, eps=config.layer_norm_eps)
290 +
291 + init_values = config.layer_scale_init_value
292 + self.lambda_1 = nn.Parameter(init_values *
293 + torch.ones(config.hidden_size),
294 + requires_grad=True)
295 + self.lambda_2 = nn.Parameter(init_values *
296 + torch.ones(config.hidden_size),
297 + requires_grad=True)
298 +
299 + def _init_attn(
300 + self,
301 + config: PretrainedConfig,
302 + quant_config: Optional[QuantizationConfig],
303 + *,
304 + num_dummy_heads: int,
305 + prefix: str = "",
306 + ):
307 + return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)
308 +
309 + def forward(
310 + self,
311 + hidden_states: torch.Tensor,
312 + ):
313 + hidden_states = hidden_states + self.attention(
314 + self.layernorm_before(hidden_states)) * self.lambda_1
315 +
316 + hidden_states = hidden_states + self.mlp(
317 + self.layernorm_after(hidden_states)) * self.lambda_2
318 +
319 + return hidden_states
320 +
321 +
322 +class InternS1VisionEncoder(nn.Module):
323 +
324 + def __init__(
325 + self,
326 + config: PretrainedConfig,
327 + quant_config: Optional[QuantizationConfig] = None,
328 + *,
329 + num_hidden_layers_override: Optional[int] = None,
330 + num_dummy_heads: int = 0,
331 + prefix: str = "",
332 + ):
333 + super().__init__()
334 +
335 + self.config = config
336 +
337 + if num_hidden_layers_override is None:
338 + num_hidden_layers = config.num_hidden_layers
339 + else:
340 + num_hidden_layers = num_hidden_layers_override
341 +
342 + self.layer = nn.ModuleList([
343 + InternS1VisionLayer(config,
344 + quant_config,
345 + num_dummy_heads=num_dummy_heads,
346 + prefix=f"{prefix}.layer.{layer_idx}")
347 + for layer_idx in range(num_hidden_layers)
348 + ])
349 +
350 + def forward(self, inputs_embeds: torch.Tensor):
351 +
352 + hidden_states = inputs_embeds
353 + for encoder_layer in self.layer:
354 + hidden_states = encoder_layer(hidden_states)
355 +
356 + return hidden_states
357 +
358 +
359 +class InternS1VisionModel(nn.Module):
360 +
361 + def __init__(
362 + self,
363 + config: PretrainedConfig,
364 + quant_config: Optional[QuantizationConfig] = None,
365 + *,
366 + num_hidden_layers_override: Optional[int] = None,
367 + num_dummy_heads: int = 0,
368 + prefix: str = "",
369 + ) -> None:
370 + super().__init__()
371 +
372 + self.config = config
373 +
374 + self.embeddings = InternS1VisionEmbeddings(config)
375 + self.encoder = InternS1VisionEncoder(
376 + config=config,
377 + num_hidden_layers_override=num_hidden_layers_override,
378 + num_dummy_heads=num_dummy_heads,
379 + prefix=f"{prefix}.encoder",
380 + )
381 + self.layernorm = (nn.Identity() if config.use_mean_pooling else
382 + nn.LayerNorm(config.hidden_size,
383 + eps=config.layer_norm_eps))
384 +
385 + def get_input_embeddings(self):
386 + return self.embeddings.patch_embeddings
387 +
388 + def forward(
389 + self,
390 + pixel_values: Optional[torch.Tensor] = None,
391 + pixel_embeds: Optional[torch.Tensor] = None,
392 + ) -> torch.FloatTensor:
393 + if pixel_values is None and pixel_embeds is None:
394 + raise ValueError(
395 + 'You have to specify pixel_values or pixel_embeds')
396 +
397 + if pixel_embeds is not None:
398 + hidden_states = pixel_embeds
399 + elif pixel_values is not None:
400 + if pixel_values.ndim == 4:
401 + hidden_states, _ = self.embeddings(pixel_values)
402 + else:
403 + raise ValueError(
404 + f'wrong pixel_values size: {pixel_values.shape}')
405 +
406 + encoder_outputs = self.encoder(inputs_embeds=hidden_states)
407 + encoder_outputs = self.layernorm(encoder_outputs)
408 +
409 + return encoder_outputs
410 +
411 + def load_weights(self, weights: Iterable[tuple[str,
412 + torch.Tensor]]) -> set[str]:
413 + params_dict = dict(self.named_parameters())
414 + loaded_params: set[str] = set()
415 + for name, loaded_weight in weights:
416 + param = params_dict[name]
417 + weight_loader = getattr(param, "weight_loader",
418 + default_weight_loader)
419 + weight_loader(param, loaded_weight)
420 + loaded_params.add(name)
421 + return loaded_params
vllm/model_executor/models/registry.py
+1
@@ -203,6 +203,7 @@ _MULTIMODAL_MODELS = {
203 "GraniteSpeechForConditionalGeneration": ("granite_speech", "GraniteSpeechForConditionalGeneration"), # noqa: E501
204 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
205 "InternVLChatModel": ("internvl", "InternVLChatModel"),
206 + "InternS1ForConditionalGeneration": ("interns1", "InternS1ForConditionalGeneration"), # noqa: E501
207 "Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
208 "SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"), # noqa: E501
209 "KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),