@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa253f1a70
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[VLM] Remove input processor from clip and siglip (#13165)

Isotr0py committed Feb 13, 2025 at 16:31 UTC fa253f1a702511e35dbbe14ac7b144def7f12d7d
2 files changed +10 -213
vllm/model_executor/models/clip.py
+8 -141
@@ -1,156 +1,24 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """Minimal implementation of CLIPVisionModel intended to be only used
3 within a vision language model."""
4 -from typing import Iterable, List, Optional, Set, Tuple, Union
4 +from typing import Iterable, Optional, Set, Tuple, Union
5
6 -import numpy as np
6 import torch
7 import torch.nn as nn
9 -from PIL import Image
8 from transformers import CLIPVisionConfig
9
10 from vllm.attention.layer import MultiHeadAttention
13 -from vllm.config import ModelConfig
11 from vllm.distributed import divide, get_tensor_model_parallel_world_size
15 -from vllm.inputs import DecoderOnlyInputs, token_inputs
12 from vllm.model_executor.layers.activation import get_act_fn
13 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 QKVParallelLinear,
15 RowParallelLinear)
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22 -from vllm.multimodal.utils import (cached_get_tokenizer,
23 - consecutive_placeholder_ranges,
24 - repeat_and_pad_placeholder_tokens)
25 -from vllm.sequence import SequenceData
18
19 from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
20
21
30 -def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
31 - assert image_size % patch_size == 0
32 - return image_size // patch_size
33 -
34 -
35 -def get_clip_num_patches(*, image_size: int, patch_size: int) -> int:
36 - grid_length = get_clip_patch_grid_length(image_size=image_size,
37 - patch_size=patch_size)
38 - return grid_length * grid_length
39 -
40 -
41 -def get_clip_image_feature_size(hf_config: CLIPVisionConfig) -> int:
42 - return get_clip_num_patches(image_size=hf_config.image_size,
43 - patch_size=hf_config.patch_size) + 1
44 -
45 -
46 -def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
47 - return get_clip_image_feature_size(hf_config)
48 -
49 -
50 -def dummy_seq_data_for_clip(hf_config: CLIPVisionConfig,
51 - seq_len: int,
52 - num_images: int,
53 - *,
54 - image_token_id: int,
55 - image_feature_size_override: Optional[int] = None,
56 - mm_key: str = "image"):
57 - if image_feature_size_override is None:
58 - image_feature_size = get_clip_image_feature_size(hf_config)
59 - else:
60 - image_feature_size = image_feature_size_override
61 -
62 - return SequenceData.from_prompt_token_counts(
63 - (image_token_id, image_feature_size * num_images),
64 - (0, seq_len - image_feature_size * num_images),
65 - ), {
66 - mm_key:
67 - consecutive_placeholder_ranges(num_items=num_images,
68 - item_size=image_feature_size)
69 - }
70 -
71 -
72 -def dummy_image_for_clip(
73 - hf_config: CLIPVisionConfig,
74 - num_images: int,
75 - *,
76 - image_width_override: Optional[int] = None,
77 - image_height_override: Optional[int] = None,
78 -):
79 - width = height = hf_config.image_size
80 - if image_width_override is not None:
81 - width = image_width_override
82 - if image_height_override is not None:
83 - height = image_height_override
84 -
85 - image = Image.new("RGB", (width, height), color=0)
86 - return {"image": image if num_images == 1 else [image] * num_images}
87 -
88 -
89 -def dummy_video_for_clip(
90 - hf_config: CLIPVisionConfig,
91 - num_frames: int,
92 - num_videos: int = 1,
93 - *,
94 - image_width_override: Optional[int] = None,
95 - image_height_override: Optional[int] = None,
96 -):
97 - pil_frame = dummy_image_for_clip(
98 - hf_config,
99 - num_images=1,
100 - image_width_override=image_width_override,
101 - image_height_override=image_height_override)
102 - np_frame = np.array(pil_frame["image"])
103 - mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
104 - video_data = [mm_data_per_video] * num_videos
105 - mm_data = {"video": video_data}
106 - return mm_data
107 -
108 -
109 -def input_processor_for_clip(
110 - model_config: ModelConfig,
111 - hf_config: CLIPVisionConfig,
112 - inputs: DecoderOnlyInputs,
113 - *,
114 - image_token_id: int,
115 - image_feature_size_override: Optional[Union[int, List[int]]] = None,
116 -):
117 - multi_modal_data = inputs.get("multi_modal_data")
118 - if multi_modal_data is None or "image" not in multi_modal_data:
119 - return inputs
120 -
121 - if "multi_modal_placeholders" in inputs and "image" in inputs[
122 - "multi_modal_placeholders"]:
123 - # The inputs already have placeholders.
124 - return inputs
125 -
126 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
127 -
128 - if image_feature_size_override is None:
129 - image_data = multi_modal_data["image"]
130 - if isinstance(image_data, Image.Image):
131 - image_feature_size = get_clip_image_feature_size(hf_config)
132 - elif isinstance(image_data, torch.Tensor):
133 - num_images, image_feature_size, hidden_size = image_data.shape
134 - else:
135 - raise TypeError(f"Invalid image type: {type(image_data)}")
136 - else:
137 - image_feature_size = image_feature_size_override
138 -
139 - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
140 - tokenizer,
141 - inputs.get("prompt"),
142 - inputs["prompt_token_ids"],
143 - placeholder_token_id=image_token_id,
144 - repeat_count=image_feature_size,
145 - )
146 -
147 - # NOTE: Create a defensive copy of the original inputs
148 - return token_inputs(prompt_token_ids=new_token_ids,
149 - prompt=new_prompt,
150 - multi_modal_data=multi_modal_data,
151 - multi_modal_placeholders={"image": ranges})
152 -
153 -
22 class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
23
24 def get_num_image_tokens(
@@ -159,10 +27,10 @@ class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
27 image_width: int,
28 image_height: int,
29 ) -> int:
162 - return get_clip_image_feature_size(self.vision_config)
30 + return self.get_patch_grid_length()**2 + 1
31
32 def get_max_image_tokens(self) -> int:
165 - return get_max_clip_image_tokens(self.vision_config)
33 + return self.get_patch_grid_length()**2 + 1
34
35 def get_image_size(self) -> int:
36 return self.vision_config.image_size
@@ -171,10 +39,9 @@ class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
39 return self.vision_config.patch_size
40
41 def get_patch_grid_length(self) -> int:
174 - return get_clip_patch_grid_length(
175 - image_size=self.vision_config.image_size,
176 - patch_size=self.vision_config.patch_size,
177 - )
42 + image_size, patch_size = self.get_image_size(), self.get_patch_size()
43 + assert image_size % patch_size == 0
44 + return image_size // patch_size
45
46
47 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
@@ -186,6 +53,7 @@ class CLIPVisionEmbeddings(nn.Module):
53 self.embed_dim = config.hidden_size
54 self.image_size = config.image_size
55 self.patch_size = config.patch_size
56 + assert self.image_size % self.patch_size == 0
57
58 self.class_embedding = nn.Parameter(torch.randn(self.embed_dim))
59
@@ -197,8 +65,7 @@ class CLIPVisionEmbeddings(nn.Module):
65 bias=False,
66 )
67
200 - self.num_patches = get_clip_num_patches(image_size=self.image_size,
201 - patch_size=self.patch_size)
68 + self.num_patches = (self.image_size // self.patch_size)**2
69 self.num_positions = self.num_patches + 1
70 self.position_embedding = nn.Embedding(self.num_positions,
71 self.embed_dim)
vllm/model_executor/models/siglip.py
+2 -72
@@ -3,18 +3,15 @@
3 within a vision language model."""
4
5 import math
6 -from typing import Iterable, List, Optional, Set, Tuple, Union
6 +from typing import Iterable, Optional, Set, Tuple, Union
7
8 -import numpy as np
8 import torch
9 from PIL import Image
10 from torch import nn
11 from transformers import SiglipVisionConfig
12
13 from vllm.attention.layer import MultiHeadAttention
15 -from vllm.config import ModelConfig
14 from vllm.distributed import divide, get_tensor_model_parallel_world_size
17 -from vllm.inputs import DecoderOnlyInputs, token_inputs
15 from vllm.model_executor.layers.activation import get_act_fn
16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 QKVParallelLinear,
@@ -23,9 +20,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 VocabParallelEmbedding)
22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26 -from vllm.multimodal.utils import (cached_get_tokenizer,
27 - consecutive_placeholder_ranges,
28 - repeat_and_pad_placeholder_tokens)
23 +from vllm.multimodal.utils import consecutive_placeholder_ranges
24 from vllm.sequence import SequenceData
25
26 from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
@@ -93,71 +88,6 @@ def dummy_image_for_siglip(
88 return {"image": image if num_images == 1 else [image] * num_images}
89
90
96 -def dummy_video_for_siglip(
97 - hf_config: SiglipVisionConfig,
98 - num_frames: int,
99 - num_videos: int = 1,
100 - *,
101 - image_width_override: Optional[int] = None,
102 - image_height_override: Optional[int] = None,
103 -):
104 - pil_frame = dummy_image_for_siglip(
105 - hf_config,
106 - num_images=1,
107 - image_width_override=image_width_override,
108 - image_height_override=image_height_override)
109 - np_frame = np.array(pil_frame["image"])
110 - mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
111 - video_data = [mm_data_per_video] * num_videos
112 - mm_data = {"video": video_data}
113 - return mm_data
114 -
115 -
116 -def input_processor_for_siglip(
117 - model_config: ModelConfig,
118 - hf_config: SiglipVisionConfig,
119 - inputs: DecoderOnlyInputs,
120 - *,
121 - image_token_id: int,
122 - image_feature_size_override: Optional[Union[int, List[int]]] = None,
123 -):
124 - multi_modal_data = inputs.get("multi_modal_data")
125 - if multi_modal_data is None or "image" not in multi_modal_data:
126 - return inputs
127 -
128 - if "multi_modal_placeholders" in inputs and "image" in inputs[
129 - "multi_modal_placeholders"]:
130 - # The inputs already have placeholders.
131 - return inputs
132 -
133 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
134 -
135 - if image_feature_size_override is None:
136 - image_data = multi_modal_data["image"]
137 - if isinstance(image_data, Image.Image):
138 - image_feature_size = get_siglip_image_feature_size(hf_config)
139 - elif isinstance(image_data, torch.Tensor):
140 - num_images, image_feature_size, hidden_size = image_data.shape
141 - else:
142 - raise TypeError(f"Invalid image type: {type(image_data)}")
143 - else:
144 - image_feature_size = image_feature_size_override
145 -
146 - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
147 - tokenizer,
148 - inputs.get("prompt"),
149 - inputs["prompt_token_ids"],
150 - placeholder_token_id=image_token_id,
151 - repeat_count=image_feature_size,
152 - )
153 -
154 - # NOTE: Create a defensive copy of the original inputs
155 - return token_inputs(prompt_token_ids=new_token_ids,
156 - prompt=new_prompt,
157 - multi_modal_data=multi_modal_data,
158 - multi_modal_placeholders={"image": ranges})
159 -
160 -
91 class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
92
93 def get_num_image_tokens(