A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[VLM] Cleanup siglip legacy code and fix broken paligemma multimodal processor (#14602)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Mar 11, 2025 at 19:27 UTC
1477ffc38176791b0ab8f7a87a5475530ff7223f
2 files changed
+14
-76
vllm/model_executor/models/paligemma.py
+10
-5
@@ -24,9 +24,10 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
24
from vllm.sequence import IntermediateTensors
25
26
from .interfaces import SupportsMultiModal, SupportsPP
27
-from .siglip import SiglipVisionModel, get_max_siglip_image_tokens
27
+from .siglip import SiglipVisionModel
28
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
29
maybe_prefix, merge_multimodal_embeddings)
30
+from .vision import get_vision_encoder_info
31
32
logger = init_logger(__name__)
33
@@ -67,6 +68,9 @@ class PaliGemmaProcessingInfo(BaseProcessingInfo):
68
def get_hf_config(self):
69
return self.ctx.get_hf_config(PaliGemmaConfig)
70
71
+ def get_vision_encoder_info(self):
72
+ return get_vision_encoder_info(self.get_hf_config())
73
+
74
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
75
return {"image": 1}
76
@@ -78,9 +82,8 @@ class PaliGemmaProcessingInfo(BaseProcessingInfo):
82
return {"image": self.get_num_image_tokens()}
83
84
def get_num_image_tokens(self) -> int:
81
- hf_config = self.get_hf_config()
82
- vision_config = hf_config.vision_config
83
- return get_max_siglip_image_tokens(vision_config)
85
+ vision_encoder_info = self.get_vision_encoder_info()
86
+ return vision_encoder_info.get_max_image_tokens()
87
88
89
class PaliGemmaDummyInputsBuilder(
@@ -173,8 +176,10 @@ class PaliGemmaMultiModalProcessor(
176
prompt: Union[str, list[int]],
177
mm_data: MultiModalDataDict,
178
hf_processor_mm_kwargs: Mapping[str, object],
179
+ return_mm_hashes: bool = False,
180
) -> MultiModalInputs:
177
- mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
181
+ mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,
182
+ return_mm_hashes)
183
prompt_token_ids = mm_inputs["prompt_token_ids"]
184
185
tokenizer = self.info.get_tokenizer()
vllm/model_executor/models/siglip.py
+4
-71
@@ -6,7 +6,6 @@ import math
6
from typing import Iterable, Optional, Set, Tuple, Union
7
8
import torch
9
-from PIL import Image
9
from torch import nn
10
from transformers import SiglipVisionConfig
11
@@ -20,74 +19,10 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
VocabParallelEmbedding)
21
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
-from vllm.multimodal.utils import consecutive_placeholder_ranges
24
-from vllm.sequence import SequenceData
22
23
from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
24
25
29
-def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
30
- # Since interpolation is applied, the image size need not be divisible
31
- # assert image_size % patch_size == 0
32
- return image_size // patch_size
33
-
34
-
35
-def get_siglip_num_patches(*, image_size: int, patch_size: int) -> int:
36
- grid_length = get_siglip_patch_grid_length(image_size=image_size,
37
- patch_size=patch_size)
38
- return grid_length * grid_length
39
-
40
-
41
-def get_siglip_image_feature_size(hf_config: SiglipVisionConfig) -> int:
42
- return get_siglip_num_patches(image_size=hf_config.image_size,
43
- patch_size=hf_config.patch_size)
44
-
45
-
46
-def get_max_siglip_image_tokens(hf_config: SiglipVisionConfig) -> int:
47
- return get_siglip_image_feature_size(hf_config)
48
-
49
-
50
-def dummy_seq_data_for_siglip(
51
- hf_config: SiglipVisionConfig,
52
- seq_len: int,
53
- num_images: int,
54
- *,
55
- image_token_id: int,
56
- image_feature_size_override: Optional[int] = None,
57
- mm_key: str = "image",
58
-):
59
- if image_feature_size_override is None:
60
- image_feature_size = get_siglip_image_feature_size(hf_config)
61
- else:
62
- image_feature_size = image_feature_size_override
63
-
64
- return SequenceData.from_prompt_token_counts(
65
- (image_token_id, image_feature_size * num_images),
66
- (0, seq_len - image_feature_size * num_images),
67
- ), {
68
- mm_key:
69
- consecutive_placeholder_ranges(num_items=num_images,
70
- item_size=image_feature_size)
71
- }
72
-
73
-
74
-def dummy_image_for_siglip(
75
- hf_config: SiglipVisionConfig,
76
- num_images: int,
77
- *,
78
- image_width_override: Optional[int] = None,
79
- image_height_override: Optional[int] = None,
80
-):
81
- width = height = hf_config.image_size
82
- if image_width_override is not None:
83
- width = image_width_override
84
- if image_height_override is not None:
85
- height = image_height_override
86
-
87
- image = Image.new("RGB", (width, height), color=0)
88
- return {"image": image if num_images == 1 else [image] * num_images}
89
-
90
-
26
class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
27
28
def get_num_image_tokens(
@@ -96,10 +31,10 @@ class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
31
image_width: int,
32
image_height: int,
33
) -> int:
99
- return get_siglip_image_feature_size(self.vision_config)
34
+ return self.get_patch_grid_length()**2
35
36
def get_max_image_tokens(self) -> int:
102
- return get_max_siglip_image_tokens(self.vision_config)
37
+ return self.get_patch_grid_length()**2
38
39
def get_image_size(self) -> int:
40
return self.vision_config.image_size
@@ -108,10 +43,8 @@ class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
43
return self.vision_config.patch_size
44
45
def get_patch_grid_length(self) -> int:
111
- return get_siglip_patch_grid_length(
112
- image_size=self.vision_config.image_size,
113
- patch_size=self.vision_config.patch_size,
114
- )
46
+ image_size, patch_size = self.get_image_size(), self.get_patch_size()
47
+ return image_size // patch_size
48
49
50
# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa