@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6206dcb29e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Model] Add PaliGemma (#5189)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Roger Wang committed Jul 6, 2024 at 18:25 UTC 6206dcb29eb99b3eebf5f00c97a5690c9b7df4f1
6 files changed +557 -2
docs/source/models/supported_models.rst
+4
@@ -186,6 +186,10 @@ Vision Language Models
186 - LLaVA-NeXT
187 - :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
188 -
189 + * - :code:`PaliGemmaForConditionalGeneration`
190 + - PaliGemma
191 + - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, etc.
192 + -
193 * - :code:`Phi3VForCausalLM`
194 - Phi-3-Vision
195 - :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
examples/paligemma_example.py new
+52
@@ -0,0 +1,52 @@
1 +import os
2 +import subprocess
3 +
4 +from PIL import Image
5 +
6 +from vllm import LLM
7 +
8 +# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9 +# You can use `.buildkite/download-images.sh` to download them
10 +
11 +
12 +def run_paligemma():
13 + llm = LLM(model="google/paligemma-3b-mix-224")
14 +
15 + prompt = "caption es"
16 +
17 + image = Image.open("images/stop_sign.jpg")
18 +
19 + outputs = llm.generate({
20 + "prompt": prompt,
21 + "multi_modal_data": {
22 + "image": image
23 + },
24 + })
25 +
26 + for o in outputs:
27 + generated_text = o.outputs[0].text
28 + print(generated_text)
29 +
30 +
31 +def main():
32 + run_paligemma()
33 +
34 +
35 +if __name__ == "__main__":
36 + # Download from s3
37 + s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
38 + local_directory = "images"
39 +
40 + # Make sure the local directory exists or create it
41 + os.makedirs(local_directory, exist_ok=True)
42 +
43 + # Use AWS CLI to sync the directory, assume anonymous access
44 + subprocess.check_call([
45 + "aws",
46 + "s3",
47 + "sync",
48 + s3_bucket_path,
49 + local_directory,
50 + "--no-sign-request",
51 + ])
52 + main()
tests/models/test_paligemma.py new
+147
@@ -0,0 +1,147 @@
1 +from typing import List, Optional, Tuple, Type
2 +
3 +import pytest
4 +from transformers import AutoTokenizer
5 +
6 +from vllm.multimodal.utils import rescale_image_size
7 +from vllm.sequence import SampleLogprobs
8 +
9 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10 +from .utils import check_logprobs_close
11 +
12 +pytestmark = pytest.mark.vlm
13 +
14 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15 + "stop_sign": "caption es",
16 + "cherry_blossom": "What is in the picture?",
17 + "boardwalk": "What is in the picture?",
18 +})
19 +
20 +IMAGE_TOKEN_ID = 257152
21 +
22 +models = ["google/paligemma-3b-mix-224"]
23 +
24 +
25 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
26 + Optional[SampleLogprobs]],
27 + model: str):
28 + """Sanitize vllm output to be comparable with hf output."""
29 + output_ids, output_str, out_logprobs = vllm_output
30 +
31 + tokenizer = AutoTokenizer.from_pretrained(model)
32 + eos_token_id = tokenizer.eos_token_id
33 +
34 + hf_output_ids = [
35 + token_id for idx, token_id in enumerate(output_ids)
36 + if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
37 + ]
38 +
39 + hf_output_str = output_str
40 +
41 + if hf_output_ids[-1] == eos_token_id:
42 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
43 +
44 + return hf_output_ids, hf_output_str, out_logprobs
45 +
46 +
47 +def run_test(
48 + hf_runner: Type[HfRunner],
49 + vllm_runner: Type[VllmRunner],
50 + image_assets: _ImageAssets,
51 + model: str,
52 + *,
53 + size_factors: List[float],
54 + dtype: str,
55 + max_tokens: int,
56 + num_logprobs: int,
57 + tensor_parallel_size: int,
58 + distributed_executor_backend: Optional[str] = None,
59 +):
60 + """Inference result should be the same between hf and vllm.
61 +
62 + All the image fixtures for the test is under tests/images.
63 + For huggingface runner, we provide the PIL images as input.
64 + For vllm runner, we provide MultiModalDataDict objects
65 + and corresponding vision language config as input.
66 + Note, the text input is also adjusted to abide by vllm contract.
67 + The text output is sanitized to be able to compare with hf.
68 + """
69 + images = [asset.pil_image for asset in image_assets]
70 +
71 + inputs_per_image = [(
72 + [prompt for _ in size_factors],
73 + [rescale_image_size(image, factor) for factor in size_factors],
74 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
75 +
76 + # NOTE: take care of the order. run vLLM first, and then run HF.
77 + # vLLM needs a fresh new process without cuda initialization.
78 + # if we run HF first, the cuda initialization will be done and it
79 + # will hurt multiprocessing backend with fork method (the default method).
80 +
81 + # max_model_len should be greater than image_feature_size
82 + with vllm_runner(model,
83 + dtype=dtype,
84 + tensor_parallel_size=tensor_parallel_size,
85 + distributed_executor_backend=distributed_executor_backend,
86 + enforce_eager=True) as vllm_model:
87 + vllm_outputs_per_image = [
88 + vllm_model.generate_greedy_logprobs(prompts,
89 + max_tokens,
90 + num_logprobs=num_logprobs,
91 + images=images)
92 + for prompts, images in inputs_per_image
93 + ]
94 +
95 + with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
96 + hf_outputs_per_image = [
97 + hf_model.generate_greedy_logprobs_limit(prompts,
98 + max_tokens,
99 + num_logprobs=num_logprobs,
100 + images=images)
101 + for prompts, images in inputs_per_image
102 + ]
103 +
104 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
105 + vllm_outputs_per_image):
106 +
107 + check_logprobs_close(
108 + outputs_0_lst=hf_outputs,
109 + outputs_1_lst=[
110 + vllm_to_hf_output(vllm_output, model)
111 + for vllm_output in vllm_outputs
112 + ],
113 + name_0="hf",
114 + name_1="vllm",
115 + )
116 +
117 +
118 +@pytest.mark.parametrize("model", models)
119 +@pytest.mark.parametrize(
120 + "size_factors",
121 + [
122 + # No image
123 + [],
124 + # Single-scale
125 + [1.0],
126 + # Single-scale, batched
127 + [1.0, 1.0, 1.0],
128 + # Multi-scale
129 + [0.25, 0.5, 1.0],
130 + ],
131 +)
132 +@pytest.mark.parametrize("dtype", ["float"])
133 +@pytest.mark.parametrize("max_tokens", [128])
134 +@pytest.mark.parametrize("num_logprobs", [5])
135 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
136 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
137 + run_test(
138 + hf_runner,
139 + vllm_runner,
140 + image_assets,
141 + model,
142 + size_factors=size_factors,
143 + dtype=dtype,
144 + max_tokens=max_tokens,
145 + num_logprobs=num_logprobs,
146 + tensor_parallel_size=1,
147 + )
vllm/model_executor/models/__init__.py
+2
@@ -49,6 +49,8 @@ _GENERATION_MODELS = {
49 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
50 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
51 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
52 + "PaliGemmaForConditionalGeneration":
53 + ("paligemma", "PaliGemmaForConditionalGeneration"),
54 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
55 "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
56 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
vllm/model_executor/models/gemma.py
+8 -2
@@ -268,16 +268,22 @@ class GemmaModel(nn.Module):
268 normalizer = self.config.hidden_size**0.5
269 self.register_buffer("normalizer", torch.tensor(normalizer))
270
271 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
272 + return self.embed_tokens(input_ids)
273 +
274 def forward(
275 self,
276 input_ids: torch.Tensor,
277 positions: torch.Tensor,
278 kv_caches: List[torch.Tensor],
279 attn_metadata: AttentionMetadata,
280 + inputs_embeds: Optional[torch.Tensor] = None,
281 ) -> torch.Tensor:
278 - hidden_states = self.embed_tokens(input_ids)
282 + if inputs_embeds is not None:
283 + hidden_states = inputs_embeds
284 + else:
285 + hidden_states = self.get_input_embeddings(input_ids)
286 hidden_states *= self.normalizer
280 -
287 residual = None
288 for i in range(len(self.layers)):
289 layer = self.layers[i]
vllm/model_executor/models/paligemma.py new
+344
@@ -0,0 +1,344 @@
1 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2 +
3 +import torch
4 +from PIL import Image
5 +from torch import nn
6 +from transformers import PaliGemmaConfig, SiglipVisionConfig, SiglipVisionModel
7 +
8 +from vllm.attention import AttentionMetadata
9 +from vllm.config import CacheConfig, MultiModalConfig
10 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11 +from vllm.logger import init_logger
12 +from vllm.model_executor.layers.linear import ColumnParallelLinear
13 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
14 +from vllm.model_executor.layers.quantization.base_config import (
15 + QuantizationConfig)
16 +from vllm.model_executor.layers.sampler import Sampler
17 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18 +from vllm.model_executor.models.gemma import GemmaModel
19 +from vllm.model_executor.sampling_metadata import SamplingMetadata
20 +from vllm.multimodal import MULTIMODAL_REGISTRY
21 +from vllm.multimodal.image import cached_get_tokenizer
22 +from vllm.sequence import SamplerOutput, SequenceData
23 +
24 +from .interfaces import SupportsVision
25 +from .utils import merge_vision_embeddings
26 +
27 +logger = init_logger(__name__)
28 +
29 +_KEYS_TO_MODIFY_MAPPING = {
30 + "language_model.model": "language_model",
31 +}
32 +
33 +
34 +def get_max_paligemma_image_tokens(ctx: InputContext):
35 + hf_config = ctx.get_hf_config(PaliGemmaConfig)
36 + text_config = hf_config.text_config
37 +
38 + return text_config.num_image_tokens
39 +
40 +
41 +def dummy_seq_data_for_paligemma(
42 + hf_config: PaliGemmaConfig,
43 + seq_len: int,
44 + *,
45 + image_token_id: int,
46 + image_feature_size_override: Optional[int] = None,
47 +):
48 + if image_feature_size_override is None:
49 + image_feature_size = hf_config.text_config.num_image_tokens
50 + else:
51 + image_feature_size = image_feature_size_override
52 +
53 + token_ids = [image_token_id] * image_feature_size
54 + token_ids += [0] * (seq_len - image_feature_size)
55 + return SequenceData(token_ids)
56 +
57 +
58 +def dummy_image_for_paligemma(
59 + hf_config: SiglipVisionConfig,
60 + *,
61 + image_width_override: Optional[int] = None,
62 + image_height_override: Optional[int] = None,
63 +):
64 + width = height = hf_config.image_size
65 + if image_width_override is not None:
66 + width = image_width_override
67 + if image_height_override is not None:
68 + height = image_height_override
69 +
70 + image = Image.new("RGB", (width, height), color=0)
71 + return {"image": image}
72 +
73 +
74 +def dummy_data_for_paligemma(ctx: InputContext, seq_len: int):
75 + hf_config = ctx.get_hf_config(PaliGemmaConfig)
76 + vision_config = hf_config.vision_config
77 +
78 + seq_data = dummy_seq_data_for_paligemma(
79 + hf_config,
80 + seq_len,
81 + image_token_id=hf_config.image_token_index,
82 + )
83 +
84 + mm_data = dummy_image_for_paligemma(vision_config)
85 + return seq_data, mm_data
86 +
87 +
88 +def input_processor_for_paligemma(ctx: InputContext, llm_inputs: LLMInputs):
89 +
90 + """
91 + The correct prompt format needs to be:
92 + '<image>' * image_feature_size + '<bos>' + prompt + '\n'
93 +
94 + See https://github.com/huggingface/transformers/blob/25245ec26dc29bcf6102e1b4ddd0dfd02e720cf5/src/transformers/models/paligemma/processing_paligemma.py#L55
95 + """ # noqa
96 +
97 + multi_modal_data = llm_inputs.get("multi_modal_data")
98 + if multi_modal_data is None or "image" not in multi_modal_data:
99 + return llm_inputs
100 +
101 + model_config = ctx.model_config
102 + hf_config = ctx.get_hf_config(PaliGemmaConfig)
103 +
104 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
105 + image_feature_size = hf_config.text_config.num_image_tokens
106 + image_token_str = tokenizer.decode(hf_config.image_token_index)
107 + bos_token = tokenizer.decode(hf_config.bos_token_id)
108 + image_token_str_pad = image_token_str * image_feature_size
109 + image_token_ids_pad = [hf_config.image_token_index] * image_feature_size
110 +
111 + orig_prompt = llm_inputs.get("prompt")
112 + orig_prompt_ids = llm_inputs.get("prompt_token_ids")
113 +
114 + if image_token_str in orig_prompt:
115 + logger.warning(
116 + "The image token '%s' was detected in the prompt and "
117 + "will be removed. Please follow the proper prompt format"
118 + " documented on HuggingFace.", image_token_str)
119 + orig_prompt = orig_prompt.replace(image_token_str, "")
120 + orig_prompt_ids.remove(hf_config.image_token_index)
121 +
122 + new_prompt = f"{image_token_str_pad}{bos_token}{orig_prompt}\n"
123 + new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline
124 +
125 + # NOTE: Create a defensive copy of the original inputs
126 + return LLMInputs(prompt_token_ids=new_token_ids,
127 + prompt=new_prompt,
128 + multi_modal_data=multi_modal_data)
129 +
130 +
131 +class PaliGemmaMultiModalProjector(nn.Module):
132 +
133 + def __init__(self, vision_hidden_size: int, projection_dim: int):
134 + super().__init__()
135 +
136 + self.linear = ColumnParallelLinear(vision_hidden_size,
137 + projection_dim,
138 + bias=True)
139 +
140 + def forward(self, image_features: torch.Tensor) -> torch.Tensor:
141 + hidden_states, _ = self.linear(image_features)
142 + return hidden_states
143 +
144 +
145 +class PaliGemmaImagePixelInputs(TypedDict):
146 + type: Literal["pixel_values"]
147 + data: torch.Tensor
148 + """Shape: (batch_size, num_channels, height, width)"""
149 +
150 +
151 +PaliGemmaImageInputs = PaliGemmaImagePixelInputs
152 +
153 +
154 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
155 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_paligemma_image_tokens)
156 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma)
157 +@INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma)
158 +class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
159 +
160 + def __init__(self,
161 + config: PaliGemmaConfig,
162 + multimodal_config: MultiModalConfig,
163 + cache_config: Optional[CacheConfig] = None,
164 + quant_config: Optional[QuantizationConfig] = None) -> None:
165 + super().__init__()
166 +
167 + self.config = config
168 + self.multimodal_config = multimodal_config
169 +
170 + # TODO(ywang96): Port over SiglipVisionModel & TP
171 + self.vision_tower = SiglipVisionModel(config.vision_config)
172 + self.multi_modal_projector = PaliGemmaMultiModalProjector(
173 + vision_hidden_size=config.vision_config.hidden_size,
174 + projection_dim=config.vision_config.projection_dim)
175 +
176 + self.quant_config = quant_config
177 + self.language_model = GemmaModel(config.text_config, cache_config,
178 + quant_config)
179 + self.unpadded_vocab_size = config.text_config.vocab_size
180 + logit_scale = getattr(config, "logit_scale", 1.0)
181 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
182 + config.vocab_size, logit_scale)
183 + self.sampler = Sampler()
184 +
185 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
186 + h = w = self.config.vision_config.image_size
187 + expected_dims = (3, h, w)
188 + actual_dims = tuple(data.shape[1:])
189 +
190 + if actual_dims != expected_dims:
191 + expected_expr = ("batch_size", *map(str, expected_dims))
192 + raise ValueError(
193 + f"The expected shape of pixel values is {expected_expr}. "
194 + f"You supplied {tuple(data.shape)}.")
195 +
196 + return data
197 +
198 + def _parse_and_validate_image_input(
199 + self, **kwargs: object) -> Optional[PaliGemmaImageInputs]:
200 + pixel_values = kwargs.pop("pixel_values", None)
201 +
202 + if pixel_values is None:
203 + return None
204 +
205 + if not isinstance(pixel_values, torch.Tensor):
206 + raise ValueError("Incorrect type of pixel values. "
207 + f"Got type: {type(pixel_values)}")
208 +
209 + return PaliGemmaImagePixelInputs(
210 + type="pixel_values",
211 + data=self._validate_pixel_values(pixel_values),
212 + )
213 +
214 + def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
215 + pixel_values: torch.Tensor) -> torch.Tensor:
216 +
217 + image_outputs = vision_tower(pixel_values, output_hidden_states=True)
218 +
219 + selected_image_features = image_outputs.last_hidden_state
220 +
221 + return selected_image_features
222 +
223 + def _process_image_pixels(
224 + self, inputs: PaliGemmaImagePixelInputs) -> torch.Tensor:
225 + assert self.vision_tower is not None
226 +
227 + pixel_values = inputs["data"]
228 +
229 + return self._image_pixels_to_features(self.vision_tower, pixel_values)
230 +
231 + def _process_image_input(
232 + self, image_input: PaliGemmaImageInputs) -> torch.Tensor:
233 +
234 + assert self.vision_tower is not None
235 + image_features = self._process_image_pixels(image_input)
236 +
237 + return self.multi_modal_projector(image_features)
238 +
239 + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
240 + kv_caches: List[torch.Tensor],
241 + attn_metadata: AttentionMetadata,
242 + **kwargs: object) -> SamplerOutput:
243 +
244 + parsed_image_input = self._parse_and_validate_image_input(**kwargs)
245 +
246 + if parsed_image_input is not None:
247 + vision_embeddings = self._process_image_input(parsed_image_input)
248 + # https://github.com/huggingface/transformers/blob/main/src/transformers/models/paligemma/modeling_paligemma.py#L294 # noqa
249 + vision_embeddings = vision_embeddings * (self.config.hidden_size**
250 + -0.5)
251 +
252 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
253 +
254 + inputs_embeds = merge_vision_embeddings(
255 + input_ids, inputs_embeds, vision_embeddings,
256 + self.config.image_token_index)
257 +
258 + input_ids = None
259 + else:
260 + inputs_embeds = None
261 +
262 + hidden_states = self.language_model(input_ids,
263 + positions,
264 + kv_caches,
265 + attn_metadata,
266 + inputs_embeds=inputs_embeds)
267 +
268 + return hidden_states
269 +
270 + # Copied from vllm/model_executor/models/gemma.py
271 + def compute_logits(self, hidden_states: torch.Tensor,
272 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
273 + logits = self.logits_processor(self.language_model.embed_tokens,
274 + hidden_states, sampling_metadata)
275 + return logits
276 +
277 + # Copied from vllm/model_executor/models/gemma.py
278 + def sample(
279 + self,
280 + logits: torch.Tensor,
281 + sampling_metadata: SamplingMetadata,
282 + ) -> Optional[SamplerOutput]:
283 + next_tokens = self.sampler(logits, sampling_metadata)
284 + return next_tokens
285 +
286 + # Adapted from vllm/model_executor/models/gemma.py
287 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
288 + stacked_params_mapping = [
289 + # (param_name, shard_name, shard_id)
290 + ("qkv_proj", "q_proj", "q"),
291 + ("qkv_proj", "k_proj", "k"),
292 + ("qkv_proj", "v_proj", "v"),
293 + ("gate_up_proj", "gate_proj", 0),
294 + ("gate_up_proj", "up_proj", 1),
295 + ]
296 + params_dict = dict(self.named_parameters())
297 + loaded_params = set()
298 + for name, loaded_weight in weights:
299 + for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
300 + if key_to_modify in name:
301 + name = name.replace(key_to_modify, new_key)
302 + use_default_weight_loading = False
303 + if "vision" in name:
304 + if self.vision_tower is not None:
305 + # We only do sharding for language model and
306 + # not vision model for now.
307 + use_default_weight_loading = True
308 + else:
309 + for (param_name, shard_name,
310 + shard_id) in stacked_params_mapping:
311 + if shard_name not in name:
312 + continue
313 + name = name.replace(shard_name, param_name)
314 + # Skip loading extra bias for GPTQ models.
315 + if name.endswith(".bias") and name not in params_dict:
316 + continue
317 + param = params_dict[name]
318 + weight_loader = param.weight_loader
319 + weight_loader(param, loaded_weight, shard_id)
320 + break
321 + else:
322 + # lm_head is not used in vllm as it is tied with
323 + # embed_token. To prevent errors, skip loading
324 + # lm_head.weight.
325 + if "lm_head.weight" in name:
326 + continue
327 + # Skip loading extra bias for GPTQ models.
328 + if name.endswith(".bias") and name not in params_dict:
329 + continue
330 + use_default_weight_loading = True
331 +
332 + if use_default_weight_loading:
333 + param = params_dict[name]
334 + weight_loader = getattr(param, "weight_loader",
335 + default_weight_loader)
336 + weight_loader(param, loaded_weight)
337 +
338 + loaded_params.add(name)
339 +
340 + unloaded_params = params_dict.keys() - loaded_params
341 + if unloaded_params:
342 + raise RuntimeError(
343 + "Some weights are not initialized from checkpoints: "
344 + f"{unloaded_params}")