@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5340a2dccf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Model] Add multi-image input support for LLaVA-Next offline inference (#7230)

zifeitong committed Aug 27, 2024 at 16:09 UTC 5340a2dccf06f502821b82db187a850ce566d07c
7 files changed +174 -52
tests/conftest.py
+10 -11
@@ -41,6 +41,10 @@ _TEST_DIR = os.path.dirname(__file__)
41 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
42 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
43
44 +PromptImageInput = Union[List[Image.Image], List[List[Image.Image]]]
45 +PromptAudioInput = Union[List[Tuple[np.ndarray, int]],
46 + List[List[Tuple[np.ndarray, int]]]]
47 +
48
49 def _read_prompts(filename: str) -> List[str]:
50 with open(filename, "r") as f:
@@ -161,7 +165,7 @@ def example_encoder_decoder_prompts(
165 decoder prompt) tuple.
166
167 Returns:
164 -
168 +
169 * Encoder prompt list
170 * Decoder prompt list (reverse of encoder prompt list)
171 '''
@@ -578,8 +582,7 @@ class VllmRunner:
582 self,
583 prompts: List[str],
584 sampling_params: SamplingParams,
581 - images: Optional[Union[List[Image.Image],
582 - List[List[Image.Image]]]] = None,
585 + images: Optional[PromptImageInput] = None,
586 ) -> List[Tuple[List[List[int]], List[str]]]:
587 if images is not None:
588 assert len(prompts) == len(images)
@@ -623,10 +626,8 @@ class VllmRunner:
626 self,
627 prompts: List[str],
628 sampling_params: SamplingParams,
626 - images: Optional[Union[List[Image.Image],
627 - List[List[Image.Image]]]] = None,
628 - audios: Optional[Union[List[Tuple[np.ndarray, int]],
629 - List[List[Tuple[np.ndarray, int]]]]] = None
629 + images: Optional[PromptImageInput] = None,
630 + audios: Optional[PromptAudioInput] = None,
631 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
632 assert sampling_params.logprobs is not None
633
@@ -676,10 +677,8 @@ class VllmRunner:
677 prompts: List[str],
678 max_tokens: int,
679 num_logprobs: int,
679 - images: Optional[Union[List[Image.Image],
680 - List[List[Image.Image]]]] = None,
681 - audios: Optional[Union[List[Tuple[np.ndarray, int]],
682 - List[List[Tuple[np.ndarray, int]]]]] = None,
680 + images: Optional[PromptImageInput] = None,
681 + audios: Optional[PromptAudioInput] = None,
682 stop_token_ids: Optional[List[int]] = None,
683 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
684 greedy_logprobs_params = SamplingParams(temperature=0.0,
tests/models/test_llava_next.py
+80 -13
@@ -6,24 +6,22 @@ from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
8
9 -from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9 +from ..conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
10 + _ImageAssets)
11 from .utils import check_logprobs_close
12
13 pytestmark = pytest.mark.vlm
14
14 -_PREFACE = (
15 - "A chat between a curious human and an artificial intelligence assistant. "
16 - "The assistant gives helpful, detailed, and polite answers to the human's "
17 - "questions.")
15 +_LIMIT_IMAGE_PER_PROMPT = 4
16
17 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18 "stop_sign":
21 - f"{_PREFACE} USER: <image>\nWhat's the content of the image? ASSISTANT:",
19 + "[INST] <image>\nWhat's the content of the image? [/INST]",
20 "cherry_blossom":
23 - f"{_PREFACE} USER: <image>\nWhat is the season? ASSISTANT:",
21 + "[INST] <image>\nWhat is the season? [/INST]",
22 })
23
26 -models = ["llava-hf/llava-v1.6-vicuna-7b-hf"]
24 +models = ["llava-hf/llava-v1.6-mistral-7b-hf"]
25
26
27 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
@@ -114,19 +112,43 @@ def run_test(
112 else:
113 raise ValueError("You must provide either `size_factors` or `sizes`")
114
115 + _run_test(hf_runner,
116 + vllm_runner,
117 + inputs_per_image,
118 + model,
119 + dtype=dtype,
120 + max_tokens=max_tokens,
121 + num_logprobs=num_logprobs,
122 + tensor_parallel_size=tensor_parallel_size,
123 + distributed_executor_backend=distributed_executor_backend)
124 +
125 +
126 +def _run_test(
127 + hf_runner: Type[HfRunner],
128 + vllm_runner: Type[VllmRunner],
129 + inputs: List[Tuple[List[str], PromptImageInput]],
130 + model: str,
131 + dtype: str,
132 + max_tokens: int,
133 + num_logprobs: int,
134 + tensor_parallel_size: int,
135 + distributed_executor_backend: Optional[str] = None,
136 +):
137 # max_model_len should be greater than image_feature_size
138 with vllm_runner(model,
139 dtype=dtype,
120 - max_model_len=4096,
140 + max_model_len=10240,
141 tensor_parallel_size=tensor_parallel_size,
142 distributed_executor_backend=distributed_executor_backend,
123 - enforce_eager=True) as vllm_model:
143 + enforce_eager=True,
144 + limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
145 + }) as vllm_model:
146 vllm_outputs_per_image = [
147 vllm_model.generate_greedy_logprobs(prompts,
148 max_tokens,
149 num_logprobs=num_logprobs,
150 images=images)
129 - for prompts, images in inputs_per_image
151 + for prompts, images in inputs
152 ]
153
154 with hf_runner(model, dtype=dtype,
@@ -136,7 +158,7 @@ def run_test(
158 max_tokens,
159 num_logprobs=num_logprobs,
160 images=images)
139 - for prompts, images in inputs_per_image
161 + for prompts, images in inputs
162 ]
163
164 for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
@@ -177,7 +199,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
199
200 All the image fixtures for the test is under tests/images.
201 For huggingface runner, we provide the PIL images as input.
180 - For vllm runner, we provide MultiModalDataDict objects
202 + For vllm runner, we provide MultiModalDataDict objects
203 and corresponding MultiModalConfig as input.
204 Note, the text input is also adjusted to abide by vllm contract.
205 The text output is sanitized to be able to compare with hf.
@@ -216,3 +238,48 @@ def test_models_fixed_sizes(hf_runner, vllm_runner, image_assets, model, sizes,
238 num_logprobs=num_logprobs,
239 tensor_parallel_size=1,
240 )
241 +
242 +
243 +@pytest.mark.parametrize("model", models)
244 +@pytest.mark.parametrize("dtype", ["half"])
245 +@pytest.mark.parametrize("max_tokens", [128])
246 +@pytest.mark.parametrize("num_logprobs", [5])
247 +def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
248 + model, dtype, max_tokens,
249 + num_logprobs) -> None:
250 + stop_sign = image_assets[0].pil_image
251 + cherry_blossom = image_assets[1].pil_image
252 +
253 + inputs = [(
254 + [
255 + "[INST] <image><image>\nDescribe 2 images. [/INST]",
256 + "[INST] <image><image>\nDescribe 2 images. [/INST]",
257 + "[INST] <image><image><image><image>\nDescribe 4 images. [/INST]",
258 + "[INST] <image>\nWhat is the season? [/INST]"
259 + ],
260 + [
261 + [stop_sign, cherry_blossom],
262 + # Images with different sizes and aspect-ratios
263 + [
264 + rescale_image_size(stop_sign, 0.1),
265 + stop_sign,
266 + ],
267 + [
268 + stop_sign,
269 + rescale_image_size(stop_sign, 0.25),
270 + cherry_blossom.resize((183, 488)),
271 + cherry_blossom.resize((488, 183))
272 + ],
273 + cherry_blossom,
274 + ])]
275 +
276 + _run_test(
277 + hf_runner,
278 + vllm_runner,
279 + inputs,
280 + model,
281 + dtype=dtype,
282 + max_tokens=max_tokens,
283 + num_logprobs=num_logprobs,
284 + tensor_parallel_size=1,
285 + )
tests/multimodal/test_utils.py
+34 -1
@@ -6,8 +6,10 @@ from typing import Dict, Tuple
6 import numpy as np
7 import pytest
8 from PIL import Image
9 +from transformers import AutoConfig, AutoTokenizer
10
10 -from vllm.multimodal.utils import async_fetch_image, fetch_image
11 +from vllm.multimodal.utils import (async_fetch_image, fetch_image,
12 + repeat_and_pad_placeholder_tokens)
13
14 # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
15 TEST_IMAGE_URLS = [
@@ -80,3 +82,34 @@ async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
82
83 data_image_async = await async_fetch_image(data_url)
84 assert _image_equals(data_image_sync, data_image_async)
85 +
86 +
87 +@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-mistral-7b-hf"])
88 +def test_repeat_and_pad_placeholder_tokens(model):
89 + config = AutoConfig.from_pretrained(model)
90 + image_token_id = config.image_token_index
91 +
92 + tokenizer = AutoTokenizer.from_pretrained(model)
93 +
94 + test_cases = [
95 + ("<image>", 2, "<image><image>", [32000, 32000]),
96 + ("<image><image>", 2, "<image><image><image>", [32000, 32000, 32000]),
97 + ("<image><image>", [3, 2], "<image><image><image><image><image>",
98 + [32000, 32000, 32000, 32000, 32000]),
99 + ("Image:<image>Image:<image>!", [3, 2],
100 + "Image:<image><image><image>Image:<image><image>!",
101 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918]),
102 + ("<image>", [3, 2], "<image><image><image>", [32000, 32000, 32000]),
103 + ]
104 +
105 + for prompt, repeat_count, expected_prompt, expected_token_ids in test_cases:
106 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
107 + tokenizer=tokenizer,
108 + prompt=prompt,
109 + prompt_token_ids=tokenizer.encode(prompt,
110 + add_special_tokens=False),
111 + placeholder_token_id=image_token_id,
112 + repeat_count=repeat_count,
113 + )
114 + assert new_prompt == expected_prompt
115 + assert new_token_ids == expected_token_ids
vllm/model_executor/models/clip.py
+4 -4
@@ -1,7 +1,7 @@
1 -"""Minimal implementation of CLIPVisionModel intended to be only used
1 +"""Minimal implementation of CLIPVisionModel intended to be only used
2 within a vision language model."""
3 from array import array
4 -from typing import Iterable, Optional, Tuple
4 +from typing import Iterable, List, Optional, Tuple, Union
5
6 import torch
7 import torch.nn as nn
@@ -84,7 +84,7 @@ def input_processor_for_clip(
84 llm_inputs: LLMInputs,
85 *,
86 image_token_id: int,
87 - image_feature_size_override: Optional[int] = None,
87 + image_feature_size_override: Optional[Union[int, List[int]]] = None,
88 ):
89 multi_modal_data = llm_inputs.get("multi_modal_data")
90 if multi_modal_data is None or "image" not in multi_modal_data:
@@ -217,7 +217,7 @@ class CLIPEncoderLayer(nn.Module):
217
218 class CLIPEncoder(nn.Module):
219 """
220 - Transformer encoder consisting of `config.num_hidden_layers` self
220 + Transformer encoder consisting of `config.num_hidden_layers` self
221 attention layers. Each layer is a [`CLIPEncoderLayer`].
222
223 Args:
vllm/model_executor/models/llava_next.py
+12 -2
@@ -19,6 +19,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
21 from vllm.sequence import IntermediateTensors, SamplerOutput
22 +from vllm.utils import is_list_of
23
24 from .clip import (CLIPVisionModel, dummy_image_for_clip,
25 dummy_seq_data_for_clip, get_clip_image_feature_size,
@@ -223,6 +224,13 @@ def input_processor_for_llava_next(ctx: InputContext, llm_inputs: LLMInputs):
224 input_height=height,
225 input_width=width,
226 )
227 + elif is_list_of(image_data, Image.Image):
228 + image_feature_size = [
229 + get_llava_next_image_feature_size(hf_config,
230 + input_height=img.height,
231 + input_width=img.width)
232 + for img in image_data
233 + ]
234 elif isinstance(image_data, torch.Tensor):
235 image_feature_size = image_data.shape[0]
236 else:
@@ -425,7 +433,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
433 self.config.image_grid_pinpoints,
434 self.config.vision_config.image_size,
435 )
428 - other_patch_embeds = other_patch_embeds \
436 + num_patches = num_patch_height * num_patch_width
437 +
438 + # Image patches might be padded for batch processing
439 + other_patch_embeds = other_patch_embeds[:num_patches] \
440 .view(num_patch_height, num_patch_width, height, width, -1)
441
442 if "unpad" in strategy:
@@ -496,7 +507,6 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal):
507 self,
508 image_input: LlavaNextImageInputs,
509 ) -> Union[torch.Tensor, List[torch.Tensor]]:
499 -
510 if image_input["type"] == "image_embeds":
511 return [image_input["data"]]
512
vllm/model_executor/models/siglip.py
+2 -2
@@ -3,7 +3,7 @@ within a vision language model."""
3
4 import math
5 from array import array
6 -from typing import Iterable, Optional, Tuple
6 +from typing import Iterable, List, Optional, Tuple, Union
7
8 import torch
9 from PIL import Image
@@ -93,7 +93,7 @@ def input_processor_for_siglip(
93 llm_inputs: LLMInputs,
94 *,
95 image_token_id: int,
96 - image_feature_size_override: Optional[int] = None,
96 + image_feature_size_override: Optional[Union[int, List[int]]] = None,
97 ):
98 multi_modal_data = llm_inputs.get("multi_modal_data")
99 if multi_modal_data is None or "image" not in multi_modal_data:
vllm/multimodal/utils.py
+32 -19
@@ -189,10 +189,13 @@ def repeat_and_pad_placeholder_tokens(
189 prompt_token_ids: List[int],
190 *,
191 placeholder_token_id: int,
192 - repeat_count: int = 1,
192 + repeat_count: Union[int, List[int]],
193 pad_token_left: Optional[int] = None,
194 pad_token_right: Optional[int] = None,
195 ) -> Tuple[Optional[str], List[int]]:
196 + if isinstance(repeat_count, int):
197 + repeat_count = [repeat_count]
198 +
199 if prompt is None:
200 new_prompt = None
201 else:
@@ -201,13 +204,6 @@ def repeat_and_pad_placeholder_tokens(
204 tokenizer.decode(pad_token_left))
205 pad_token_str_right = (None if pad_token_right is None else
206 tokenizer.decode(pad_token_right))
204 - replacement_str = "".join(
205 - repeat_and_pad_token(
206 - placeholder_token_str,
207 - repeat_count=repeat_count,
208 - pad_token_left=pad_token_str_left,
209 - pad_token_right=pad_token_str_right,
210 - ))
207
208 placeholder_token_count = prompt.count(placeholder_token_str)
209 # This is an arbitrary number to distinguish between the two cases
@@ -216,28 +212,45 @@ def repeat_and_pad_placeholder_tokens(
212 "Please follow the prompt format that is "
213 "documented on HuggingFace which does not involve "
214 "repeating %s tokens.", placeholder_token_str)
219 - elif placeholder_token_count > 1:
220 - logger.warning("Multiple multi-modal input is not supported yet, "
221 - "so any extra placeholder tokens will be treated "
222 - "as plain text.")
223 -
224 - # The image tokens are removed to be consistent with HuggingFace
225 - new_prompt = prompt.replace(placeholder_token_str, replacement_str, 1)
215 + if placeholder_token_count < len(repeat_count):
216 + logger.warning(
217 + "The number of multi-modal placeholder tokens in the prompt "
218 + "is less than the number of multi-modal inputs. Extra "
219 + "placeholder tokens will be treated as plain text")
220 + repeat_count = repeat_count[:placeholder_token_count]
221 +
222 + prompt_parts = prompt.split(placeholder_token_str,
223 + maxsplit=len(repeat_count))
224 + new_prompt = ""
225 + for i, repeat_count_item in enumerate(repeat_count):
226 + replacement_str = "".join(
227 + repeat_and_pad_token(
228 + placeholder_token_str,
229 + repeat_count=repeat_count_item,
230 + pad_token_left=pad_token_str_left,
231 + pad_token_right=pad_token_str_right,
232 + ))
233 + # The image tokens are removed to be consistent with HuggingFace
234 + new_prompt += prompt_parts[i] + replacement_str
235 + new_prompt += prompt_parts[-1]
236
237 new_token_ids: List[int] = []
238 + placeholder_token_idx = 0
239 for i, token in enumerate(prompt_token_ids):
240 if token == placeholder_token_id:
241 replacement_ids = repeat_and_pad_token(
242 placeholder_token_id,
232 - repeat_count=repeat_count,
243 + repeat_count=repeat_count[placeholder_token_idx],
244 pad_token_left=pad_token_left,
245 pad_token_right=pad_token_right,
246 )
247 new_token_ids.extend(replacement_ids)
248 + placeholder_token_idx += 1
249
238 - # No need to further scan the list since we only replace once
239 - new_token_ids.extend(prompt_token_ids[i + 1:])
240 - break
250 + # No need to further scan the list since we replaced all tokens
251 + if placeholder_token_idx >= len(repeat_count):
252 + new_token_ids.extend(prompt_token_ids[i + 1:])
253 + break
254 else:
255 new_token_ids.append(token)
256