@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 722d46edb9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Compute Llava Next Max Tokens / Dummy Data From Gridpoints (#9650)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Oct 24, 2024 at 11:42 UTC 722d46edb974315c7d2d8feed75520ea7a30d7fa
2 files changed +93 -14
tests/models/decoder_only/vision_language/test_llava_next.py
+65 -1
@@ -3,12 +3,13 @@ from typing import List, Optional, Tuple, Type, overload
3 import pytest
4 from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6 +from vllm.inputs import InputContext
7 from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
9
10 from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11 _ImageAssets)
11 -from ...utils import check_logprobs_close
12 +from ...utils import build_model_context, check_logprobs_close
13
14 _LIMIT_IMAGE_PER_PROMPT = 4
15
@@ -22,6 +23,19 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
23 models = ["llava-hf/llava-v1.6-mistral-7b-hf"]
24
25
26 +@pytest.fixture()
27 +def get_max_llava_next_image_tokens():
28 + from vllm.model_executor.models.llava_next import (
29 + get_max_llava_next_image_tokens)
30 + return get_max_llava_next_image_tokens
31 +
32 +
33 +@pytest.fixture()
34 +def dummy_data_for_llava_next():
35 + from vllm.model_executor.models.llava_next import dummy_data_for_llava_next
36 + return dummy_data_for_llava_next
37 +
38 +
39 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
40 Optional[SampleLogprobs]],
41 model: str):
@@ -281,3 +295,53 @@ def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
295 num_logprobs=num_logprobs,
296 tensor_parallel_size=1,
297 )
298 +
299 +
300 +@pytest.mark.parametrize("gridpoints,expected_max_tokens", [
301 + ([[336, 336]], 1176),
302 + ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]], 2928),
303 +])
304 +def test_get_max_llava_next_image_tokens(gridpoints, expected_max_tokens,
305 + get_max_llava_next_image_tokens):
306 + ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
307 +
308 + # Update the config image_grid_pinpoints
309 + # and calculate the resulting max tokens
310 + ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
311 +
312 + actual_max_tokens = get_max_llava_next_image_tokens(
313 + InputContext(ctx.model_config))
314 +
315 + assert expected_max_tokens == actual_max_tokens
316 +
317 +
318 +@pytest.mark.parametrize(
319 + "gridpoints,expected_size",
320 + [
321 + # One point; it has to be the largest
322 + ([[336, 336]], (336, 336)),
323 + # Default for most llava next models; the 2x2 tile is the largest
324 + ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]],
325 + (672, 672)),
326 + # If two rectangular gridpoints are the same, the more vertical
327 + # one has the higher feature count due to newline features
328 + ([[336, 672], [672, 336]], (672, 336))
329 + ])
330 +def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
331 + gridpoints, expected_size):
332 + ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
333 +
334 + # Update the config image_grid_pinpoints
335 + ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
336 + seq_len = 5000 # bigger than the max feature size for any image
337 +
338 + seq_data, mm_data = dummy_data_for_llava_next(
339 + ctx,
340 + seq_len=seq_len,
341 + mm_counts={"image": 1},
342 + )
343 +
344 + # The dummy data dims should match the gridpoint with the biggest feat size
345 + assert mm_data["image"].height == expected_size[0]
346 + assert mm_data["image"].width == expected_size[1]
347 + assert len(seq_data.get_token_ids()) >= seq_len
vllm/model_executor/models/llava_next.py
+28 -13
@@ -33,9 +33,6 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
33 from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
34 init_vllm_registered_model)
35
36 -# Result in the max possible feature size (2x2 grid of 336x336px tiles)
37 -MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
38 -
36
37 class LlavaNextImagePixelInputs(TypedDict):
38 type: Literal["pixel_values"]
@@ -149,11 +146,28 @@ def get_llava_next_image_feature_size(
146
147
148 def get_max_llava_next_image_tokens(ctx: InputContext):
152 - return get_llava_next_image_feature_size(
153 - ctx.get_hf_config(LlavaNextConfig),
154 - input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
155 - input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
156 - )
149 + """Compute the max feature size for all possible image grid pinpoints."""
150 + return _get_pinpoint_with_largest_features(ctx)[0]
151 +
152 +
153 +def _get_pinpoint_with_largest_features(
154 + ctx: InputContext) -> Tuple[int, Tuple[int, int]]:
155 + """Get the grid pinpoint with the largest features & its feature size."""
156 + hf_config = ctx.get_hf_config(LlavaNextConfig)
157 + largest_feature_size = 0
158 + largest_feature_pinpoint = None
159 + for (height, width) in hf_config.image_grid_pinpoints:
160 + feat_size = get_llava_next_image_feature_size(
161 + hf_config,
162 + input_height=height,
163 + input_width=width,
164 + )
165 + if feat_size > largest_feature_size:
166 + largest_feature_size = feat_size
167 + largest_feature_pinpoint = (height, width)
168 + if not largest_feature_size or largest_feature_pinpoint is None:
169 + raise ValueError("Cannot have a largest feature size of 0!")
170 + return largest_feature_size, largest_feature_pinpoint
171
172
173 def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
@@ -162,7 +176,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
176 vision_config = hf_config.vision_config
177 num_images = mm_counts["image"]
178
165 - image_feature_size = get_max_llava_next_image_tokens(ctx)
179 + image_feature_size, pinpoint = _get_pinpoint_with_largest_features(ctx)
180 + max_feat_height, max_feat_width = pinpoint
181
182 if isinstance(vision_config, CLIPVisionConfig):
183 seq_data = dummy_seq_data_for_clip(
@@ -176,8 +191,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
191 mm_data = dummy_image_for_clip(
192 vision_config,
193 num_images,
179 - image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
180 - image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
194 + image_width_override=max_feat_width,
195 + image_height_override=max_feat_height,
196 )
197
198 return seq_data, mm_data
@@ -193,8 +208,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
208 mm_data = dummy_image_for_siglip(
209 vision_config,
210 num_images,
196 - image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
197 - image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
211 + image_width_override=max_feat_width,
212 + image_height_override=max_feat_height,
213 )
214
215 return seq_data, mm_data