A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Compute Llava Next Max Tokens / Dummy Data From Gridpoints (#9650)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
Alex Brooks committed
Oct 24, 2024 at 11:42 UTC
722d46edb974315c7d2d8feed75520ea7a30d7fa
2 files changed
+93
-14
tests/models/decoder_only/vision_language/test_llava_next.py
+65
-1
@@ -3,12 +3,13 @@ from typing import List, Optional, Tuple, Type, overload
3
import pytest
4
from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5
6
+from vllm.inputs import InputContext
7
from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
9
10
from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11
_ImageAssets)
11
-from ...utils import check_logprobs_close
12
+from ...utils import build_model_context, check_logprobs_close
13
14
_LIMIT_IMAGE_PER_PROMPT = 4
15
@@ -22,6 +23,19 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
23
models = ["llava-hf/llava-v1.6-mistral-7b-hf"]
24
25
26
+@pytest.fixture()
27
+def get_max_llava_next_image_tokens():
28
+ from vllm.model_executor.models.llava_next import (
29
+ get_max_llava_next_image_tokens)
30
+ return get_max_llava_next_image_tokens
31
+
32
+
33
+@pytest.fixture()
34
+def dummy_data_for_llava_next():
35
+ from vllm.model_executor.models.llava_next import dummy_data_for_llava_next
36
+ return dummy_data_for_llava_next
37
+
38
+
39
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
40
Optional[SampleLogprobs]],
41
model: str):
@@ -281,3 +295,53 @@ def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
295
num_logprobs=num_logprobs,
296
tensor_parallel_size=1,
297
)
298
+
299
+
300
+@pytest.mark.parametrize("gridpoints,expected_max_tokens", [
301
+ ([[336, 336]], 1176),
302
+ ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]], 2928),
303
+])
304
+def test_get_max_llava_next_image_tokens(gridpoints, expected_max_tokens,
305
+ get_max_llava_next_image_tokens):
306
+ ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
307
+
308
+ # Update the config image_grid_pinpoints
309
+ # and calculate the resulting max tokens
310
+ ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
311
+
312
+ actual_max_tokens = get_max_llava_next_image_tokens(
313
+ InputContext(ctx.model_config))
314
+
315
+ assert expected_max_tokens == actual_max_tokens
316
+
317
+
318
+@pytest.mark.parametrize(
319
+ "gridpoints,expected_size",
320
+ [
321
+ # One point; it has to be the largest
322
+ ([[336, 336]], (336, 336)),
323
+ # Default for most llava next models; the 2x2 tile is the largest
324
+ ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]],
325
+ (672, 672)),
326
+ # If two rectangular gridpoints are the same, the more vertical
327
+ # one has the higher feature count due to newline features
328
+ ([[336, 672], [672, 336]], (672, 336))
329
+ ])
330
+def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
331
+ gridpoints, expected_size):
332
+ ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
333
+
334
+ # Update the config image_grid_pinpoints
335
+ ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
336
+ seq_len = 5000 # bigger than the max feature size for any image
337
+
338
+ seq_data, mm_data = dummy_data_for_llava_next(
339
+ ctx,
340
+ seq_len=seq_len,
341
+ mm_counts={"image": 1},
342
+ )
343
+
344
+ # The dummy data dims should match the gridpoint with the biggest feat size
345
+ assert mm_data["image"].height == expected_size[0]
346
+ assert mm_data["image"].width == expected_size[1]
347
+ assert len(seq_data.get_token_ids()) >= seq_len
vllm/model_executor/models/llava_next.py
+28
-13
@@ -33,9 +33,6 @@ from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
33
from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
34
init_vllm_registered_model)
35
36
-# Result in the max possible feature size (2x2 grid of 336x336px tiles)
37
-MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
38
-
36
37
class LlavaNextImagePixelInputs(TypedDict):
38
type: Literal["pixel_values"]
@@ -149,11 +146,28 @@ def get_llava_next_image_feature_size(
146
147
148
def get_max_llava_next_image_tokens(ctx: InputContext):
152
- return get_llava_next_image_feature_size(
153
- ctx.get_hf_config(LlavaNextConfig),
154
- input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
155
- input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
156
- )
149
+ """Compute the max feature size for all possible image grid pinpoints."""
150
+ return _get_pinpoint_with_largest_features(ctx)[0]
151
+
152
+
153
+def _get_pinpoint_with_largest_features(
154
+ ctx: InputContext) -> Tuple[int, Tuple[int, int]]:
155
+ """Get the grid pinpoint with the largest features & its feature size."""
156
+ hf_config = ctx.get_hf_config(LlavaNextConfig)
157
+ largest_feature_size = 0
158
+ largest_feature_pinpoint = None
159
+ for (height, width) in hf_config.image_grid_pinpoints:
160
+ feat_size = get_llava_next_image_feature_size(
161
+ hf_config,
162
+ input_height=height,
163
+ input_width=width,
164
+ )
165
+ if feat_size > largest_feature_size:
166
+ largest_feature_size = feat_size
167
+ largest_feature_pinpoint = (height, width)
168
+ if not largest_feature_size or largest_feature_pinpoint is None:
169
+ raise ValueError("Cannot have a largest feature size of 0!")
170
+ return largest_feature_size, largest_feature_pinpoint
171
172
173
def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
@@ -162,7 +176,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
176
vision_config = hf_config.vision_config
177
num_images = mm_counts["image"]
178
165
- image_feature_size = get_max_llava_next_image_tokens(ctx)
179
+ image_feature_size, pinpoint = _get_pinpoint_with_largest_features(ctx)
180
+ max_feat_height, max_feat_width = pinpoint
181
182
if isinstance(vision_config, CLIPVisionConfig):
183
seq_data = dummy_seq_data_for_clip(
@@ -176,8 +191,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
191
mm_data = dummy_image_for_clip(
192
vision_config,
193
num_images,
179
- image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
180
- image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
194
+ image_width_override=max_feat_width,
195
+ image_height_override=max_feat_height,
196
)
197
198
return seq_data, mm_data
@@ -193,8 +208,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
208
mm_data = dummy_image_for_siglip(
209
vision_config,
210
num_images,
196
- image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
197
- image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
211
+ image_width_override=max_feat_width,
212
+ image_height_override=max_feat_height,
213
)
214
215
return seq_data, mm_data