@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 874f7c292a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix max image feature size for Llava-one-vision (#12104)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 16, 2025 at 06:54 UTC 874f7c292a4f4f5dbb89b12426187e5a70f006d6
3 files changed +129 -2
tests/models/multimodal/processing/test_llava_next.py
+61
@@ -13,6 +13,67 @@ from vllm.multimodal.utils import cached_get_tokenizer
13 from ...utils import build_model_context
14
15
16 +def _validate_image_max_tokens_one(
17 + processor: BaseMultiModalProcessor,
18 + max_tokens: int,
19 + failed_size_excs: list[tuple[ImageSize, Exception]],
20 + image_size: ImageSize,
21 +) -> None:
22 + info = processor.info
23 + feature_size = info.get_num_image_tokens(image_width=image_size.width,
24 + image_height=image_size.height)
25 +
26 + try:
27 + assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
28 + except Exception as exc:
29 + failed_size_excs.append((image_size, exc))
30 +
31 +
32 +@pytest.mark.skip("This test takes around 5 minutes to run. "
33 + "Comment this out to run it manually.")
34 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
35 +def test_processor_max_tokens(model_id):
36 + ctx = build_model_context(
37 + model_name=model_id,
38 + tokenizer_name=model_id,
39 + mm_processor_kwargs=None,
40 + limit_mm_per_prompt={"image": 1},
41 + )
42 + processor = MULTIMODAL_REGISTRY.create_processor(
43 + ctx.model_config,
44 + tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
45 + )
46 + info = processor.info
47 +
48 + seen_aspect_ratios = set[float]()
49 + image_sizes = list[ImageSize]()
50 +
51 + # The aspect ratio of the grid layout is between 1 and 2
52 + # NOTE: Assumes that feature size calculation is the same if we
53 + # swap the width and height of the image
54 + for w, h in itertools.product(range(32, 4096), repeat=2):
55 + aspect_ratio = w / h
56 + if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
57 + image_sizes.append(ImageSize(w, h))
58 + seen_aspect_ratios.add(aspect_ratio)
59 +
60 + failed_size_excs = list[tuple[ImageSize, Exception]]()
61 +
62 + validate_one = partial(
63 + _validate_image_max_tokens_one,
64 + processor,
65 + info.get_max_image_tokens(), # type: ignore
66 + failed_size_excs,
67 + )
68 + pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
69 +
70 + if failed_size_excs:
71 + msg = "Found failing image sizes:" \
72 + + "\n========\n".join(f"[{size}]\n{exc}"
73 + for size, exc in failed_size_excs)
74 + raise AssertionError(msg)
75 +
76 +
77 def _validate_image_prompt_replacements_one(
78 processor: BaseMultiModalProcessor,
79 num_imgs: int,
tests/models/multimodal/processing/test_llava_onevision.py
+62
@@ -13,6 +13,68 @@ from vllm.multimodal.utils import cached_get_tokenizer
13 from ...utils import build_model_context
14
15
16 +def _validate_image_max_tokens_one(
17 + processor: BaseMultiModalProcessor,
18 + max_tokens: int,
19 + failed_size_excs: list[tuple[ImageSize, Exception]],
20 + image_size: ImageSize,
21 +) -> None:
22 + info = processor.info
23 + feature_size = info.get_num_image_tokens(image_width=image_size.width,
24 + image_height=image_size.height)
25 +
26 + try:
27 + assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
28 + except Exception as exc:
29 + failed_size_excs.append((image_size, exc))
30 +
31 +
32 +@pytest.mark.skip("This test takes around 5 minutes to run. "
33 + "Comment this out to run it manually.")
34 +@pytest.mark.parametrize("model_id",
35 + ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
36 +def test_processor_max_tokens(model_id):
37 + ctx = build_model_context(
38 + model_name=model_id,
39 + tokenizer_name=model_id,
40 + mm_processor_kwargs=None,
41 + limit_mm_per_prompt={"image": 1},
42 + )
43 + processor = MULTIMODAL_REGISTRY.create_processor(
44 + ctx.model_config,
45 + tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
46 + )
47 + info = processor.info
48 +
49 + seen_aspect_ratios = set[float]()
50 + image_sizes = list[ImageSize]()
51 +
52 + # The aspect ratio of the grid layout is between 1 and 6
53 + # NOTE: Assumes that feature size calculation is the same if we
54 + # swap the width and height of the image
55 + for w, h in itertools.product(range(32, 4096), repeat=2):
56 + aspect_ratio = w / h
57 + if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
58 + image_sizes.append(ImageSize(w, h))
59 + seen_aspect_ratios.add(aspect_ratio)
60 +
61 + failed_size_excs = list[tuple[ImageSize, Exception]]()
62 +
63 + validate_one = partial(
64 + _validate_image_max_tokens_one,
65 + processor,
66 + info.get_max_image_tokens(), # type: ignore
67 + failed_size_excs,
68 + )
69 + pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
70 +
71 + if failed_size_excs:
72 + msg = "Found failing image sizes:" \
73 + + "\n========\n".join(f"[{size}]\n{exc}"
74 + for size, exc in failed_size_excs)
75 + raise AssertionError(msg)
76 +
77 +
78 def _validate_image_prompt_replacements_one(
79 processor: BaseMultiModalProcessor,
80 num_imgs: int,
vllm/model_executor/models/llava_onevision.py
+6 -2
@@ -19,8 +19,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
21 NestedTensors)
22 -from vllm.multimodal.parse import (MultiModalDataItems, VideoEmbeddingItems,
23 - VideoProcessorItems)
22 +from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
23 + VideoEmbeddingItems, VideoProcessorItems)
24 from vllm.multimodal.processing import PromptReplacement
25 from vllm.multimodal.profiling import ProcessorInputs
26 from vllm.sequence import IntermediateTensors
@@ -145,6 +145,10 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
145
146 return (unpadded_features, newline_features)
147
148 + def get_image_size_with_most_features(self) -> ImageSize:
149 + # NOTE: This hardcoded value is found via processor tests
150 + return ImageSize(width=1153, height=944)
151 +
152 def _get_num_frame_tokens(
153 self,
154 *,