@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f37be38eb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Comprehensively test and fix LLaVA-NeXT feature size calculation (#11800)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 7, 2025 at 18:25 UTC 8f37be38ebfe0295a4925837c501c87149997a4d
6 files changed +253 -89
requirements-test.in
+1
@@ -13,6 +13,7 @@ einops # required for MPT, qwen-vl and Mamba
13 httpx
14 librosa # required for audio tests
15 peft
16 +pqdm
17 ray[adag]==2.40.0
18 sentence-transformers # required for embedding tests
19 soundfile # required for audio tests
requirements-test.txt
+4
@@ -48,6 +48,8 @@ botocore==1.35.57
48 # awscli
49 # boto3
50 # s3transfer
51 +bounded-pool-executor==0.0.3
52 + # via pqdm
53 buildkite-test-collector==0.1.9
54 # via -r requirements-test.in
55 certifi==2024.8.30
@@ -342,6 +344,8 @@ pooch==1.8.2
344 # via librosa
345 portalocker==2.10.1
346 # via sacrebleu
347 +pqdm==0.2.0
348 + # via -r requirements-test.in
349 propcache==0.2.0
350 # via yarl
351 protobuf==5.28.3
tests/models/decoder_only/vision_language/processing/test_llava_next.py
+108 -21
@@ -1,8 +1,13 @@
1 +import itertools
2 +from functools import partial
3 +
4 import pytest
5 from PIL import Image
6 +from pqdm.threads import pqdm
7 from transformers import AutoTokenizer
8
9 from vllm.inputs import InputProcessingContext
10 +from vllm.multimodal.parse import ImageSize
11
12 from ....utils import build_model_context
13
@@ -15,20 +20,69 @@ def processor_for_llava_next():
20 return LlavaNextMultiModalProcessor
21
22
23 +def _validate_image_prompt_replacements_one(
24 + processor,
25 + num_imgs: int,
26 + failed_size_excs: list[tuple[ImageSize, Exception]],
27 + image_size: ImageSize,
28 +) -> None:
29 + prompt = "<image>" * num_imgs
30 + image = Image.new("RGB", size=image_size)
31 + mm_data = {"image": [image] * num_imgs}
32 +
33 + try:
34 + # The processor will throw an error if there is a mismatch
35 + # in the prompt replacements
36 + processed_inputs = processor.apply(prompt, mm_data, {})
37 +
38 + image_placeholders = processed_inputs["mm_placeholders"]["image"]
39 + assert len(image_placeholders) == num_imgs
40 +
41 + first_placeholder = image_placeholders[0]
42 +
43 + # NOTE: There is a BOS token
44 + assert first_placeholder["offset"] == 1
45 + assert first_placeholder["length"] == (
46 + len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
47 +
48 + except Exception as exc:
49 + failed_size_excs.append((image_size, exc))
50 +
51 +
52 +def _test_image_prompt_replacements(
53 + processor,
54 + *,
55 + num_imgs: int,
56 + image_sizes: list[ImageSize],
57 +) -> None:
58 + """
59 + Ensure LlavaNextMultiModalProcessor
60 + handles prompt replacement properly for input images.
61 + """
62 + failed_size_excs = list[tuple[ImageSize, Exception]]()
63 +
64 + validate_one = partial(
65 + _validate_image_prompt_replacements_one,
66 + processor,
67 + num_imgs,
68 + failed_size_excs,
69 + )
70 + pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
71 +
72 + if failed_size_excs:
73 + msg = "Found failing image sizes:" \
74 + + "\n========\n".join(f"[{size}]\n{exc}"
75 + for size, exc in failed_size_excs)
76 + raise AssertionError(msg)
77 +
78 +
79 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
19 -@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
20 - (488, 183), (198, 176), (176, 198),
21 - (161, 184), (184, 161)])
80 @pytest.mark.parametrize("num_imgs", [1, 2])
23 -def test_processor_prompt_replacements(
81 +def test_processor_prompt_replacements_regression(
82 processor_for_llava_next,
83 model_id: str,
26 - image_size: tuple[int, int],
84 num_imgs: int,
85 ):
29 - """
30 - Ensure LlavaNextMultiModalProcessor handles prompt replacement properly.
31 - """
86 ctx = build_model_context(
87 model_name=model_id,
88 tokenizer_name=model_id,
@@ -37,22 +91,55 @@ def test_processor_prompt_replacements(
91 )
92 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
93 ctx = InputProcessingContext(ctx.model_config, tokenizer)
94 + processor = processor_for_llava_next(ctx)
95 +
96 + image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
97 + (488, 183), (2560, 1669)]
98 + image_sizes = [
99 + size for w, h in image_ratios
100 + for size in [ImageSize(w, h), ImageSize(h, w)]
101 + ]
102 +
103 + _test_image_prompt_replacements(
104 + processor,
105 + num_imgs=num_imgs,
106 + image_sizes=image_sizes,
107 + )
108
41 - # Build the image str / prompt based on the number of images we pass
42 - prompt = "<image>" * num_imgs
43 - mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
109
45 - # The processor will throw an error if there is a mismatch
46 - # in the prompt replacements
110 +@pytest.mark.skip("This test takes around 2 hours to run. "
111 + "Comment this out to run it manually.")
112 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
113 +@pytest.mark.parametrize("num_imgs", [1])
114 +def test_processor_prompt_replacements_all(
115 + processor_for_llava_next,
116 + model_id: str,
117 + num_imgs: int,
118 +):
119 + ctx = build_model_context(
120 + model_name=model_id,
121 + tokenizer_name=model_id,
122 + mm_processor_kwargs=None,
123 + limit_mm_per_prompt={"image": num_imgs},
124 + )
125 + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126 + ctx = InputProcessingContext(ctx.model_config, tokenizer)
127 processor = processor_for_llava_next(ctx)
48 - processed_inputs = processor.apply(prompt, mm_data, {})
128
50 - image_placeholders = processed_inputs["mm_placeholders"]["image"]
51 - assert len(image_placeholders) == num_imgs
129 + seen_aspect_ratios = set[float]()
130 + image_sizes = list[ImageSize]()
131
53 - first_placeholder = image_placeholders[0]
132 + # The aspect ratio of the grid layout is between 1 and 2
133 + # NOTE: Assumes that feature size calculation is the same if we
134 + # swap the width and height of the image
135 + for w, h in itertools.product(range(64, 1024), repeat=2):
136 + aspect_ratio = w / h
137 + if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
138 + image_sizes.append(ImageSize(w, h))
139 + seen_aspect_ratios.add(aspect_ratio)
140
55 - # NOTE: There is a BOS token
56 - assert first_placeholder["offset"] == 1
57 - assert first_placeholder["length"] == (
58 - len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
141 + _test_image_prompt_replacements(
142 + processor,
143 + num_imgs=num_imgs,
144 + image_sizes=image_sizes,
145 + )
tests/models/decoder_only/vision_language/processing/test_llava_onevision.py
+107 -22
@@ -1,8 +1,13 @@
1 +import itertools
2 +from functools import partial
3 +
4 import pytest
5 from PIL import Image
6 +from pqdm.threads import pqdm
7 from transformers import AutoTokenizer
8
9 from vllm.inputs import InputProcessingContext
10 +from vllm.multimodal.parse import ImageSize
11
12 from ....utils import build_model_context
13
@@ -15,22 +20,68 @@ def processor_for_llava_onevision():
20 return LlavaOnevisionMultiModalProcessor
21
22
23 +def _validate_image_prompt_replacements_one(
24 + processor,
25 + num_imgs: int,
26 + failed_size_excs: list[tuple[ImageSize, Exception]],
27 + image_size: ImageSize,
28 +) -> None:
29 + prompt = "<image>" * num_imgs
30 + image = Image.new("RGB", size=image_size)
31 + mm_data = {"image": [image] * num_imgs}
32 +
33 + try:
34 + # The processor will throw an error if there is a mismatch
35 + # in the prompt replacements
36 + processed_inputs = processor.apply(prompt, mm_data, {})
37 +
38 + image_placeholders = processed_inputs["mm_placeholders"]["image"]
39 + assert len(image_placeholders) == num_imgs
40 +
41 + first_placeholder = image_placeholders[0]
42 +
43 + assert first_placeholder["offset"] == 0
44 + assert first_placeholder["length"] == len(
45 + processed_inputs["prompt_token_ids"]) // num_imgs
46 + except Exception as exc:
47 + failed_size_excs.append((image_size, exc))
48 +
49 +
50 +def _test_image_prompt_replacements(
51 + processor,
52 + *,
53 + num_imgs: int,
54 + image_sizes: list[ImageSize],
55 +) -> None:
56 + """
57 + Ensure LlavaOnevisionMultiModalProcessor
58 + handles prompt replacement properly for input images.
59 + """
60 + failed_size_excs = list[tuple[ImageSize, Exception]]()
61 +
62 + validate_one = partial(
63 + _validate_image_prompt_replacements_one,
64 + processor,
65 + num_imgs,
66 + failed_size_excs,
67 + )
68 + pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
69 +
70 + if failed_size_excs:
71 + msg = "Found failing image sizes:" \
72 + + "\n========\n".join(f"[{size}]\n{exc}"
73 + for size, exc in failed_size_excs)
74 + raise AssertionError(msg)
75 +
76 +
77 @pytest.mark.parametrize("model_id",
78 ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
20 -@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21 - (488, 183), (198, 176), (176, 198),
22 - (161, 184), (184, 161)])
79 @pytest.mark.parametrize("num_imgs", [1, 2])
24 -def test_processor_prompt_replacements(
80 +def test_processor_prompt_replacements_regression(
81 processor_for_llava_onevision,
82 model_id: str,
27 - image_size: tuple[int, int],
83 num_imgs: int,
84 ):
30 - """
31 - Ensure LlavaOnevisionMultiModalProcessor handles prompt replacement
32 - properly.
33 - """
85 ctx = build_model_context(
86 model_name=model_id,
87 tokenizer_name=model_id,
@@ -39,22 +90,56 @@ def test_processor_prompt_replacements(
90 )
91 tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
92 ctx = InputProcessingContext(ctx.model_config, tokenizer)
93 + processor = processor_for_llava_onevision(ctx)
94
43 - # Build the image str / prompt based on the number of images we pass
44 - prompt = "<image>" * num_imgs
45 - mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
95 + image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
96 + (488, 183), (2560, 1669)]
97 + image_sizes = [
98 + size for w, h in image_ratios
99 + for size in [ImageSize(w, h), ImageSize(h, w)]
100 + ]
101 +
102 + _test_image_prompt_replacements(
103 + processor,
104 + num_imgs=num_imgs,
105 + image_sizes=image_sizes,
106 + )
107
47 - # The processor will throw an error if there is a mismatch
48 - # in the prompt replacements
108 +
109 +@pytest.mark.skip("This test takes around 2 hours to run. "
110 + "Comment this out to run it manually.")
111 +@pytest.mark.parametrize("model_id",
112 + ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
113 +@pytest.mark.parametrize("num_imgs", [1])
114 +def test_processor_prompt_replacements_all(
115 + processor_for_llava_onevision,
116 + model_id: str,
117 + num_imgs: int,
118 +):
119 + ctx = build_model_context(
120 + model_name=model_id,
121 + tokenizer_name=model_id,
122 + mm_processor_kwargs=None,
123 + limit_mm_per_prompt={"image": num_imgs},
124 + )
125 + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126 + ctx = InputProcessingContext(ctx.model_config, tokenizer)
127 processor = processor_for_llava_onevision(ctx)
50 - processed_inputs = processor.apply(prompt, mm_data, {})
128
52 - image_placeholders = processed_inputs["mm_placeholders"]["image"]
53 - assert len(image_placeholders) == num_imgs
129 + seen_aspect_ratios = set[float]()
130 + image_sizes = list[ImageSize]()
131
55 - first_placeholder = image_placeholders[0]
132 + # The aspect ratio of the grid layout is between 1 and 6
133 + # NOTE: Assumes that feature size calculation is the same if we
134 + # swap the width and height of the image
135 + for w, h in itertools.product(range(64, 1024), repeat=2):
136 + aspect_ratio = w / h
137 + if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
138 + image_sizes.append(ImageSize(w, h))
139 + seen_aspect_ratios.add(aspect_ratio)
140
57 - # NOTE: There is a BOS token
58 - assert first_placeholder["offset"] == 0
59 - assert first_placeholder["length"] == len(
60 - processed_inputs["prompt_token_ids"]) // num_imgs
141 + _test_image_prompt_replacements(
142 + processor,
143 + num_imgs=num_imgs,
144 + image_sizes=image_sizes,
145 + )
vllm/model_executor/models/llava_next.py
+15 -22
@@ -2,7 +2,6 @@ from functools import cached_property
2 from typing import (Final, Iterable, List, Literal, Mapping, Optional,
3 Protocol, Set, Tuple, TypedDict, Union)
4
5 -import numpy as np
5 import torch
6 import torch.nn as nn
7 from transformers import BatchFeature, LlavaNextConfig, LlavaNextProcessor
@@ -74,7 +73,7 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
73 def _get_hf_processor(self):
74 return self.ctx.get_hf_processor(LlavaNextProcessor)
75
77 - # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
76 + # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
77 def _get_num_image_tokens(
78 self,
79 *,
@@ -111,7 +110,7 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
110
111 return unpadded_feature_size + newline_feature_size + base_feature_size
112
114 - # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
113 + # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
114 def _get_num_unpadded_features(
115 self,
116 *,
@@ -121,29 +120,23 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
120 num_patch_height: int,
121 num_patch_width: int,
122 ) -> tuple[int, int]:
124 - # NOTE: Use float32 to remain consistent with HF output
125 - current_height_f = np.float32(npatches * num_patch_height)
126 - current_width_f = np.float32(npatches * num_patch_width)
123 + current_height = npatches * num_patch_height
124 + current_width = npatches * num_patch_width
125
128 - original_width_f = np.float32(original_width)
129 - original_height_f = np.float32(original_height)
126 + aspect_ratio = original_width / original_height
127 + current_aspect_ratio = current_width / current_height
128
131 - original_aspect_ratio = original_width_f / original_height_f
132 - current_aspect_ratio = current_width_f / current_height_f
133 -
134 - if original_aspect_ratio > current_aspect_ratio:
135 - scale_factor = current_width_f / original_width_f
136 - new_height = int(original_height_f * scale_factor)
137 - padding = (current_height_f - new_height) // 2
138 - current_height_f -= 2 * padding
129 + if aspect_ratio > current_aspect_ratio:
130 + new_height = (original_height * current_width) // original_width
131 + padding = (current_height - new_height) // 2
132 + current_height = current_height - (2 * padding)
133 else:
140 - scale_factor = current_height_f / original_height_f
141 - new_width = int(original_width_f * scale_factor)
142 - padding = (current_width_f - new_width) // 2
143 - current_width_f -= 2 * padding
134 + new_width = (original_width * current_height) // original_height
135 + padding = (current_width - new_width) // 2
136 + current_width = current_width - (2 * padding)
137
145 - unpadded_features = int(current_height_f * current_width_f)
146 - newline_features = int(current_height_f)
138 + unpadded_features = current_height * current_width
139 + newline_features = current_height
140
141 return (unpadded_features, newline_features)
142
vllm/model_executor/models/llava_onevision.py
+18 -24
@@ -3,7 +3,6 @@ from functools import cached_property
3 from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4 Protocol, Set, Tuple, TypedDict, Union)
5
6 -import numpy as np
6 import torch
7 import torch.nn as nn
8 from transformers import (BatchFeature, LlavaOnevisionConfig,
@@ -98,6 +97,8 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
97 def _get_hf_processor(self):
98 return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
99
100 + # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
101 + # with additional logic afterwards taken from LlavaOnevisionProcessor
102 def _get_num_unpadded_features(
103 self,
104 *,
@@ -107,35 +108,28 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
108 num_patch_height: int,
109 num_patch_width: int,
110 ) -> tuple[int, int]:
110 - # NOTE: Use float32 to remain consistent with HF output
111 - current_height_f = np.float32(npatches * num_patch_height)
112 - current_width_f = np.float32(npatches * num_patch_width)
111 + current_height = npatches * num_patch_height
112 + current_width = npatches * num_patch_width
113
114 - original_width_f = np.float32(original_width)
115 - original_height_f = np.float32(original_height)
114 + aspect_ratio = original_width / original_height
115 + current_aspect_ratio = current_width / current_height
116
117 - original_aspect_ratio = original_width_f / original_height_f
118 - current_aspect_ratio = current_width_f / current_height_f
119 -
120 - if original_aspect_ratio > current_aspect_ratio:
121 - scale_factor = current_width_f / original_width_f
122 - new_height = int(original_height_f * scale_factor)
123 - padding = (current_height_f - new_height) // 2
124 - current_height_f -= 2 * padding
117 + if aspect_ratio > current_aspect_ratio:
118 + new_height = (original_height * current_width) // original_width
119 + padding = (current_height - new_height) // 2
120 + current_height = current_height - (2 * padding)
121 else:
126 - scale_factor = current_height_f / original_height_f
127 - new_width = int(original_width_f * scale_factor)
128 - padding = (current_width_f - new_width) // 2
129 - current_width_f -= 2 * padding
122 + new_width = (original_width * current_height) // original_height
123 + padding = (current_width - new_width) // 2
124 + current_width = current_width - (2 * padding)
125
131 - unpadded_features = int(current_height_f * current_width_f)
132 - newline_features = int(current_height_f)
126 + unpadded_features = current_height * current_width
127 + newline_features = current_height
128
134 - ratio = math.sqrt(current_height_f * current_width_f /
135 - (9 * npatches**2))
129 + ratio = math.sqrt(current_height * current_width / (9 * npatches**2))
130 if ratio > 1.1:
137 - height_factor = int(current_height_f // ratio)
138 - width_factor = int(current_width_f // ratio)
131 + height_factor = int(current_height // ratio)
132 + width_factor = int(current_width // ratio)
133 unpadded_features = height_factor * width_factor
134 newline_features = height_factor
135