A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Comprehensively test and fix LLaVA-NeXT feature size calculation (#11800)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 7, 2025 at 18:25 UTC
8f37be38ebfe0295a4925837c501c87149997a4d
6 files changed
+253
-89
requirements-test.in
+1
@@ -13,6 +13,7 @@ einops # required for MPT, qwen-vl and Mamba
13
httpx
14
librosa # required for audio tests
15
peft
16
+pqdm
17
ray[adag]==2.40.0
18
sentence-transformers # required for embedding tests
19
soundfile # required for audio tests
requirements-test.txt
+4
@@ -48,6 +48,8 @@ botocore==1.35.57
48
# awscli
49
# boto3
50
# s3transfer
51
+bounded-pool-executor==0.0.3
52
+ # via pqdm
53
buildkite-test-collector==0.1.9
54
# via -r requirements-test.in
55
certifi==2024.8.30
@@ -342,6 +344,8 @@ pooch==1.8.2
344
# via librosa
345
portalocker==2.10.1
346
# via sacrebleu
347
+pqdm==0.2.0
348
+ # via -r requirements-test.in
349
propcache==0.2.0
350
# via yarl
351
protobuf==5.28.3
tests/models/decoder_only/vision_language/processing/test_llava_next.py
+108
-21
@@ -1,8 +1,13 @@
1
+import itertools
2
+from functools import partial
3
+
4
import pytest
5
from PIL import Image
6
+from pqdm.threads import pqdm
7
from transformers import AutoTokenizer
8
9
from vllm.inputs import InputProcessingContext
10
+from vllm.multimodal.parse import ImageSize
11
12
from ....utils import build_model_context
13
@@ -15,20 +20,69 @@ def processor_for_llava_next():
20
return LlavaNextMultiModalProcessor
21
22
23
+def _validate_image_prompt_replacements_one(
24
+ processor,
25
+ num_imgs: int,
26
+ failed_size_excs: list[tuple[ImageSize, Exception]],
27
+ image_size: ImageSize,
28
+) -> None:
29
+ prompt = "<image>" * num_imgs
30
+ image = Image.new("RGB", size=image_size)
31
+ mm_data = {"image": [image] * num_imgs}
32
+
33
+ try:
34
+ # The processor will throw an error if there is a mismatch
35
+ # in the prompt replacements
36
+ processed_inputs = processor.apply(prompt, mm_data, {})
37
+
38
+ image_placeholders = processed_inputs["mm_placeholders"]["image"]
39
+ assert len(image_placeholders) == num_imgs
40
+
41
+ first_placeholder = image_placeholders[0]
42
+
43
+ # NOTE: There is a BOS token
44
+ assert first_placeholder["offset"] == 1
45
+ assert first_placeholder["length"] == (
46
+ len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
47
+
48
+ except Exception as exc:
49
+ failed_size_excs.append((image_size, exc))
50
+
51
+
52
+def _test_image_prompt_replacements(
53
+ processor,
54
+ *,
55
+ num_imgs: int,
56
+ image_sizes: list[ImageSize],
57
+) -> None:
58
+ """
59
+ Ensure LlavaNextMultiModalProcessor
60
+ handles prompt replacement properly for input images.
61
+ """
62
+ failed_size_excs = list[tuple[ImageSize, Exception]]()
63
+
64
+ validate_one = partial(
65
+ _validate_image_prompt_replacements_one,
66
+ processor,
67
+ num_imgs,
68
+ failed_size_excs,
69
+ )
70
+ pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
71
+
72
+ if failed_size_excs:
73
+ msg = "Found failing image sizes:" \
74
+ + "\n========\n".join(f"[{size}]\n{exc}"
75
+ for size, exc in failed_size_excs)
76
+ raise AssertionError(msg)
77
+
78
+
79
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
19
-@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
20
- (488, 183), (198, 176), (176, 198),
21
- (161, 184), (184, 161)])
80
@pytest.mark.parametrize("num_imgs", [1, 2])
23
-def test_processor_prompt_replacements(
81
+def test_processor_prompt_replacements_regression(
82
processor_for_llava_next,
83
model_id: str,
26
- image_size: tuple[int, int],
84
num_imgs: int,
85
):
29
- """
30
- Ensure LlavaNextMultiModalProcessor handles prompt replacement properly.
31
- """
86
ctx = build_model_context(
87
model_name=model_id,
88
tokenizer_name=model_id,
@@ -37,22 +91,55 @@ def test_processor_prompt_replacements(
91
)
92
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
93
ctx = InputProcessingContext(ctx.model_config, tokenizer)
94
+ processor = processor_for_llava_next(ctx)
95
+
96
+ image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
97
+ (488, 183), (2560, 1669)]
98
+ image_sizes = [
99
+ size for w, h in image_ratios
100
+ for size in [ImageSize(w, h), ImageSize(h, w)]
101
+ ]
102
+
103
+ _test_image_prompt_replacements(
104
+ processor,
105
+ num_imgs=num_imgs,
106
+ image_sizes=image_sizes,
107
+ )
108
41
- # Build the image str / prompt based on the number of images we pass
42
- prompt = "<image>" * num_imgs
43
- mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
109
45
- # The processor will throw an error if there is a mismatch
46
- # in the prompt replacements
110
+@pytest.mark.skip("This test takes around 2 hours to run. "
111
+ "Comment this out to run it manually.")
112
+@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
113
+@pytest.mark.parametrize("num_imgs", [1])
114
+def test_processor_prompt_replacements_all(
115
+ processor_for_llava_next,
116
+ model_id: str,
117
+ num_imgs: int,
118
+):
119
+ ctx = build_model_context(
120
+ model_name=model_id,
121
+ tokenizer_name=model_id,
122
+ mm_processor_kwargs=None,
123
+ limit_mm_per_prompt={"image": num_imgs},
124
+ )
125
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126
+ ctx = InputProcessingContext(ctx.model_config, tokenizer)
127
processor = processor_for_llava_next(ctx)
48
- processed_inputs = processor.apply(prompt, mm_data, {})
128
50
- image_placeholders = processed_inputs["mm_placeholders"]["image"]
51
- assert len(image_placeholders) == num_imgs
129
+ seen_aspect_ratios = set[float]()
130
+ image_sizes = list[ImageSize]()
131
53
- first_placeholder = image_placeholders[0]
132
+ # The aspect ratio of the grid layout is between 1 and 2
133
+ # NOTE: Assumes that feature size calculation is the same if we
134
+ # swap the width and height of the image
135
+ for w, h in itertools.product(range(64, 1024), repeat=2):
136
+ aspect_ratio = w / h
137
+ if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
138
+ image_sizes.append(ImageSize(w, h))
139
+ seen_aspect_ratios.add(aspect_ratio)
140
55
- # NOTE: There is a BOS token
56
- assert first_placeholder["offset"] == 1
57
- assert first_placeholder["length"] == (
58
- len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
141
+ _test_image_prompt_replacements(
142
+ processor,
143
+ num_imgs=num_imgs,
144
+ image_sizes=image_sizes,
145
+ )
tests/models/decoder_only/vision_language/processing/test_llava_onevision.py
+107
-22
@@ -1,8 +1,13 @@
1
+import itertools
2
+from functools import partial
3
+
4
import pytest
5
from PIL import Image
6
+from pqdm.threads import pqdm
7
from transformers import AutoTokenizer
8
9
from vllm.inputs import InputProcessingContext
10
+from vllm.multimodal.parse import ImageSize
11
12
from ....utils import build_model_context
13
@@ -15,22 +20,68 @@ def processor_for_llava_onevision():
20
return LlavaOnevisionMultiModalProcessor
21
22
23
+def _validate_image_prompt_replacements_one(
24
+ processor,
25
+ num_imgs: int,
26
+ failed_size_excs: list[tuple[ImageSize, Exception]],
27
+ image_size: ImageSize,
28
+) -> None:
29
+ prompt = "<image>" * num_imgs
30
+ image = Image.new("RGB", size=image_size)
31
+ mm_data = {"image": [image] * num_imgs}
32
+
33
+ try:
34
+ # The processor will throw an error if there is a mismatch
35
+ # in the prompt replacements
36
+ processed_inputs = processor.apply(prompt, mm_data, {})
37
+
38
+ image_placeholders = processed_inputs["mm_placeholders"]["image"]
39
+ assert len(image_placeholders) == num_imgs
40
+
41
+ first_placeholder = image_placeholders[0]
42
+
43
+ assert first_placeholder["offset"] == 0
44
+ assert first_placeholder["length"] == len(
45
+ processed_inputs["prompt_token_ids"]) // num_imgs
46
+ except Exception as exc:
47
+ failed_size_excs.append((image_size, exc))
48
+
49
+
50
+def _test_image_prompt_replacements(
51
+ processor,
52
+ *,
53
+ num_imgs: int,
54
+ image_sizes: list[ImageSize],
55
+) -> None:
56
+ """
57
+ Ensure LlavaOnevisionMultiModalProcessor
58
+ handles prompt replacement properly for input images.
59
+ """
60
+ failed_size_excs = list[tuple[ImageSize, Exception]]()
61
+
62
+ validate_one = partial(
63
+ _validate_image_prompt_replacements_one,
64
+ processor,
65
+ num_imgs,
66
+ failed_size_excs,
67
+ )
68
+ pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
69
+
70
+ if failed_size_excs:
71
+ msg = "Found failing image sizes:" \
72
+ + "\n========\n".join(f"[{size}]\n{exc}"
73
+ for size, exc in failed_size_excs)
74
+ raise AssertionError(msg)
75
+
76
+
77
@pytest.mark.parametrize("model_id",
78
["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
20
-@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21
- (488, 183), (198, 176), (176, 198),
22
- (161, 184), (184, 161)])
79
@pytest.mark.parametrize("num_imgs", [1, 2])
24
-def test_processor_prompt_replacements(
80
+def test_processor_prompt_replacements_regression(
81
processor_for_llava_onevision,
82
model_id: str,
27
- image_size: tuple[int, int],
83
num_imgs: int,
84
):
30
- """
31
- Ensure LlavaOnevisionMultiModalProcessor handles prompt replacement
32
- properly.
33
- """
85
ctx = build_model_context(
86
model_name=model_id,
87
tokenizer_name=model_id,
@@ -39,22 +90,56 @@ def test_processor_prompt_replacements(
90
)
91
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
92
ctx = InputProcessingContext(ctx.model_config, tokenizer)
93
+ processor = processor_for_llava_onevision(ctx)
94
43
- # Build the image str / prompt based on the number of images we pass
44
- prompt = "<image>" * num_imgs
45
- mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
95
+ image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
96
+ (488, 183), (2560, 1669)]
97
+ image_sizes = [
98
+ size for w, h in image_ratios
99
+ for size in [ImageSize(w, h), ImageSize(h, w)]
100
+ ]
101
+
102
+ _test_image_prompt_replacements(
103
+ processor,
104
+ num_imgs=num_imgs,
105
+ image_sizes=image_sizes,
106
+ )
107
47
- # The processor will throw an error if there is a mismatch
48
- # in the prompt replacements
108
+
109
+@pytest.mark.skip("This test takes around 2 hours to run. "
110
+ "Comment this out to run it manually.")
111
+@pytest.mark.parametrize("model_id",
112
+ ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
113
+@pytest.mark.parametrize("num_imgs", [1])
114
+def test_processor_prompt_replacements_all(
115
+ processor_for_llava_onevision,
116
+ model_id: str,
117
+ num_imgs: int,
118
+):
119
+ ctx = build_model_context(
120
+ model_name=model_id,
121
+ tokenizer_name=model_id,
122
+ mm_processor_kwargs=None,
123
+ limit_mm_per_prompt={"image": num_imgs},
124
+ )
125
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126
+ ctx = InputProcessingContext(ctx.model_config, tokenizer)
127
processor = processor_for_llava_onevision(ctx)
50
- processed_inputs = processor.apply(prompt, mm_data, {})
128
52
- image_placeholders = processed_inputs["mm_placeholders"]["image"]
53
- assert len(image_placeholders) == num_imgs
129
+ seen_aspect_ratios = set[float]()
130
+ image_sizes = list[ImageSize]()
131
55
- first_placeholder = image_placeholders[0]
132
+ # The aspect ratio of the grid layout is between 1 and 6
133
+ # NOTE: Assumes that feature size calculation is the same if we
134
+ # swap the width and height of the image
135
+ for w, h in itertools.product(range(64, 1024), repeat=2):
136
+ aspect_ratio = w / h
137
+ if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
138
+ image_sizes.append(ImageSize(w, h))
139
+ seen_aspect_ratios.add(aspect_ratio)
140
57
- # NOTE: There is a BOS token
58
- assert first_placeholder["offset"] == 0
59
- assert first_placeholder["length"] == len(
60
- processed_inputs["prompt_token_ids"]) // num_imgs
141
+ _test_image_prompt_replacements(
142
+ processor,
143
+ num_imgs=num_imgs,
144
+ image_sizes=image_sizes,
145
+ )
vllm/model_executor/models/llava_next.py
+15
-22
@@ -2,7 +2,6 @@ from functools import cached_property
2
from typing import (Final, Iterable, List, Literal, Mapping, Optional,
3
Protocol, Set, Tuple, TypedDict, Union)
4
5
-import numpy as np
5
import torch
6
import torch.nn as nn
7
from transformers import BatchFeature, LlavaNextConfig, LlavaNextProcessor
@@ -74,7 +73,7 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
73
def _get_hf_processor(self):
74
return self.ctx.get_hf_processor(LlavaNextProcessor)
75
77
- # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
76
+ # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
77
def _get_num_image_tokens(
78
self,
79
*,
@@ -111,7 +110,7 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
110
111
return unpadded_feature_size + newline_feature_size + base_feature_size
112
114
- # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
113
+ # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
114
def _get_num_unpadded_features(
115
self,
116
*,
@@ -121,29 +120,23 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
120
num_patch_height: int,
121
num_patch_width: int,
122
) -> tuple[int, int]:
124
- # NOTE: Use float32 to remain consistent with HF output
125
- current_height_f = np.float32(npatches * num_patch_height)
126
- current_width_f = np.float32(npatches * num_patch_width)
123
+ current_height = npatches * num_patch_height
124
+ current_width = npatches * num_patch_width
125
128
- original_width_f = np.float32(original_width)
129
- original_height_f = np.float32(original_height)
126
+ aspect_ratio = original_width / original_height
127
+ current_aspect_ratio = current_width / current_height
128
131
- original_aspect_ratio = original_width_f / original_height_f
132
- current_aspect_ratio = current_width_f / current_height_f
133
-
134
- if original_aspect_ratio > current_aspect_ratio:
135
- scale_factor = current_width_f / original_width_f
136
- new_height = int(original_height_f * scale_factor)
137
- padding = (current_height_f - new_height) // 2
138
- current_height_f -= 2 * padding
129
+ if aspect_ratio > current_aspect_ratio:
130
+ new_height = (original_height * current_width) // original_width
131
+ padding = (current_height - new_height) // 2
132
+ current_height = current_height - (2 * padding)
133
else:
140
- scale_factor = current_height_f / original_height_f
141
- new_width = int(original_width_f * scale_factor)
142
- padding = (current_width_f - new_width) // 2
143
- current_width_f -= 2 * padding
134
+ new_width = (original_width * current_height) // original_height
135
+ padding = (current_width - new_width) // 2
136
+ current_width = current_width - (2 * padding)
137
145
- unpadded_features = int(current_height_f * current_width_f)
146
- newline_features = int(current_height_f)
138
+ unpadded_features = current_height * current_width
139
+ newline_features = current_height
140
141
return (unpadded_features, newline_features)
142
vllm/model_executor/models/llava_onevision.py
+18
-24
@@ -3,7 +3,6 @@ from functools import cached_property
3
from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4
Protocol, Set, Tuple, TypedDict, Union)
5
6
-import numpy as np
6
import torch
7
import torch.nn as nn
8
from transformers import (BatchFeature, LlavaOnevisionConfig,
@@ -98,6 +97,8 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
97
def _get_hf_processor(self):
98
return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
99
100
+ # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
101
+ # with additional logic afterwards taken from LlavaOnevisionProcessor
102
def _get_num_unpadded_features(
103
self,
104
*,
@@ -107,35 +108,28 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
108
num_patch_height: int,
109
num_patch_width: int,
110
) -> tuple[int, int]:
110
- # NOTE: Use float32 to remain consistent with HF output
111
- current_height_f = np.float32(npatches * num_patch_height)
112
- current_width_f = np.float32(npatches * num_patch_width)
111
+ current_height = npatches * num_patch_height
112
+ current_width = npatches * num_patch_width
113
114
- original_width_f = np.float32(original_width)
115
- original_height_f = np.float32(original_height)
114
+ aspect_ratio = original_width / original_height
115
+ current_aspect_ratio = current_width / current_height
116
117
- original_aspect_ratio = original_width_f / original_height_f
118
- current_aspect_ratio = current_width_f / current_height_f
119
-
120
- if original_aspect_ratio > current_aspect_ratio:
121
- scale_factor = current_width_f / original_width_f
122
- new_height = int(original_height_f * scale_factor)
123
- padding = (current_height_f - new_height) // 2
124
- current_height_f -= 2 * padding
117
+ if aspect_ratio > current_aspect_ratio:
118
+ new_height = (original_height * current_width) // original_width
119
+ padding = (current_height - new_height) // 2
120
+ current_height = current_height - (2 * padding)
121
else:
126
- scale_factor = current_height_f / original_height_f
127
- new_width = int(original_width_f * scale_factor)
128
- padding = (current_width_f - new_width) // 2
129
- current_width_f -= 2 * padding
122
+ new_width = (original_width * current_height) // original_height
123
+ padding = (current_width - new_width) // 2
124
+ current_width = current_width - (2 * padding)
125
131
- unpadded_features = int(current_height_f * current_width_f)
132
- newline_features = int(current_height_f)
126
+ unpadded_features = current_height * current_width
127
+ newline_features = current_height
128
134
- ratio = math.sqrt(current_height_f * current_width_f /
135
- (9 * npatches**2))
129
+ ratio = math.sqrt(current_height * current_width / (9 * npatches**2))
130
if ratio > 1.1:
137
- height_factor = int(current_height_f // ratio)
138
- width_factor = int(current_width_f // ratio)
131
+ height_factor = int(current_height // ratio)
132
+ width_factor = int(current_width // ratio)
133
unpadded_features = height_factor * width_factor
134
newline_features = height_factor
135