A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Fix LLaVA-NeXT feature size precision error (for real) (#11772)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 7, 2025 at 09:10 UTC
08fb75c72e39dcb4f0751dc59583b95bda4d3656
4 files changed
+47
-45
tests/models/decoder_only/vision_language/processing/test_llava_next.py
+2
-1
@@ -17,7 +17,8 @@ def processor_for_llava_next():
17
18
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
19
@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
20
- (488, 183), (198, 176), (176, 198)])
20
+ (488, 183), (198, 176), (176, 198),
21
+ (161, 184), (184, 161)])
22
@pytest.mark.parametrize("num_imgs", [1, 2])
23
def test_processor_prompt_replacements(
24
processor_for_llava_next,
tests/models/decoder_only/vision_language/processing/test_llava_onevision.py
+2
-1
@@ -18,7 +18,8 @@ def processor_for_llava_onevision():
18
@pytest.mark.parametrize("model_id",
19
["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
20
@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21
- (488, 183), (198, 176), (176, 198)])
21
+ (488, 183), (198, 176), (176, 198),
22
+ (161, 184), (184, 161)])
23
@pytest.mark.parametrize("num_imgs", [1, 2])
24
def test_processor_prompt_replacements(
25
processor_for_llava_onevision,
vllm/model_executor/models/llava_next.py
+19
-20
@@ -121,30 +121,29 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
121
num_patch_height: int,
122
num_patch_width: int,
123
) -> tuple[int, int]:
124
- current_height = npatches * num_patch_height
125
- current_width = npatches * num_patch_width
126
-
124
# NOTE: Use float32 to remain consistent with HF output
128
- original_aspect_ratio = np.array(original_width / original_height,
129
- dtype=np.float32)
130
- current_aspect_ratio = np.array(current_width / current_height,
131
- dtype=np.float32)
125
+ current_height_f = np.float32(npatches * num_patch_height)
126
+ current_width_f = np.float32(npatches * num_patch_width)
127
+
128
+ original_width_f = np.float32(original_width)
129
+ original_height_f = np.float32(original_height)
130
+
131
+ original_aspect_ratio = original_width_f / original_height_f
132
+ current_aspect_ratio = current_width_f / current_height_f
133
134
if original_aspect_ratio > current_aspect_ratio:
134
- scale_factor = np.array(current_width / original_width,
135
- dtype=np.float32)
136
- new_height = int(original_height * scale_factor)
137
- padding = (current_height - new_height) // 2
138
- current_height -= 2 * padding
135
+ scale_factor = current_width_f / original_width_f
136
+ new_height = int(original_height_f * scale_factor)
137
+ padding = (current_height_f - new_height) // 2
138
+ current_height_f -= 2 * padding
139
else:
140
- scale_factor = np.array(current_height / original_height,
141
- dtype=np.float32)
142
- new_width = int(original_width * scale_factor)
143
- padding = (current_width - new_width) // 2
144
- current_width -= 2 * padding
145
-
146
- unpadded_features = current_height * current_width
147
- newline_features = current_height
140
+ scale_factor = current_height_f / original_height_f
141
+ new_width = int(original_width_f * scale_factor)
142
+ padding = (current_width_f - new_width) // 2
143
+ current_width_f -= 2 * padding
144
+
145
+ unpadded_features = int(current_height_f * current_width_f)
146
+ newline_features = int(current_height_f)
147
148
return (unpadded_features, newline_features)
149
vllm/model_executor/models/llava_onevision.py
+24
-23
@@ -107,36 +107,37 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
107
num_patch_height: int,
108
num_patch_width: int,
109
) -> tuple[int, int]:
110
- current_height = npatches * num_patch_height
111
- current_width = npatches * num_patch_width
112
-
110
# NOTE: Use float32 to remain consistent with HF output
114
- original_aspect_ratio = np.array(original_width / original_height,
115
- dtype=np.float32)
116
- current_aspect_ratio = np.array(current_width / current_height,
117
- dtype=np.float32)
111
+ current_height_f = np.float32(npatches * num_patch_height)
112
+ current_width_f = np.float32(npatches * num_patch_width)
113
+
114
+ original_width_f = np.float32(original_width)
115
+ original_height_f = np.float32(original_height)
116
+
117
+ original_aspect_ratio = original_width_f / original_height_f
118
+ current_aspect_ratio = current_width_f / current_height_f
119
120
if original_aspect_ratio > current_aspect_ratio:
120
- scale_factor = np.array(current_width / original_width,
121
- dtype=np.float32)
122
- new_height = int(original_height * scale_factor)
123
- padding = (current_height - new_height) // 2
124
- current_height -= 2 * padding
121
+ scale_factor = current_width_f / original_width_f
122
+ new_height = int(original_height_f * scale_factor)
123
+ padding = (current_height_f - new_height) // 2
124
+ current_height_f -= 2 * padding
125
else:
126
- scale_factor = np.array(current_height / original_height,
127
- dtype=np.float32)
128
- new_width = int(original_width * scale_factor)
129
- padding = (current_width - new_width) // 2
130
- current_width -= 2 * padding
126
+ scale_factor = current_height_f / original_height_f
127
+ new_width = int(original_width_f * scale_factor)
128
+ padding = (current_width_f - new_width) // 2
129
+ current_width_f -= 2 * padding
130
132
- unpadded_features = current_height * current_width
133
- newline_features = current_height
131
+ unpadded_features = int(current_height_f * current_width_f)
132
+ newline_features = int(current_height_f)
133
135
- ratio = math.sqrt(current_height * current_width / (9 * npatches**2))
134
+ ratio = math.sqrt(current_height_f * current_width_f /
135
+ (9 * npatches**2))
136
if ratio > 1.1:
137
- unpadded_features = int(current_height // ratio) * int(
138
- current_width // ratio)
139
- newline_features = int(current_height // ratio)
137
+ height_factor = int(current_height_f // ratio)
138
+ width_factor = int(current_width_f // ratio)
139
+ unpadded_features = height_factor * width_factor
140
+ newline_features = height_factor
141
142
return (unpadded_features, newline_features)
143