A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Fix precision error in LLaVA-NeXT (#11735)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 4, 2025 at 23:45 UTC
ba214dffbeec070051b61c1985ce6342c947f598
3 files changed
+26
-14
tests/models/decoder_only/vision_language/processing/test_llava_next.py
+1
-2
@@ -15,10 +15,9 @@ def processor_for_llava_next():
15
return LlavaNextMultiModalProcessor
16
17
18
-# FIXME: image_size [(198, 176), (176, 198)]
18
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
19
@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21
- (488, 183)])
20
+ (488, 183), (198, 176), (176, 198)])
21
@pytest.mark.parametrize("num_imgs", [1, 2])
22
def test_processor_prompt_replacements(
23
processor_for_llava_next,
vllm/model_executor/models/llava_next.py
+10
-4
@@ -2,6 +2,7 @@ from functools import cached_property
2
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3
TypedDict, Union)
4
5
+import numpy as np
6
import torch
7
import torch.nn as nn
8
from transformers import BatchFeature, LlavaNextConfig, LlavaNextProcessor
@@ -139,16 +140,21 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
140
current_height = npatches * num_patch_height
141
current_width = npatches * num_patch_width
142
142
- original_aspect_ratio = original_width / original_height
143
- current_aspect_ratio = current_width / current_height
143
+ # NOTE: HF resizes based on float32
144
+ original_aspect_ratio = np.array(original_width / original_height,
145
+ dtype=np.float32)
146
+ current_aspect_ratio = np.array(current_width / current_height,
147
+ dtype=np.float32)
148
149
if original_aspect_ratio > current_aspect_ratio:
146
- scale_factor = current_width / original_width
150
+ scale_factor = np.array(current_width / original_width,
151
+ dtype=np.float32)
152
new_height = int(original_height * scale_factor)
153
padding = (current_height - new_height) // 2
154
current_height -= 2 * padding
155
else:
151
- scale_factor = current_height / original_height
156
+ scale_factor = np.array(current_height / original_height,
157
+ dtype=np.float32)
158
new_width = int(original_width * scale_factor)
159
padding = (current_width - new_width) // 2
160
current_width -= 2 * padding
vllm/model_executor/models/llava_onevision.py
+15
-8
@@ -3,6 +3,7 @@ from functools import cached_property
3
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
TypedDict, Union)
5
6
+import numpy as np
7
import torch
8
import torch.nn as nn
9
from transformers import (BatchFeature, LlavaOnevisionConfig,
@@ -127,18 +128,24 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
128
current_height = npatches * num_patch_height
129
current_width = npatches * num_patch_width
130
130
- original_aspect_ratio = original_width / original_height
131
- current_aspect_ratio = current_width / current_height
131
+ # NOTE: HF resizes based on float32
132
+ original_aspect_ratio = np.array(original_width / original_height,
133
+ dtype=np.float32)
134
+ current_aspect_ratio = np.array(current_width / current_height,
135
+ dtype=np.float32)
136
+
137
if original_aspect_ratio > current_aspect_ratio:
133
- new_height = int(original_height *
134
- (current_width / original_width))
138
+ scale_factor = np.array(current_width / original_width,
139
+ dtype=np.float32)
140
+ new_height = int(original_height * scale_factor)
141
padding = (current_height - new_height) // 2
136
- current_height -= padding * 2
142
+ current_height -= 2 * padding
143
else:
138
- new_width = int(original_width *
139
- (current_height / original_height))
144
+ scale_factor = np.array(current_height / original_height,
145
+ dtype=np.float32)
146
+ new_width = int(original_width * scale_factor)
147
padding = (current_width - new_width) // 2
141
- current_width -= padding * 2
148
+ current_width -= 2 * padding
149
150
unpadded_features = current_height * current_width
151
newline_features = current_height