@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba214dffbe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Fix precision error in LLaVA-NeXT (#11735)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 4, 2025 at 23:45 UTC ba214dffbeec070051b61c1985ce6342c947f598
3 files changed +26 -14
tests/models/decoder_only/vision_language/processing/test_llava_next.py
+1 -2
@@ -15,10 +15,9 @@ def processor_for_llava_next():
15 return LlavaNextMultiModalProcessor
16
17
18 -# FIXME: image_size [(198, 176), (176, 198)]
18 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
19 @pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21 - (488, 183)])
20 + (488, 183), (198, 176), (176, 198)])
21 @pytest.mark.parametrize("num_imgs", [1, 2])
22 def test_processor_prompt_replacements(
23 processor_for_llava_next,
vllm/model_executor/models/llava_next.py
+10 -4
@@ -2,6 +2,7 @@ from functools import cached_property
2 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3 TypedDict, Union)
4
5 +import numpy as np
6 import torch
7 import torch.nn as nn
8 from transformers import BatchFeature, LlavaNextConfig, LlavaNextProcessor
@@ -139,16 +140,21 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
140 current_height = npatches * num_patch_height
141 current_width = npatches * num_patch_width
142
142 - original_aspect_ratio = original_width / original_height
143 - current_aspect_ratio = current_width / current_height
143 + # NOTE: HF resizes based on float32
144 + original_aspect_ratio = np.array(original_width / original_height,
145 + dtype=np.float32)
146 + current_aspect_ratio = np.array(current_width / current_height,
147 + dtype=np.float32)
148
149 if original_aspect_ratio > current_aspect_ratio:
146 - scale_factor = current_width / original_width
150 + scale_factor = np.array(current_width / original_width,
151 + dtype=np.float32)
152 new_height = int(original_height * scale_factor)
153 padding = (current_height - new_height) // 2
154 current_height -= 2 * padding
155 else:
151 - scale_factor = current_height / original_height
156 + scale_factor = np.array(current_height / original_height,
157 + dtype=np.float32)
158 new_width = int(original_width * scale_factor)
159 padding = (current_width - new_width) // 2
160 current_width -= 2 * padding
vllm/model_executor/models/llava_onevision.py
+15 -8
@@ -3,6 +3,7 @@ from functools import cached_property
3 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4 TypedDict, Union)
5
6 +import numpy as np
7 import torch
8 import torch.nn as nn
9 from transformers import (BatchFeature, LlavaOnevisionConfig,
@@ -127,18 +128,24 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
128 current_height = npatches * num_patch_height
129 current_width = npatches * num_patch_width
130
130 - original_aspect_ratio = original_width / original_height
131 - current_aspect_ratio = current_width / current_height
131 + # NOTE: HF resizes based on float32
132 + original_aspect_ratio = np.array(original_width / original_height,
133 + dtype=np.float32)
134 + current_aspect_ratio = np.array(current_width / current_height,
135 + dtype=np.float32)
136 +
137 if original_aspect_ratio > current_aspect_ratio:
133 - new_height = int(original_height *
134 - (current_width / original_width))
138 + scale_factor = np.array(current_width / original_width,
139 + dtype=np.float32)
140 + new_height = int(original_height * scale_factor)
141 padding = (current_height - new_height) // 2
136 - current_height -= padding * 2
142 + current_height -= 2 * padding
143 else:
138 - new_width = int(original_width *
139 - (current_height / original_height))
144 + scale_factor = np.array(current_height / original_height,
145 + dtype=np.float32)
146 + new_width = int(original_width * scale_factor)
147 padding = (current_width - new_width) // 2
141 - current_width -= padding * 2
148 + current_width -= 2 * padding
149
150 unpadded_features = current_height * current_width
151 newline_features = current_height