A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[bug fix] Fix llava next feature size calculation. (#6339)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com>
xwjiang2010 committed
Jul 11, 2024 at 10:21 UTC
1df43de9bb2cceecdc0dc2dc5c650a327aeabe0f
2 files changed
+23
-9
tests/models/test_llava_next.py
+13
-1
@@ -1,8 +1,10 @@
1
from typing import List, Optional, Tuple
2
3
import pytest
4
-from transformers import AutoTokenizer
4
+from transformers import AutoConfig, AutoTokenizer
5
6
+from vllm.model_executor.models.llava_next import (
7
+ get_llava_next_image_feature_size)
8
from vllm.multimodal.utils import rescale_image_size
9
from vllm.sequence import SampleLogprobs
10
@@ -120,3 +122,13 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
122
name_0="hf",
123
name_1="vllm",
124
)
125
+
126
+
127
+@pytest.mark.parametrize("height_and_width_and_result", [(1669, 2560, 2144),
128
+ (183, 488, 776)])
129
+def test_image_feature_size(height_and_width_and_result):
130
+ height, width, result = height_and_width_and_result
131
+ config = AutoConfig.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
132
+ assert get_llava_next_image_feature_size(config,
133
+ input_height=height,
134
+ input_width=width) == result
vllm/model_executor/models/llava_next.py
+10
-8
@@ -74,19 +74,21 @@ def _get_llava_next_num_unpadded_features(
74
) -> Tuple[int, int]:
75
current_height = npatches * num_patch_height
76
current_width = npatches * num_patch_width
77
+ current_height = torch.tensor(current_height).to("cuda")
78
+ current_width = torch.tensor(current_width).to("cuda")
79
80
aspect_ratio: float = width / height
81
current_aspect_ratio: float = current_width / current_height
82
if aspect_ratio > current_aspect_ratio:
81
- new_height = (height * current_width) // width
82
- if new_height % 2 == 1:
83
- new_height += 1
84
- current_height = new_height
83
+ scale_factor = current_width / width
84
+ new_height = int(height * scale_factor)
85
+ padding = (current_height - new_height) // 2
86
+ current_height -= padding * 2
87
else:
86
- new_width = (width * current_height) // height
87
- if new_width % 2 == 1:
88
- new_width += 1
89
- current_width = new_width
88
+ scale_factor = current_height / height
89
+ new_width = int(width * scale_factor)
90
+ padding = (current_width - new_width) // 2
91
+ current_width -= padding * 2
92
93
unpadded_features = current_height * current_width
94
newline_features = current_height