A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[VLM] Improve consistency between feature size calculation and dummy data for profiling (#6146)
Roger Wang committed
Jul 4, 2024 at 18:29 UTC
a41357e941b81067cd053e94da41adae470990df
2 files changed
+18
-26
vllm/model_executor/models/llava_next.py
+8
-13
@@ -37,6 +37,9 @@ _KEYS_TO_MODIFY_MAPPING = {
37
"language_model.model": "language_model",
38
}
39
40
+# Result in the max possible feature size (2x2 grid of 336x336px tiles)
41
+MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
42
+
43
44
class LlavaNextImagePixelInputs(TypedDict):
45
type: Literal["pixel_values"]
@@ -128,13 +131,11 @@ def get_llava_next_image_feature_size(
131
132
133
def get_max_llava_next_image_tokens(ctx: InputContext):
131
- # Result in the max possible feature size (2x2 grid of 336x336px tiles)
132
- dummy_height = dummy_width = 448
134
135
return get_llava_next_image_feature_size(
136
ctx.get_hf_config(LlavaNextConfig),
136
- input_height=dummy_height,
137
- input_width=dummy_width,
137
+ input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
138
+ input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
139
)
140
141
@@ -142,13 +143,7 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
143
hf_config = ctx.get_hf_config(LlavaNextConfig)
144
vision_config = hf_config.vision_config
145
145
- # Result in the max possible feature size (2x2 grid of 336x336px tiles)
146
- dummy_height = dummy_width = 448
147
- image_feature_size = get_llava_next_image_feature_size(
148
- hf_config,
149
- input_height=dummy_height,
150
- input_width=dummy_width,
151
- )
146
+ image_feature_size = get_max_llava_next_image_tokens(ctx)
147
148
if isinstance(vision_config, CLIPVisionConfig):
149
seq_data = dummy_seq_data_for_clip(
@@ -160,8 +155,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
155
156
mm_data = dummy_image_for_clip(
157
vision_config,
163
- image_width_override=dummy_width,
164
- image_height_override=dummy_height,
158
+ image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
159
+ image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
160
)
161
162
return seq_data, mm_data
vllm/model_executor/models/phi3v.py
+10
-13
@@ -53,6 +53,10 @@ _KEYS_TO_MODIFY_MAPPING = {
53
# Cannot find the following 2 numbers from hf config.
54
_IMAGE_TOKEN_ID = 32044
55
56
+# Result in the max possible feature size (h:w = 16:1)
57
+MAX_IMAGE_FEATURE_SIZE_HEIGHT = 8000
58
+MAX_IMAGE_FEATURE_SIZE_WIDTH = 50
59
+
60
CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
61
hidden_act="quick_gelu",
62
hidden_size=1024,
@@ -322,24 +326,17 @@ def get_phi3v_image_feature_size(
326
327
328
def get_max_phi3v_image_tokens(ctx: InputContext):
325
- # Result in the max possible feature size (h:w = 16:1)
326
- dummy_height, dummy_width = 8000, 50
329
330
return get_phi3v_image_feature_size(
331
ctx.get_hf_config(PretrainedConfig),
330
- input_height=dummy_height,
331
- input_width=dummy_width,
332
+ input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
333
+ input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
334
)
335
336
337
def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
336
- # Result in the max possible feature size (h:w = 16:1)
337
- dummy_height, dummy_width = 8000, 50
338
- image_feature_size = get_phi3v_image_feature_size(
339
- ctx.get_hf_config(PretrainedConfig),
340
- input_height=dummy_height,
341
- input_width=dummy_width,
342
- )
338
+
339
+ image_feature_size = get_max_phi3v_image_tokens(ctx)
340
341
seq_data = dummy_seq_data_for_clip(
342
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
@@ -349,8 +346,8 @@ def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
346
)
347
mm_data = dummy_image_for_clip(
348
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
352
- image_width_override=dummy_width,
353
- image_height_override=dummy_height,
349
+ image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
350
+ image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
351
)
352
353
return seq_data, mm_data