@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a41357e941
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[VLM] Improve consistency between feature size calculation and dummy data for profiling (#6146)

Roger Wang committed Jul 4, 2024 at 18:29 UTC a41357e941b81067cd053e94da41adae470990df
2 files changed +18 -26
vllm/model_executor/models/llava_next.py
+8 -13
@@ -37,6 +37,9 @@ _KEYS_TO_MODIFY_MAPPING = {
37 "language_model.model": "language_model",
38 }
39
40 +# Result in the max possible feature size (2x2 grid of 336x336px tiles)
41 +MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
42 +
43
44 class LlavaNextImagePixelInputs(TypedDict):
45 type: Literal["pixel_values"]
@@ -128,13 +131,11 @@ def get_llava_next_image_feature_size(
131
132
133 def get_max_llava_next_image_tokens(ctx: InputContext):
131 - # Result in the max possible feature size (2x2 grid of 336x336px tiles)
132 - dummy_height = dummy_width = 448
134
135 return get_llava_next_image_feature_size(
136 ctx.get_hf_config(LlavaNextConfig),
136 - input_height=dummy_height,
137 - input_width=dummy_width,
137 + input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
138 + input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
139 )
140
141
@@ -142,13 +143,7 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
143 hf_config = ctx.get_hf_config(LlavaNextConfig)
144 vision_config = hf_config.vision_config
145
145 - # Result in the max possible feature size (2x2 grid of 336x336px tiles)
146 - dummy_height = dummy_width = 448
147 - image_feature_size = get_llava_next_image_feature_size(
148 - hf_config,
149 - input_height=dummy_height,
150 - input_width=dummy_width,
151 - )
146 + image_feature_size = get_max_llava_next_image_tokens(ctx)
147
148 if isinstance(vision_config, CLIPVisionConfig):
149 seq_data = dummy_seq_data_for_clip(
@@ -160,8 +155,8 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
155
156 mm_data = dummy_image_for_clip(
157 vision_config,
163 - image_width_override=dummy_width,
164 - image_height_override=dummy_height,
158 + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
159 + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
160 )
161
162 return seq_data, mm_data
vllm/model_executor/models/phi3v.py
+10 -13
@@ -53,6 +53,10 @@ _KEYS_TO_MODIFY_MAPPING = {
53 # Cannot find the following 2 numbers from hf config.
54 _IMAGE_TOKEN_ID = 32044
55
56 +# Result in the max possible feature size (h:w = 16:1)
57 +MAX_IMAGE_FEATURE_SIZE_HEIGHT = 8000
58 +MAX_IMAGE_FEATURE_SIZE_WIDTH = 50
59 +
60 CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
61 hidden_act="quick_gelu",
62 hidden_size=1024,
@@ -322,24 +326,17 @@ def get_phi3v_image_feature_size(
326
327
328 def get_max_phi3v_image_tokens(ctx: InputContext):
325 - # Result in the max possible feature size (h:w = 16:1)
326 - dummy_height, dummy_width = 8000, 50
329
330 return get_phi3v_image_feature_size(
331 ctx.get_hf_config(PretrainedConfig),
330 - input_height=dummy_height,
331 - input_width=dummy_width,
332 + input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
333 + input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
334 )
335
336
337 def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
336 - # Result in the max possible feature size (h:w = 16:1)
337 - dummy_height, dummy_width = 8000, 50
338 - image_feature_size = get_phi3v_image_feature_size(
339 - ctx.get_hf_config(PretrainedConfig),
340 - input_height=dummy_height,
341 - input_width=dummy_width,
342 - )
338 +
339 + image_feature_size = get_max_phi3v_image_tokens(ctx)
340
341 seq_data = dummy_seq_data_for_clip(
342 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
@@ -349,8 +346,8 @@ def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
346 )
347 mm_data = dummy_image_for_clip(
348 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
352 - image_width_override=dummy_width,
353 - image_height_override=dummy_height,
349 + image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
350 + image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
351 )
352
353 return seq_data, mm_data