@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / daed30c4a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix feature size calculation for LLaVA-NeXT (#6982)

Cyrus Leung committed Jul 31, 2024 at 23:46 UTC daed30c4a917c870f8fbddf45e3b027710c0842b
5 files changed +98 -50
tests/models/test_llava_next.py
+70 -18
@@ -1,7 +1,7 @@
1 -from typing import List, Optional, Tuple, Type
1 +from typing import List, Optional, Tuple, Type, overload
2
3 import pytest
4 -from transformers import AutoConfig, AutoTokenizer
4 +from transformers import AutoTokenizer
5
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
@@ -50,6 +50,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
50 return hf_output_ids, hf_output_str, out_logprobs
51
52
53 +@overload
54 def run_test(
55 hf_runner: Type[HfRunner],
56 vllm_runner: Type[VllmRunner],
@@ -62,13 +63,55 @@ def run_test(
63 num_logprobs: int,
64 tensor_parallel_size: int,
65 distributed_executor_backend: Optional[str] = None,
66 +):
67 + ...
68 +
69 +
70 +@overload
71 +def run_test(
72 + hf_runner: Type[HfRunner],
73 + vllm_runner: Type[VllmRunner],
74 + image_assets: _ImageAssets,
75 + model: str,
76 + *,
77 + sizes: List[Tuple[int, int]],
78 + dtype: str,
79 + max_tokens: int,
80 + num_logprobs: int,
81 + tensor_parallel_size: int,
82 + distributed_executor_backend: Optional[str] = None,
83 +):
84 + ...
85 +
86 +
87 +def run_test(
88 + hf_runner: Type[HfRunner],
89 + vllm_runner: Type[VllmRunner],
90 + image_assets: _ImageAssets,
91 + model: str,
92 + *,
93 + size_factors: Optional[List[float]] = None,
94 + sizes: Optional[List[Tuple[int, int]]] = None,
95 + dtype: str,
96 + max_tokens: int,
97 + num_logprobs: int,
98 + tensor_parallel_size: int,
99 + distributed_executor_backend: Optional[str] = None,
100 ):
101 images = [asset.pil_image for asset in image_assets]
102
68 - inputs_per_image = [(
69 - [prompt for _ in size_factors],
70 - [rescale_image_size(image, factor) for factor in size_factors],
71 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
103 + if size_factors is not None:
104 + inputs_per_image = [(
105 + [prompt for _ in size_factors],
106 + [rescale_image_size(image, factor) for factor in size_factors],
107 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
108 + elif sizes is not None:
109 + inputs_per_image = [(
110 + [prompt for _ in sizes],
111 + [image.resize(size) for size in sizes],
112 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
113 + else:
114 + raise ValueError("You must provide either `size_factors` or `sizes`")
115
116 # max_model_len should be greater than image_feature_size
117 with vllm_runner(model,
@@ -150,15 +193,24 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
193 )
194
195
153 -@pytest.mark.parametrize("height_and_width_and_result", [(1669, 2560, 2144),
154 - (183, 488, 776)])
155 -def test_image_feature_size(height_and_width_and_result):
156 - # Avoid initializing CUDA too early in distributed tests
157 - from vllm.model_executor.models.llava_next import (
158 - get_llava_next_image_feature_size)
159 -
160 - height, width, result = height_and_width_and_result
161 - config = AutoConfig.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf")
162 - assert get_llava_next_image_feature_size(config,
163 - input_height=height,
164 - input_width=width) == result
196 +@pytest.mark.parametrize("model", models)
197 +@pytest.mark.parametrize(
198 + "sizes",
199 + [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
200 +)
201 +@pytest.mark.parametrize("dtype", ["half"])
202 +@pytest.mark.parametrize("max_tokens", [128])
203 +@pytest.mark.parametrize("num_logprobs", [5])
204 +def test_models_fixed_sizes(hf_runner, vllm_runner, image_assets, model, sizes,
205 + dtype, max_tokens, num_logprobs) -> None:
206 + run_test(
207 + hf_runner,
208 + vllm_runner,
209 + image_assets,
210 + model,
211 + sizes=sizes,
212 + dtype=dtype,
213 + max_tokens=max_tokens,
214 + num_logprobs=num_logprobs,
215 + tensor_parallel_size=1,
216 + )
vllm/model_executor/models/fuyu.py
+1 -1
@@ -169,7 +169,7 @@ def input_processor_for_fuyu(ctx: InputContext, llm_inputs: LLMInputs):
169 raise TypeError(f"Invalid image type: {type(image_data)}")
170
171 # process prompts
172 - prompt = llm_inputs["prompt"]
172 + prompt = llm_inputs.get("prompt")
173 prompt_token_ids = llm_inputs["prompt_token_ids"]
174 tokenizer = cached_get_tokenizer(model_config.model)
175 # dim0 is batch_size, dim1 is subseq_size which will always be 1
vllm/model_executor/models/internvl.py
+3 -3
@@ -20,7 +20,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 from vllm.model_executor.models import ModelRegistry
21 from vllm.model_executor.models.intern_vit import InternVisionModel
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 -from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensors
23 +from vllm.multimodal import MULTIMODAL_REGISTRY
24 from vllm.multimodal.base import MultiModalInputs
25 from vllm.multimodal.image import cached_get_tokenizer
26 from vllm.sequence import IntermediateTensors, SamplerOutput
@@ -43,7 +43,7 @@ MAX_IMAGE_FEATURE_SIZE_HEIGHT = 500
43
44 class InternVLImagePixelInputs(TypedDict):
45 type: Literal["pixel_values"]
46 - data: BatchedTensors
46 + data: Union[torch.Tensor, List[torch.Tensor]]
47 """
48 Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
49
@@ -193,7 +193,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
193 tokenizer = cached_get_tokenizer(model_config.tokenizer,
194 trust_remote_code=True)
195
196 - prompt = llm_inputs["prompt"]
196 + prompt = llm_inputs.get("prompt")
197 prompt_token_ids = llm_inputs["prompt_token_ids"]
198 if prompt is None:
199 prompt = tokenizer.decode(prompt_token_ids)
vllm/model_executor/models/llava_next.py
+22 -26
@@ -21,7 +21,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 from vllm.model_executor.models.clip import CLIPVisionModel
22 from vllm.model_executor.models.llama import LlamaModel
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 -from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensors
24 +from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.sequence import IntermediateTensors, SamplerOutput
26
27 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
@@ -43,7 +43,7 @@ MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
43
44 class LlavaNextImagePixelInputs(TypedDict):
45 type: Literal["pixel_values"]
46 - data: BatchedTensors
46 + data: Union[torch.Tensor, List[torch.Tensor]]
47 """
48 Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
49
@@ -62,31 +62,26 @@ class LlavaNextImagePixelInputs(TypedDict):
62 LlavaNextImageInputs = LlavaNextImagePixelInputs
63
64
65 -# Taken from: https://github.com/huggingface/text-generation-inference/blob/v2.0.4/server/text_generation_server/models/vlm_causal_lm.py#L91
66 -# NOTE: new_height and new_width are further incremented to properly invert the
67 -# floordiv operation: https://github.com/huggingface/transformers/blob/v4.42.2/src/transformers/models/llava_next/modeling_llava_next.py#L133
65 +# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L79
66 def _get_llava_next_num_unpadded_features(
69 - height: int,
70 - width: int,
67 + original_height: int,
68 + original_width: int,
69 npatches: int,
70 num_patch_height: int,
71 num_patch_width: int,
72 ) -> Tuple[int, int]:
73 current_height = npatches * num_patch_height
74 current_width = npatches * num_patch_width
77 - current_height = torch.tensor(current_height).to("cuda")
78 - current_width = torch.tensor(current_width).to("cuda")
75
80 - aspect_ratio: float = width / height
81 - current_aspect_ratio: float = current_width / current_height
76 + aspect_ratio = original_width / original_height
77 + current_aspect_ratio = current_width / current_height
78 +
79 if aspect_ratio > current_aspect_ratio:
83 - scale_factor = current_width / width
84 - new_height = int(height * scale_factor)
80 + new_height = (original_height * current_width) // original_width
81 padding = (current_height - new_height) // 2
82 current_height -= padding * 2
83 else:
88 - scale_factor = current_height / height
89 - new_width = int(width * scale_factor)
84 + new_width = (original_width * current_height) // original_height
85 padding = (current_width - new_width) // 2
86 current_width -= padding * 2
87
@@ -95,7 +90,7 @@ def _get_llava_next_num_unpadded_features(
90 return (unpadded_features, newline_features)
91
92
98 -# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.0.4/server/text_generation_server/models/vlm_causal_lm.py#L111
93 +# Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
94 def get_llava_next_image_feature_size(
95 hf_config: LlavaNextConfig,
96 *,
@@ -111,9 +106,7 @@ def get_llava_next_image_feature_size(
106 )
107 base_feature_size = num_patches * num_patches
108
114 - # Note: We follow the "wrong" width/height order
115 - # [ref: PR huggingface/transformers#31588]
116 - num_patch_width, num_patch_height = get_anyres_image_grid_shape(
109 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
110 image_size=(input_height, input_width),
111 grid_pinpoints=hf_config.image_grid_pinpoints,
112 patch_size=vision_config.image_size,
@@ -349,11 +342,12 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
342 if patch_embeddings.shape[0] > 1:
343 other_patch_embeds = patch_embeddings[1:]
344
345 + # Move to CPU to avoid floating-point errors
346 + orig_height, orig_width = image_size.tolist()
347 +
348 # image_aspect_ratio == "anyres"
353 - # Note: We follow the "wrong" width/height order
354 - # [ref: PR huggingface/transformers#31588]
355 - num_patch_width, num_patch_height = get_anyres_image_grid_shape(
356 - image_size,
349 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
350 + (orig_height, orig_width),
351 self.config.image_grid_pinpoints,
352 self.config.vision_config.image_size,
353 )
@@ -365,7 +359,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
359 .permute(4, 0, 2, 1, 3).contiguous() \
360 .flatten(1, 2).flatten(2, 3)
361 other_patch_embeds = unpad_image(other_patch_embeds,
368 - image_size)
362 + (orig_height, orig_width))
363 other_patch_embeds = torch.cat((
364 other_patch_embeds,
365 self.image_newline[:, None, None] \
@@ -398,7 +392,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
392 def _process_image_pixels(
393 self,
394 inputs: LlavaNextImagePixelInputs,
401 - ) -> BatchedTensors:
395 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
396 assert self.vision_tower is not None
397
398 pixel_values = inputs["data"]
@@ -425,7 +419,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
419 ]
420
421 def _process_image_input(
428 - self, image_input: LlavaNextImageInputs) -> BatchedTensors:
422 + self,
423 + image_input: LlavaNextImageInputs,
424 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
425 patch_embeddings = self._process_image_pixels(image_input)
426
427 image_sizes = image_input.get("image_sizes")
vllm/model_executor/models/phi3v.py
+2 -2
@@ -36,7 +36,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
36 from vllm.model_executor.models.clip import CLIPVisionModel
37 from vllm.model_executor.models.llama import LlamaModel
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 -from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensors
39 +from vllm.multimodal import MULTIMODAL_REGISTRY
40 from vllm.multimodal.image import cached_get_tokenizer
41 from vllm.sequence import IntermediateTensors, SamplerOutput
42
@@ -261,7 +261,7 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
261
262 class Phi3VImagePixelInputs(TypedDict):
263 type: Literal["pixel_values"]
264 - data: BatchedTensors
264 + data: Union[torch.Tensor, List[torch.Tensor]]
265 """
266 Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
267