A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[VLM] Cleanup validation and update docs (#6149)
Cyrus Leung committed
Jul 5, 2024 at 13:49 UTC
ea4b5704833cb31377bcbc2f00959f6c09909099
3 files changed
+87
-82
vllm/model_executor/models/llava.py
+30
-18
@@ -149,14 +149,16 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
149
config.vocab_size, logit_scale)
150
self.sampler = Sampler()
151
152
- def _validate_image_data(self, data: torch.Tensor) -> torch.Tensor:
153
- if list(data.shape)[1:] != [
154
- 3, self.config.vision_config.image_size,
155
- self.config.vision_config.image_size
156
- ]:
152
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
153
+ h = w = self.config.vision_config.image_size
154
+ expected_dims = (3, h, w)
155
+ actual_dims = tuple(data.shape[1:])
156
+
157
+ if actual_dims != expected_dims:
158
+ expected_expr = ("batch_size", *map(str, expected_dims))
159
raise ValueError(
158
- "The expected image tensor shape is batch dimension plus "
159
- "channel, height and width.")
160
+ f"The expected shape of pixel values is {expected_expr}. "
161
+ f"You supplied {tuple(data.shape)}.")
162
163
return data
164
@@ -173,7 +175,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
175
176
return LlavaImagePixelInputs(
177
type="pixel_values",
176
- data=self._validate_image_data(pixel_values),
178
+ data=self._validate_pixel_values(pixel_values),
179
)
180
181
def _select_image_features(self, image_features: torch.Tensor, *,
@@ -226,18 +228,25 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
228
229
One key thing to understand is the `input_ids` already accounts for the
230
positions of the to-be-inserted image embeddings.
231
+
232
Concretely, consider a text prompt:
230
- "<image>\nUSER: What's the content of the image?\nASSISTANT:".
233
+ `"USER: <image>\\nWhat's the content of the image?\\nASSISTANT:"`.
234
+
235
Tokenizer outputs:
232
- [1, 32000, 29871, 13, 11889, 29901, 1724, 29915, 29879, 278,
233
- 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901].
234
- The to-be-inserted image has a size of 576 (24 * 24) along the context
235
- length dimension.
236
- `input_ids` is thus [1, 32000, ..., 32000, 29871, 13, 11889, 29901,
237
- 1724, 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933,
238
- 9047, 13566, 29901].
239
- There will be 576 `32000` in the `input_ids`.
240
- (32000 is the token id for `<image>`.)
236
+ `[1, 3148, 1001, 29901, 29871, 32000, 29871, 13, 5618, 29915, 29879,
237
+ 278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901]`.
238
+
239
+ To reserve space in KV cache, we have to insert placeholder tokens
240
+ before they are inputted to the model, so the input processor prepends
241
+ additional image tokens (denoted as `32000`), resulting in:
242
+ `[1, 3148, 1001, 29901, 29871, 32000, ..., 32000, 29871, 13, 5618,
243
+ 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566,
244
+ 29901]`.
245
+
246
+ We insert 575 tokens so that including the original image token in the
247
+ input, there are a total of 576 (24 * 24) image tokens, which
248
+ corresponds to the number of image tokens inputted to the language
249
+ model, i.e. the number of image tokens outputted by the visual encoder.
250
251
This way, the `positions` and `attn_metadata` are consistent
252
with the `input_ids`.
@@ -246,6 +255,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
255
input_ids: Flattened (concatenated) input_ids corresponding to a
256
batch.
257
pixel_values: The pixels in each input image.
258
+
259
+ See also:
260
+ :class:`LlavaImageInputs`
261
"""
262
image_input = self._parse_and_validate_image_input(**kwargs)
263
vllm/model_executor/models/llava_next.py
+41
-50
@@ -47,7 +47,8 @@ class LlavaNextImagePixelInputs(TypedDict):
47
"""
48
Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
49
50
- Note that `num_patches` may be different for each batch.
50
+ Note that `num_patches` may be different for each batch, in which case
51
+ the data is passed as a list instead of a batched tensor.
52
"""
53
54
image_sizes: NotRequired[torch.Tensor]
@@ -255,40 +256,20 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
256
self, data: Union[torch.Tensor, List[torch.Tensor]]
257
) -> Union[torch.Tensor, List[torch.Tensor]]:
258
258
- def _validate_shape(data: torch.Tensor):
259
-
260
- dim = data.dim()
261
- height = width = self.config.vision_config.image_size
262
- # All 4d image tensors have the same number of patches,
263
- # so data is a 5d batch of these tensors
264
- if dim == 5:
265
- if list(data.shape)[2:] != [
266
- 3, self.config.vision_config.image_size,
267
- self.config.vision_config.image_size
268
- ]:
269
- raise ValueError(
270
- "Expected pixel value tensor in shape of: (batch size, "
271
- f"patch number, 3, {height}, {width}), got {data.shape}"
272
- )
273
-
274
- # 4d image tensors have different number of patches,
275
- # so data is each individual tensor.
276
- elif dim == 4:
277
- if list(data.shape)[1:] != [
278
- 3, self.config.vision_config.image_size,
279
- self.config.vision_config.image_size
280
- ]:
281
- raise ValueError(
282
- "Expected pixel value tensor in shape of: (patch "
283
- f"number, 3, {height}, {width}), got {data.shape}")
284
- else:
259
+ h = w = self.config.vision_config.image_size
260
+ expected_dims = (3, h, w)
261
+
262
+ def _validate_shape(d: torch.Tensor):
263
+ actual_dims = tuple(d.shape[1:])
264
+
265
+ if actual_dims != expected_dims:
266
+ expected_expr = ("num_patches", *map(str, expected_dims))
267
raise ValueError(
286
- f"Invalid pixel value tensor of shape {data.shape}")
268
+ "The expected shape of pixel values in each batch element "
269
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
270
288
- if isinstance(data, torch.Tensor):
289
- _validate_shape(data)
290
- else:
291
- [_validate_shape(d) for d in data]
271
+ for d in data:
272
+ _validate_shape(d)
273
274
return data
275
@@ -464,18 +445,33 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
445
446
One key thing to understand is the `input_ids` already accounts for the
447
positions of the to-be-inserted image embeddings.
448
+
449
Concretely, consider a text prompt:
468
- "<image>\nUSER: What's the content of the image?\nASSISTANT:".
450
+ `"A chat between a curious human and an artificial intelligence
451
+ assistant. The assistant gives helpful, detailed, and polite answers to
452
+ the human's questions.
453
+ USER: <image>\\nWhat is shown in this image? ASSISTANT:"`.
454
+
455
Tokenizer outputs:
470
- [1, 32000, 29871, 13, 11889, 29901, 1724, 29915, 29879, 278,
471
- 2793, 310, 278, 1967, 29973, 13, 22933, 9047, 13566, 29901].
472
- The to-be-inserted image has a size of 576 (24 * 24) along the context
473
- length dimension.
474
- `input_ids` is thus [1, 32000, ..., 32000, 29871, 13, 11889, 29901,
475
- 1724, 29915, 29879, 278, 2793, 310, 278, 1967, 29973, 13, 22933,
476
- 9047, 13566, 29901].
477
- There will be 576 `32000` in the `input_ids`.
478
- (32000 is the token id for `<image>`.)
456
+ `[1, 319, 13563, 1546, 263, 12758, 5199, 322, 385, 23116, 21082, 20255,
457
+ 29889, 450, 20255, 4076, 8444, 29892, 13173, 29892, 322, 1248, 568,
458
+ 6089, 304, 278, 5199, 29915, 29879, 5155, 29889, 3148, 1001, 29901,
459
+ 29871, 32000, 13, 5618, 338, 4318, 297, 445, 1967, 29973, 319, 1799,
460
+ 9047, 13566, 29901]`.
461
+
462
+ To reserve space in KV cache, we have to insert placeholder tokens
463
+ before they are inputted to the model, so the input processor prepends
464
+ additional image tokens (denoted as `32000`), resulting in:
465
+ `[1, 319, 13563, 1546, 263, 12758, 5199, 322, 385, 23116, 21082, 20255,
466
+ 29889, 450, 20255, 4076, 8444, 29892, 13173, 29892, 322, 1248, 568,
467
+ 6089, 304, 278, 5199, 29915, 29879, 5155, 29889, 3148, 1001, 29901,
468
+ 29871, 32000, ..., 32000, 13, 5618, 338, 4318, 297, 445, 1967, 29973,
469
+ 319, 1799, 9047, 13566, 29901]`.
470
+
471
+ Unlike in LLaVA-1.5, the number of image tokens inputted to the language
472
+ model depends on the original size of the input image. Including the
473
+ original image token in the input, the required number of image tokens
474
+ is given by :func:`get_llava_next_image_feature_size`.
475
476
This way, the `positions` and `attn_metadata` are consistent
477
with the `input_ids`.
@@ -484,15 +480,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
480
input_ids: Flattened (concatenated) input_ids corresponding to a
481
batch.
482
pixel_values: The pixels in each grid patch for each input image.
487
- Expects a batch with shape `[1, num_patches, 3, h, w]`.
483
image_sizes: The original `(height, width)` for each input image.
489
- Expects a batch with shape `[1, 2]`.
490
-
484
+
485
See also:
492
- Each input maps to huggingface implementation, as follows:
493
-
494
- - `pixel_values`: https://github.com/huggingface/transformers/blob/v4.41.1/src/transformers/models/llava_next/modeling_llava_next.py#L690
495
- - `image_sizes`: https://github.com/huggingface/transformers/blob/v4.41.1/src/transformers/models/llava_next/modeling_llava_next.py#L691
486
+ :class:`LlavaNextImageInputs`
487
"""
488
image_input = self._parse_and_validate_image_input(**kwargs)
489
vllm/model_executor/models/phi3v.py
+16
-14
@@ -263,7 +263,8 @@ class Phi3VImagePixelInputs(TypedDict):
263
"""
264
Shape: `(batch_size, 1 + num_patches, num_channels, height, width)`
265
266
- Note that `num_patches` may be different for each batch.
266
+ Note that `num_patches` may be different for each batch, in which case
267
+ the data is passed as a list instead of a batched tensor.
268
"""
269
270
image_sizes: torch.Tensor
@@ -466,8 +467,8 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
467
def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
468
if list(data.shape[1:]) != [2]:
469
raise ValueError(
469
- f"The expected image sizes shape is batch dimension plus "
470
- f"{[2]}. You supplied {data.shape}.")
470
+ f"The expected shape of image sizes is batch dimension plus "
471
+ f"{[2]}. You supplied {tuple(data.shape)}.")
472
473
return data
474
@@ -475,19 +476,20 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
476
self, data: Union[torch.Tensor, List[torch.Tensor]]
477
) -> Union[torch.Tensor, List[torch.Tensor]]:
478
478
- def _validate_shape(data: torch.Tensor):
479
- if list(data.shape)[2:] != [
480
- 3, CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size,
481
- CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size
482
- ]:
479
+ h = w = CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size
480
+ expected_dims = (3, h, w)
481
+
482
+ def _validate_shape(d: torch.Tensor):
483
+ actual_dims = tuple(d.shape[1:])
484
+
485
+ if actual_dims != expected_dims:
486
+ expected_expr = ("num_patches", *map(str, expected_dims))
487
raise ValueError(
484
- "The expected pixel value tensor shape is batch dimension "
485
- "plus patch number, channel, height and width.")
488
+ "The expected shape of pixel values in each batch element "
489
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
490
487
- if isinstance(data, torch.Tensor):
488
- _validate_shape(data)
489
- else:
490
- [_validate_shape(d) for d in data]
491
+ for d in data:
492
+ _validate_shape(d)
493
494
return data
495