A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[VLM] Use shared field to pass token ids to model
Cyrus Leung committed
Feb 6, 2025 at 05:30 UTC
a4ce74c14a469b178c446a34ce48f158909a8e74
2 files changed
+235
-46
vllm/model_executor/models/internvl.py
+3
-3
@@ -564,8 +564,7 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
564
# Since there may be extra tokens in the feature placeholders,
565
# we need to pass the image token ID to the model to select the
566
# tokens to merge from the vision encoder outputs
567
- processed_outputs["image_token_id"] = [image_token_id
568
- ] * len(image_data)
567
+ processed_outputs["image_token_id"] = torch.tensor(image_token_id)
568
569
return processed_outputs
570
@@ -575,13 +574,14 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
574
hf_processor_mm_kwargs: Mapping[str, object],
575
) -> Mapping[str, MultiModalFieldConfig]:
576
image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
577
+ num_images = len(image_num_patches)
578
579
return dict(
580
pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
581
"image", image_num_patches),
582
image_num_patches=MultiModalFieldConfig.batched("image"),
583
image_embeds=MultiModalFieldConfig.batched("image"),
584
- image_token_id=MultiModalFieldConfig.batched("image"),
584
+ image_token_id=MultiModalFieldConfig.shared("image", num_images),
585
)
586
587
def _get_prompt_replacements(
vllm/multimodal/inputs.py
+232
-43
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
4
from collections import UserDict, defaultdict
5
from collections.abc import Mapping, Sequence
6
from dataclasses import dataclass
7
+from functools import partial
8
from itertools import accumulate
9
from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
10
Union, cast, final)
@@ -164,51 +165,112 @@ A dictionary containing nested tensors which have been batched via
165
166
@dataclass(frozen=True)
167
class MultiModalFieldElem:
167
- """Contains metadata and data of an item in :class:`MultiModalKwargs`."""
168
- field: "BaseMultiModalField"
168
+ """
169
+ Represents a keyword argument corresponding to a multi-modal item
170
+ in :class:`MultiModalKwargs`.
171
+ """
172
+
173
+ modality: str
174
+ """
175
+ The modality of the corresponding multi-modal item.
176
+ Each multi-modal item can consist of multiple keyword arguments.
177
+ """
178
+
179
+ key: str
180
+ """
181
+ The key of this field in :class:`MultiModalKwargs`,
182
+ i.e. the name of the keyword argument to be passed to the model.
183
+ """
184
+
185
data: NestedTensors
186
+ """
187
+ The tensor data of this field in :class:`MultiModalKwargs`,
188
+ i.e. the value of the keyword argument to be passed to the model.
189
+ """
190
+
191
+ field: "BaseMultiModalField"
192
+ """
193
+ Defines how to combine the tensor data of this field with others
194
+ in order to batch multi-modal items together for model inference.
195
+ """
196
197
def __eq__(self, other: object) -> bool:
198
if not isinstance(other, self.__class__):
199
return False
200
175
- return (self.field == other.field
176
- and nested_tensors_equal(self.data, other.data))
201
+ return ((self.modality, self.key) == (other.modality, other.key)
202
+ and nested_tensors_equal(self.data, other.data)
203
+ and type(self.field) == type(other.field)) # noqa: E721
204
205
206
@dataclass(frozen=True)
207
class BaseMultiModalField(ABC):
181
- """Abstract base class for a field in :class:`MultiModalKwargs`."""
182
- key: str
183
- modality: str
208
+ """
209
+ Defines how to interpret tensor data belonging to a keyword argument in
210
+ :class:`MultiModalKwargs` for multiple multi-modal items, and vice versa.
211
+ """
212
+
213
+ def _field_factory(self, *, modality: str, key: str):
214
+ f = partial(
215
+ MultiModalFieldElem,
216
+ modality=modality,
217
+ key=key,
218
+ field=self,
219
+ )
220
+
221
+ # Allow passing data as positional argument
222
+ def factory(data: NestedTensors) -> MultiModalFieldElem:
223
+ return f(data=data)
224
+
225
+ return factory
226
227
@abstractmethod
186
- def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
228
+ def build_elems(
229
+ self,
230
+ modality: str,
231
+ key: str,
232
+ data: NestedTensors,
233
+ ) -> Sequence[MultiModalFieldElem]:
234
+ """
235
+ Construct :class:`MultiModalFieldElem` instances to represent
236
+ the provided data.
237
+
238
+ This is the inverse of :meth:`reduce_data`.
239
+ """
240
raise NotImplementedError
241
189
- def _build_elem(self, data: NestedTensors) -> MultiModalFieldElem:
190
- return MultiModalFieldElem(self, data)
242
+ @abstractmethod
243
+ def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
244
+ raise NotImplementedError
245
192
- def reduce(self, batch: list[MultiModalFieldElem]) -> MultiModalFieldElem:
193
- """Merge multiple instances of :class:`MultiModalFieldElem` together."""
194
- fields = [item.field for item in batch]
195
- if len(set(fields)) > 1:
196
- raise ValueError(f"Cannot merge different {fields=}")
246
+ def reduce_data(self, elems: list[MultiModalFieldElem]) -> NestedTensors:
247
+ """
248
+ Merge the data from multiple instances of :class:`MultiModalFieldElem`.
249
198
- data = self._reduce_data([item.data for item in batch])
250
+ This is the inverse of :meth:`build_elems`.
251
+ """
252
+ field_types = [type(item.field) for item in elems]
253
+ if len(set(field_types)) > 1:
254
+ raise ValueError(f"Cannot merge different {field_types=}")
255
200
- return self._build_elem(data)
256
+ return self._reduce_data([item.data for item in elems])
257
258
259
@dataclass(frozen=True)
260
class MultiModalBatchedField(BaseMultiModalField):
261
"""
206
- A :class:`BaseMultiModalField` implementation where an element in the batch
207
- is obtained by indexing into the first dimension of the underlying data.
262
+ See also:
263
+ :func:`MultiModalFieldConfig.batched`
264
"""
265
210
- def build_elems(self, batch: NestedTensors) -> list[MultiModalFieldElem]:
211
- return [self._build_elem(item) for item in batch]
266
+ def build_elems(
267
+ self,
268
+ modality: str,
269
+ key: str,
270
+ data: NestedTensors,
271
+ ) -> Sequence[MultiModalFieldElem]:
272
+ field_factory = self._field_factory(modality=modality, key=key)
273
+ return [field_factory(item) for item in data]
274
275
def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
276
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
@@ -227,16 +289,20 @@ class MultiModalBatchedField(BaseMultiModalField):
289
@dataclass(frozen=True)
290
class MultiModalFlatField(BaseMultiModalField):
291
"""
230
- A :class:`BaseMultiModalField` implementation where an element in the batch
231
- is obtained by slicing along the first dimension of the underlying data.
292
+ See also:
293
+ :func:`MultiModalFieldConfig.flat`
294
+ :func:`MultiModalFieldConfig.flat_from_sizes`
295
"""
296
+ slices: Sequence[slice]
297
298
def build_elems(
299
self,
236
- batch: NestedTensors,
237
- slices: Sequence[slice],
238
- ) -> list[MultiModalFieldElem]:
239
- return [self._build_elem(batch[slice_]) for slice_ in slices]
300
+ modality: str,
301
+ key: str,
302
+ data: NestedTensors,
303
+ ) -> Sequence[MultiModalFieldElem]:
304
+ field_factory = self._field_factory(modality=modality, key=key)
305
+ return [field_factory(data[s]) for s in self.slices]
306
307
def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
308
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
@@ -252,25 +318,121 @@ class MultiModalFlatField(BaseMultiModalField):
318
return [e for elem in batch for e in elem]
319
320
321
+@dataclass(frozen=True)
322
+class MultiModalSharedField(BaseMultiModalField):
323
+ """
324
+ See also:
325
+ :func:`MultiModalFieldConfig.shared`
326
+ """
327
+ batch_size: int
328
+
329
+ def build_elems(
330
+ self,
331
+ modality: str,
332
+ key: str,
333
+ data: NestedTensors,
334
+ ) -> Sequence[MultiModalFieldElem]:
335
+ field_factory = self._field_factory(modality=modality, key=key)
336
+ return [field_factory(data)] * self.batch_size
337
+
338
+ def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
339
+ return batch[0]
340
+
341
+
342
class MultiModalFieldConfig:
343
344
@staticmethod
345
def batched(modality: str):
346
+ """
347
+ Defines a field where an element in the batch is obtained by
348
+ indexing into the first dimension of the underlying data.
349
+
350
+ Args:
351
+ modality: The modality of the multi-modal item that uses this
352
+ keyword argument.
353
+
354
+ Example:
355
+
356
+ .. code-block::
357
+
358
+ Input:
359
+ Data: [[AAAA]
360
+ [BBBB]
361
+ [CCCC]]
362
+
363
+ Output:
364
+ Element 1: [AAAA]
365
+ Element 2: [BBBB]
366
+ Element 3: [CCCC]
367
+ """
368
return MultiModalFieldConfig(
260
- field_cls=MultiModalBatchedField,
369
+ field=MultiModalBatchedField(),
370
modality=modality,
371
)
372
373
@staticmethod
374
def flat(modality: str, slices: Sequence[slice]):
375
+ """
376
+ Defines a field where an element in the batch is obtained by
377
+ slicing along the first dimension of the underlying data.
378
+
379
+ Args:
380
+ modality: The modality of the multi-modal item that uses this
381
+ keyword argument.
382
+ slices: For each multi-modal item, a slice that is used to extract
383
+ the data corresponding to it.
384
+
385
+ Example:
386
+
387
+ .. code-block::
388
+
389
+ Given:
390
+ slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
391
+
392
+ Input:
393
+ Data: [AAABBBBCC]
394
+
395
+ Output:
396
+ Element 1: [AAA]
397
+ Element 2: [BBBB]
398
+ Element 3: [CC]
399
+ """
400
return MultiModalFieldConfig(
267
- field_cls=MultiModalFlatField,
401
+ field=MultiModalFlatField(slices=slices),
402
modality=modality,
269
- slices=slices,
403
)
404
405
@staticmethod
406
def flat_from_sizes(modality: str, size_per_item: torch.Tensor):
407
+ """
408
+ Defines a field where an element in the batch is obtained by
409
+ slicing along the first dimension of the underlying data.
410
+
411
+ Args:
412
+ modality: The modality of the multi-modal item that uses this
413
+ keyword argument.
414
+ slices: For each multi-modal item, the size of the slice that
415
+ is used to extract the data corresponding to it.
416
+
417
+ Example:
418
+
419
+ .. code-block::
420
+
421
+ Given:
422
+ size_per_item: [3, 4, 2]
423
+
424
+ Input:
425
+ Data: [AAABBBBCC]
426
+
427
+ Output:
428
+ Element 1: [AAA]
429
+ Element 2: [BBBB]
430
+ Element 3: [CC]
431
+
432
+ See also:
433
+ :func:`MultiModalFieldConfig.flat`
434
+ """
435
+
436
slice_idxs = [0, *accumulate(size_per_item)]
437
slices = [
438
slice(slice_idxs[i], slice_idxs[i + 1])
@@ -279,25 +441,52 @@ class MultiModalFieldConfig:
441
442
return MultiModalFieldConfig.flat(modality, slices)
443
282
- def __init__(
283
- self,
284
- field_cls: type[BaseMultiModalField],
285
- modality: str,
286
- **field_config: Any,
287
- ) -> None:
444
+ @staticmethod
445
+ def shared(modality: str, batch_size: int):
446
+ """
447
+ Defines a field where an element in the batch is obtained by
448
+ taking the entirety of the underlying data.
449
+
450
+ This means that the data is the same for each element in the batch.
451
+
452
+ Args:
453
+ modality: The modality of the multi-modal item that uses this
454
+ keyword argument.
455
+ batch_size: The number of multi-modal items which share this data.
456
+
457
+ Example:
458
+
459
+ .. code-block::
460
+
461
+ Given:
462
+ batch_size: 4
463
+
464
+ Input:
465
+ Data: [XYZ]
466
+
467
+ Output:
468
+ Element 1: [XYZ]
469
+ Element 2: [XYZ]
470
+ Element 3: [XYZ]
471
+ Element 4: [XYZ]
472
+ """
473
+ return MultiModalFieldConfig(
474
+ field=MultiModalSharedField(batch_size),
475
+ modality=modality,
476
+ )
477
+
478
+ def __init__(self, field: BaseMultiModalField, modality: str) -> None:
479
super().__init__()
480
290
- self.field_cls = field_cls
481
+ self.field = field
482
self.modality = modality
292
- self.field_config = field_config
483
484
def build_elems(
485
self,
486
key: str,
487
batch: NestedTensors,
488
) -> Sequence[MultiModalFieldElem]:
299
- field = self.field_cls(key=key, modality=self.modality)
300
- return field.build_elems(batch, **self.field_config) # type: ignore
489
+ return self.field.build_elems(self.modality, key, batch)
490
491
492
class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
@@ -308,11 +497,11 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
497
498
@staticmethod
499
def from_elems(elems: Sequence[MultiModalFieldElem]):
311
- return MultiModalKwargsItem({elem.field.key: elem for elem in elems})
500
+ return MultiModalKwargsItem({elem.key: elem for elem in elems})
501
502
@property
503
def modality(self) -> str:
315
- modalities = {elem.field.modality for elem in self.data.values()}
504
+ modalities = {elem.modality for elem in self.data.values()}
505
assert len(modalities) == 1, f"Found different modalities={modalities}"
506
return next(iter(modalities))
507
@@ -372,7 +561,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
561
elems_by_key[key].append(elem)
562
563
data = {
375
- key: elems[0].field.reduce(elems).data
564
+ key: elems[0].field.reduce_data(elems)
565
for key, elems in elems_by_key.items() if len(elems) > 0
566
}
567