@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4ce74c14a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[VLM] Use shared field to pass token ids to model

Cyrus Leung committed Feb 6, 2025 at 05:30 UTC a4ce74c14a469b178c446a34ce48f158909a8e74
2 files changed +235 -46
vllm/model_executor/models/internvl.py
+3 -3
@@ -564,8 +564,7 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
564 # Since there may be extra tokens in the feature placeholders,
565 # we need to pass the image token ID to the model to select the
566 # tokens to merge from the vision encoder outputs
567 - processed_outputs["image_token_id"] = [image_token_id
568 - ] * len(image_data)
567 + processed_outputs["image_token_id"] = torch.tensor(image_token_id)
568
569 return processed_outputs
570
@@ -575,13 +574,14 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
574 hf_processor_mm_kwargs: Mapping[str, object],
575 ) -> Mapping[str, MultiModalFieldConfig]:
576 image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
577 + num_images = len(image_num_patches)
578
579 return dict(
580 pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
581 "image", image_num_patches),
582 image_num_patches=MultiModalFieldConfig.batched("image"),
583 image_embeds=MultiModalFieldConfig.batched("image"),
584 - image_token_id=MultiModalFieldConfig.batched("image"),
584 + image_token_id=MultiModalFieldConfig.shared("image", num_images),
585 )
586
587 def _get_prompt_replacements(
vllm/multimodal/inputs.py
+232 -43
@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
4 from collections import UserDict, defaultdict
5 from collections.abc import Mapping, Sequence
6 from dataclasses import dataclass
7 +from functools import partial
8 from itertools import accumulate
9 from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
10 Union, cast, final)
@@ -164,51 +165,112 @@ A dictionary containing nested tensors which have been batched via
165
166 @dataclass(frozen=True)
167 class MultiModalFieldElem:
167 - """Contains metadata and data of an item in :class:`MultiModalKwargs`."""
168 - field: "BaseMultiModalField"
168 + """
169 + Represents a keyword argument corresponding to a multi-modal item
170 + in :class:`MultiModalKwargs`.
171 + """
172 +
173 + modality: str
174 + """
175 + The modality of the corresponding multi-modal item.
176 + Each multi-modal item can consist of multiple keyword arguments.
177 + """
178 +
179 + key: str
180 + """
181 + The key of this field in :class:`MultiModalKwargs`,
182 + i.e. the name of the keyword argument to be passed to the model.
183 + """
184 +
185 data: NestedTensors
186 + """
187 + The tensor data of this field in :class:`MultiModalKwargs`,
188 + i.e. the value of the keyword argument to be passed to the model.
189 + """
190 +
191 + field: "BaseMultiModalField"
192 + """
193 + Defines how to combine the tensor data of this field with others
194 + in order to batch multi-modal items together for model inference.
195 + """
196
197 def __eq__(self, other: object) -> bool:
198 if not isinstance(other, self.__class__):
199 return False
200
175 - return (self.field == other.field
176 - and nested_tensors_equal(self.data, other.data))
201 + return ((self.modality, self.key) == (other.modality, other.key)
202 + and nested_tensors_equal(self.data, other.data)
203 + and type(self.field) == type(other.field)) # noqa: E721
204
205
206 @dataclass(frozen=True)
207 class BaseMultiModalField(ABC):
181 - """Abstract base class for a field in :class:`MultiModalKwargs`."""
182 - key: str
183 - modality: str
208 + """
209 + Defines how to interpret tensor data belonging to a keyword argument in
210 + :class:`MultiModalKwargs` for multiple multi-modal items, and vice versa.
211 + """
212 +
213 + def _field_factory(self, *, modality: str, key: str):
214 + f = partial(
215 + MultiModalFieldElem,
216 + modality=modality,
217 + key=key,
218 + field=self,
219 + )
220 +
221 + # Allow passing data as positional argument
222 + def factory(data: NestedTensors) -> MultiModalFieldElem:
223 + return f(data=data)
224 +
225 + return factory
226
227 @abstractmethod
186 - def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
228 + def build_elems(
229 + self,
230 + modality: str,
231 + key: str,
232 + data: NestedTensors,
233 + ) -> Sequence[MultiModalFieldElem]:
234 + """
235 + Construct :class:`MultiModalFieldElem` instances to represent
236 + the provided data.
237 +
238 + This is the inverse of :meth:`reduce_data`.
239 + """
240 raise NotImplementedError
241
189 - def _build_elem(self, data: NestedTensors) -> MultiModalFieldElem:
190 - return MultiModalFieldElem(self, data)
242 + @abstractmethod
243 + def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
244 + raise NotImplementedError
245
192 - def reduce(self, batch: list[MultiModalFieldElem]) -> MultiModalFieldElem:
193 - """Merge multiple instances of :class:`MultiModalFieldElem` together."""
194 - fields = [item.field for item in batch]
195 - if len(set(fields)) > 1:
196 - raise ValueError(f"Cannot merge different {fields=}")
246 + def reduce_data(self, elems: list[MultiModalFieldElem]) -> NestedTensors:
247 + """
248 + Merge the data from multiple instances of :class:`MultiModalFieldElem`.
249
198 - data = self._reduce_data([item.data for item in batch])
250 + This is the inverse of :meth:`build_elems`.
251 + """
252 + field_types = [type(item.field) for item in elems]
253 + if len(set(field_types)) > 1:
254 + raise ValueError(f"Cannot merge different {field_types=}")
255
200 - return self._build_elem(data)
256 + return self._reduce_data([item.data for item in elems])
257
258
259 @dataclass(frozen=True)
260 class MultiModalBatchedField(BaseMultiModalField):
261 """
206 - A :class:`BaseMultiModalField` implementation where an element in the batch
207 - is obtained by indexing into the first dimension of the underlying data.
262 + See also:
263 + :func:`MultiModalFieldConfig.batched`
264 """
265
210 - def build_elems(self, batch: NestedTensors) -> list[MultiModalFieldElem]:
211 - return [self._build_elem(item) for item in batch]
266 + def build_elems(
267 + self,
268 + modality: str,
269 + key: str,
270 + data: NestedTensors,
271 + ) -> Sequence[MultiModalFieldElem]:
272 + field_factory = self._field_factory(modality=modality, key=key)
273 + return [field_factory(item) for item in data]
274
275 def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
276 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
@@ -227,16 +289,20 @@ class MultiModalBatchedField(BaseMultiModalField):
289 @dataclass(frozen=True)
290 class MultiModalFlatField(BaseMultiModalField):
291 """
230 - A :class:`BaseMultiModalField` implementation where an element in the batch
231 - is obtained by slicing along the first dimension of the underlying data.
292 + See also:
293 + :func:`MultiModalFieldConfig.flat`
294 + :func:`MultiModalFieldConfig.flat_from_sizes`
295 """
296 + slices: Sequence[slice]
297
298 def build_elems(
299 self,
236 - batch: NestedTensors,
237 - slices: Sequence[slice],
238 - ) -> list[MultiModalFieldElem]:
239 - return [self._build_elem(batch[slice_]) for slice_ in slices]
300 + modality: str,
301 + key: str,
302 + data: NestedTensors,
303 + ) -> Sequence[MultiModalFieldElem]:
304 + field_factory = self._field_factory(modality=modality, key=key)
305 + return [field_factory(data[s]) for s in self.slices]
306
307 def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
308 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
@@ -252,25 +318,121 @@ class MultiModalFlatField(BaseMultiModalField):
318 return [e for elem in batch for e in elem]
319
320
321 +@dataclass(frozen=True)
322 +class MultiModalSharedField(BaseMultiModalField):
323 + """
324 + See also:
325 + :func:`MultiModalFieldConfig.shared`
326 + """
327 + batch_size: int
328 +
329 + def build_elems(
330 + self,
331 + modality: str,
332 + key: str,
333 + data: NestedTensors,
334 + ) -> Sequence[MultiModalFieldElem]:
335 + field_factory = self._field_factory(modality=modality, key=key)
336 + return [field_factory(data)] * self.batch_size
337 +
338 + def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
339 + return batch[0]
340 +
341 +
342 class MultiModalFieldConfig:
343
344 @staticmethod
345 def batched(modality: str):
346 + """
347 + Defines a field where an element in the batch is obtained by
348 + indexing into the first dimension of the underlying data.
349 +
350 + Args:
351 + modality: The modality of the multi-modal item that uses this
352 + keyword argument.
353 +
354 + Example:
355 +
356 + .. code-block::
357 +
358 + Input:
359 + Data: [[AAAA]
360 + [BBBB]
361 + [CCCC]]
362 +
363 + Output:
364 + Element 1: [AAAA]
365 + Element 2: [BBBB]
366 + Element 3: [CCCC]
367 + """
368 return MultiModalFieldConfig(
260 - field_cls=MultiModalBatchedField,
369 + field=MultiModalBatchedField(),
370 modality=modality,
371 )
372
373 @staticmethod
374 def flat(modality: str, slices: Sequence[slice]):
375 + """
376 + Defines a field where an element in the batch is obtained by
377 + slicing along the first dimension of the underlying data.
378 +
379 + Args:
380 + modality: The modality of the multi-modal item that uses this
381 + keyword argument.
382 + slices: For each multi-modal item, a slice that is used to extract
383 + the data corresponding to it.
384 +
385 + Example:
386 +
387 + .. code-block::
388 +
389 + Given:
390 + slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
391 +
392 + Input:
393 + Data: [AAABBBBCC]
394 +
395 + Output:
396 + Element 1: [AAA]
397 + Element 2: [BBBB]
398 + Element 3: [CC]
399 + """
400 return MultiModalFieldConfig(
267 - field_cls=MultiModalFlatField,
401 + field=MultiModalFlatField(slices=slices),
402 modality=modality,
269 - slices=slices,
403 )
404
405 @staticmethod
406 def flat_from_sizes(modality: str, size_per_item: torch.Tensor):
407 + """
408 + Defines a field where an element in the batch is obtained by
409 + slicing along the first dimension of the underlying data.
410 +
411 + Args:
412 + modality: The modality of the multi-modal item that uses this
413 + keyword argument.
414 + slices: For each multi-modal item, the size of the slice that
415 + is used to extract the data corresponding to it.
416 +
417 + Example:
418 +
419 + .. code-block::
420 +
421 + Given:
422 + size_per_item: [3, 4, 2]
423 +
424 + Input:
425 + Data: [AAABBBBCC]
426 +
427 + Output:
428 + Element 1: [AAA]
429 + Element 2: [BBBB]
430 + Element 3: [CC]
431 +
432 + See also:
433 + :func:`MultiModalFieldConfig.flat`
434 + """
435 +
436 slice_idxs = [0, *accumulate(size_per_item)]
437 slices = [
438 slice(slice_idxs[i], slice_idxs[i + 1])
@@ -279,25 +441,52 @@ class MultiModalFieldConfig:
441
442 return MultiModalFieldConfig.flat(modality, slices)
443
282 - def __init__(
283 - self,
284 - field_cls: type[BaseMultiModalField],
285 - modality: str,
286 - **field_config: Any,
287 - ) -> None:
444 + @staticmethod
445 + def shared(modality: str, batch_size: int):
446 + """
447 + Defines a field where an element in the batch is obtained by
448 + taking the entirety of the underlying data.
449 +
450 + This means that the data is the same for each element in the batch.
451 +
452 + Args:
453 + modality: The modality of the multi-modal item that uses this
454 + keyword argument.
455 + batch_size: The number of multi-modal items which share this data.
456 +
457 + Example:
458 +
459 + .. code-block::
460 +
461 + Given:
462 + batch_size: 4
463 +
464 + Input:
465 + Data: [XYZ]
466 +
467 + Output:
468 + Element 1: [XYZ]
469 + Element 2: [XYZ]
470 + Element 3: [XYZ]
471 + Element 4: [XYZ]
472 + """
473 + return MultiModalFieldConfig(
474 + field=MultiModalSharedField(batch_size),
475 + modality=modality,
476 + )
477 +
478 + def __init__(self, field: BaseMultiModalField, modality: str) -> None:
479 super().__init__()
480
290 - self.field_cls = field_cls
481 + self.field = field
482 self.modality = modality
292 - self.field_config = field_config
483
484 def build_elems(
485 self,
486 key: str,
487 batch: NestedTensors,
488 ) -> Sequence[MultiModalFieldElem]:
299 - field = self.field_cls(key=key, modality=self.modality)
300 - return field.build_elems(batch, **self.field_config) # type: ignore
489 + return self.field.build_elems(self.modality, key, batch)
490
491
492 class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
@@ -308,11 +497,11 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
497
498 @staticmethod
499 def from_elems(elems: Sequence[MultiModalFieldElem]):
311 - return MultiModalKwargsItem({elem.field.key: elem for elem in elems})
500 + return MultiModalKwargsItem({elem.key: elem for elem in elems})
501
502 @property
503 def modality(self) -> str:
315 - modalities = {elem.field.modality for elem in self.data.values()}
504 + modalities = {elem.modality for elem in self.data.values()}
505 assert len(modalities) == 1, f"Found different modalities={modalities}"
506 return next(iter(modalities))
507
@@ -372,7 +561,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
561 elems_by_key[key].append(elem)
562
563 data = {
375 - key: elems[0].field.reduce(elems).data
564 + key: elems[0].field.reduce_data(elems)
565 for key, elems in elems_by_key.items() if len(elems) > 0
566 }
567