A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Optim] Compute multimodal hash only once per item (#17314)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 29, 2025 at 09:40 UTC
506475de5f1a48c225e7cd7f87efd40ad6f3c9c0
6 files changed
+233
-128
vllm/model_executor/models/deepseek_vl2.py
+9
-7
@@ -22,8 +22,8 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
22
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
23
ImageSize, MultiModalDataItems)
24
from vllm.multimodal.processing import (BaseMultiModalProcessor,
25
- BaseProcessingInfo, PromptReplacement,
26
- PromptUpdate)
25
+ BaseProcessingInfo, MultiModalHashes,
26
+ PromptReplacement, PromptUpdate)
27
from vllm.multimodal.profiling import BaseDummyInputsBuilder
28
from vllm.sequence import IntermediateTensors
29
from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
@@ -279,24 +279,26 @@ class DeepseekVL2MultiModalProcessor(
279
prompt: Union[str, list[int]],
280
mm_data_items: MultiModalDataItems,
281
hf_processor_mm_kwargs: Mapping[str, object],
282
- ) -> tuple[list[int], MultiModalKwargs, bool]:
282
+ *,
283
+ return_mm_hashes: bool,
284
+ ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
285
# The processor logic is different for len(images) <= 2 vs > 2
286
# Since the processing cache assumes that the processor output is
287
# invariant of how many images are passed per prompt, we only
288
# perform caching for the most common case
289
if mm_data_items.get_count("image", strict=False) > 2:
288
- # This code path corresponds to the cache being disabled
289
- return self._apply_hf_processor_main(
290
+ return self._apply_hf_processor(
291
prompt=prompt,
291
- mm_items=mm_data_items,
292
+ mm_data_items=mm_data_items,
293
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
293
- enable_hf_prompt_update=True,
294
+ return_mm_hashes=return_mm_hashes,
295
)
296
297
return super()._cached_apply_hf_processor(
298
prompt=prompt,
299
mm_data_items=mm_data_items,
300
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
301
+ return_mm_hashes=return_mm_hashes,
302
)
303
304
vllm/model_executor/models/h2ovl.py
+9
-7
@@ -19,8 +19,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.multimodal.inputs import MultiModalKwargs
20
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
21
MultiModalDataItems)
22
-from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
23
- PromptUpdateDetails)
22
+from vllm.multimodal.processing import (MultiModalHashes, PromptReplacement,
23
+ PromptUpdate, PromptUpdateDetails)
24
from vllm.transformers_utils.tokenizer import AnyTokenizer
25
26
from .intern_vit import InternVisionModel
@@ -488,24 +488,26 @@ class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
488
prompt: Union[str, list[int]],
489
mm_data_items: MultiModalDataItems,
490
hf_processor_mm_kwargs: Mapping[str, object],
491
- ) -> tuple[list[int], MultiModalKwargs, bool]:
491
+ *,
492
+ return_mm_hashes: bool,
493
+ ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
494
# The processor logic is different for len(images) <= 1 vs > 1
495
# Since the processing cache assumes that the processor output is
496
# invariant of how many images are passed per prompt, we only
497
# perform caching for the most common case
498
if mm_data_items.get_count("image", strict=False) > 1:
497
- # This code path corresponds to the cache being disabled
498
- return self._apply_hf_processor_main(
499
+ return self._apply_hf_processor(
500
prompt=prompt,
500
- mm_items=mm_data_items,
501
+ mm_data_items=mm_data_items,
502
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
502
- enable_hf_prompt_update=True,
503
+ return_mm_hashes=return_mm_hashes,
504
)
505
506
return super()._cached_apply_hf_processor(
507
prompt=prompt,
508
mm_data_items=mm_data_items,
509
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
510
+ return_mm_hashes=return_mm_hashes,
511
)
512
513
vllm/model_executor/models/llava.py
-3
@@ -396,14 +396,12 @@ def _build_llava_or_pixtral_hf_processor(
396
dummy_inputs: BaseDummyInputsBuilder[_I],
397
*,
398
cache: Optional[ProcessingCache] = None,
399
- enable_sanity_checks: bool = True,
399
) -> BaseMultiModalProcessor:
400
if isinstance(info, PixtralHFProcessingInfo):
401
return PixtralHFMultiModalProcessor(
402
info,
403
dummy_inputs, # type: ignore
404
cache=cache,
406
- enable_sanity_checks=enable_sanity_checks,
405
)
406
407
if isinstance(info, LlavaProcessingInfo):
@@ -411,7 +409,6 @@ def _build_llava_or_pixtral_hf_processor(
409
info,
410
dummy_inputs, # type: ignore
411
cache=cache,
414
- enable_sanity_checks=enable_sanity_checks,
412
)
413
414
raise NotImplementedError(type(info))
vllm/model_executor/models/mistral3.py
-2
@@ -312,14 +312,12 @@ def _build_mistral3_processor(
312
dummy_inputs: BaseDummyInputsBuilder[_I],
313
*,
314
cache: Optional[ProcessingCache] = None,
315
- enable_sanity_checks: bool = True,
315
) -> BaseMultiModalProcessor:
316
assert isinstance(info, Mistral3ProcessingInfo)
317
return Mistral3MultiModalProcessor(
318
info,
319
dummy_inputs, # type: ignore
320
cache=cache,
322
- enable_sanity_checks=enable_sanity_checks,
321
)
322
323
vllm/model_executor/models/pixtral.py
+10
-5
@@ -36,8 +36,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
36
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37
MultiModalDataItems)
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
- BaseProcessingInfo, PromptReplacement,
40
- PromptUpdate, PromptUpdateDetails)
39
+ BaseProcessingInfo, MultiModalHashes,
40
+ PromptReplacement, PromptUpdate,
41
+ PromptUpdateDetails)
42
from vllm.multimodal.profiling import BaseDummyInputsBuilder
43
from vllm.sequence import IntermediateTensors
44
from vllm.transformers_utils.tokenizer import (MistralTokenizer,
@@ -271,15 +272,19 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
272
prompt: Union[str, list[int]],
273
mm_data_items: MultiModalDataItems,
274
hf_processor_mm_kwargs: Mapping[str, object],
274
- ) -> tuple[list[int], MultiModalKwargs, bool]:
275
- prompt_ids, mm_kwargs, _ = super()._cached_apply_hf_processor(
275
+ *,
276
+ return_mm_hashes: bool,
277
+ ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
278
+ prompt_ids, mm_kwargs, mm_hashes, _ = super(
279
+ )._cached_apply_hf_processor(
280
prompt=prompt,
281
mm_data_items=mm_data_items,
282
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
283
+ return_mm_hashes=return_mm_hashes,
284
)
285
286
# NOTE: The tokens are already inserted by the chat template
282
- return prompt_ids, mm_kwargs, True
287
+ return prompt_ids, mm_kwargs, mm_hashes, True
288
289
290
@MULTIMODAL_REGISTRY.register_processor(PixtralMultiModalProcessor,
vllm/multimodal/processing.py
+205
-104
@@ -876,6 +876,16 @@ def find_mm_placeholders(
876
_V = TypeVar("_V", bound="Union[MultiModalKwargs, MultiModalKwargsItem]")
877
878
879
+class ProcessingCacheOptionalItem(NamedTuple):
880
+ key: str
881
+ value: Optional[MultiModalKwargsItem]
882
+
883
+
884
+class ProcessingCacheItem(NamedTuple):
885
+ key: str
886
+ value: MultiModalKwargsItem
887
+
888
+
889
class ProcessingCache:
890
891
@staticmethod
@@ -980,6 +990,22 @@ class ProcessingCache:
990
991
return self._cache.get(cache_key)
992
993
+ def get_item(
994
+ self,
995
+ model_id: str,
996
+ modality: str,
997
+ input_item: object,
998
+ input_kwargs: Mapping[str, object],
999
+ ) -> ProcessingCacheOptionalItem:
1000
+ cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
1001
+ **{modality: input_item},
1002
+ **input_kwargs)
1003
+
1004
+ return ProcessingCacheOptionalItem(
1005
+ key=cache_key,
1006
+ value=self._cache.get(cache_key),
1007
+ )
1008
+
1009
def put(
1010
self,
1011
model_id: str,
@@ -997,6 +1023,9 @@ class ProcessingCache:
1023
**input_kwargs)
1024
self._cache[cache_key] = output_kwargs
1025
1026
+ def put_item(self, item: ProcessingCacheItem) -> None:
1027
+ self._cache[item.key] = item.value
1028
+
1029
1030
class BaseProcessingInfo:
1031
"""Base class to provide the information necessary for data processing."""
@@ -1052,6 +1081,11 @@ class BaseProcessingInfo:
1081
1082
_I = TypeVar("_I", bound=BaseProcessingInfo)
1083
1084
+MultiModalHashes = dict[str, list[str]]
1085
+"""
1086
+A collection of hashes with a similar structure as :class:`MultiModalKwargs`.
1087
+"""
1088
+
1089
1090
class BaseMultiModalProcessor(ABC, Generic[_I]):
1091
"""
@@ -1064,14 +1098,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1098
info: _I,
1099
dummy_inputs: "BaseDummyInputsBuilder[_I]",
1100
*,
1067
- cache: Optional[ProcessingCache] = None,
1068
- enable_sanity_checks: bool = True) -> None:
1101
+ cache: Optional[ProcessingCache] = None) -> None:
1102
super().__init__()
1103
1104
self.info = info
1105
self.dummy_inputs = dummy_inputs
1106
self.cache = cache
1074
- self.enable_sanity_checks = enable_sanity_checks
1107
1108
self.data_parser = self._get_data_parser()
1109
@@ -1340,46 +1372,144 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1372
1373
return prompt_ids, mm_kwargs, False
1374
1375
+ def _get_cache_missing_items(
1376
+ self,
1377
+ cache: ProcessingCache,
1378
+ mm_data_items: MultiModalDataItems,
1379
+ hf_processor_mm_kwargs: Mapping[str, object],
1380
+ ) -> tuple[dict[str, list[ProcessingCacheOptionalItem]], dict[
1381
+ str, list[object]]]:
1382
+ model_id = self.info.model_id
1383
+
1384
+ mm_cache_items = {
1385
+ modality: [
1386
+ cache.get_item(model_id, modality, item,
1387
+ hf_processor_mm_kwargs) for item in items
1388
+ ]
1389
+ for modality, items in mm_data_items.items()
1390
+ }
1391
+
1392
+ mm_missing_idxs = {
1393
+ modality: [
1394
+ idx for idx, item in enumerate(cache_items)
1395
+ if item.value is None
1396
+ ]
1397
+ for modality, cache_items in mm_cache_items.items()
1398
+ }
1399
+ mm_missing_data = {
1400
+ modality: [mm_data_items[modality][idx] for idx in idxs]
1401
+ for modality, idxs in mm_missing_idxs.items()
1402
+ }
1403
+
1404
+ return mm_cache_items, mm_missing_data
1405
+
1406
+ def _hash_mm_items(
1407
+ self,
1408
+ mm_items: MultiModalDataItems,
1409
+ hf_processor_mm_kwargs: Mapping[str, object],
1410
+ ) -> MultiModalHashes:
1411
+ """Create MM hashes to be returned (only used in V1)."""
1412
+ model_id = self.info.model_id
1413
+
1414
+ return {
1415
+ modality: [
1416
+ MultiModalHasher.hash_kwargs(model_id=model_id,
1417
+ **{modality: item},
1418
+ **hf_processor_mm_kwargs)
1419
+ for item in items
1420
+ ]
1421
+ for modality, items in mm_items.items()
1422
+ }
1423
+
1424
+ def _merge_mm_kwargs(
1425
+ self,
1426
+ cache: ProcessingCache,
1427
+ mm_cache_items: dict[str, list[ProcessingCacheOptionalItem]],
1428
+ mm_missing_data: dict[str, list[object]],
1429
+ mm_missing_kwargs: MultiModalKwargs,
1430
+ ) -> dict[str, list[ProcessingCacheItem]]:
1431
+ mm_missing_next_idx = {modality: 0 for modality in mm_missing_data}
1432
+
1433
+ merged_items = defaultdict[str, list[ProcessingCacheItem]](list)
1434
+ for modality, cache_items in mm_cache_items.items():
1435
+ for cache_item in cache_items:
1436
+ if cache_item.value is None:
1437
+ kw_item = mm_missing_kwargs.get_item(
1438
+ modality,
1439
+ mm_missing_next_idx[modality],
1440
+ )
1441
+ cache_item_new = ProcessingCacheItem(
1442
+ key=cache_item.key,
1443
+ value=kw_item,
1444
+ )
1445
+
1446
+ cache.put_item(cache_item_new)
1447
+ mm_missing_next_idx[modality] += 1
1448
+ else:
1449
+ cache_item_new = ProcessingCacheItem(
1450
+ key=cache_item.key,
1451
+ value=cache_item.value,
1452
+ )
1453
+
1454
+ merged_items[modality].append(cache_item_new)
1455
+
1456
+ return dict(merged_items)
1457
+
1458
+ def _apply_hf_processor(
1459
+ self,
1460
+ prompt: Union[str, list[int]],
1461
+ mm_data_items: MultiModalDataItems,
1462
+ hf_processor_mm_kwargs: Mapping[str, object],
1463
+ *,
1464
+ return_mm_hashes: bool,
1465
+ ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
1466
+ (
1467
+ prompt_ids,
1468
+ mm_kwargs,
1469
+ is_update_applied,
1470
+ ) = self._apply_hf_processor_main(
1471
+ prompt=prompt,
1472
+ mm_items=mm_data_items,
1473
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1474
+ enable_hf_prompt_update=True,
1475
+ )
1476
+
1477
+ mm_hashes = (self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs)
1478
+ if return_mm_hashes else None)
1479
+
1480
+ return prompt_ids, mm_kwargs, mm_hashes, is_update_applied
1481
+
1482
def _cached_apply_hf_processor(
1483
self,
1484
prompt: Union[str, list[int]],
1485
mm_data_items: MultiModalDataItems,
1486
hf_processor_mm_kwargs: Mapping[str, object],
1348
- ) -> tuple[list[int], MultiModalKwargs, bool]:
1487
+ *,
1488
+ return_mm_hashes: bool,
1489
+ ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:
1490
"""
1491
Apply the HF processor on the full prompt text,
1492
caching the results and reusing cached results.
1493
"""
1494
cache = self.cache
1354
- model_id = self.info.model_id
1495
1496
_, passthrough_data = self._get_hf_mm_data(mm_data_items)
1497
if cache is None or passthrough_data:
1358
- return self._apply_hf_processor_main(
1498
+ return self._apply_hf_processor(
1499
prompt=prompt,
1360
- mm_items=mm_data_items,
1500
+ mm_data_items=mm_data_items,
1501
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1362
- enable_hf_prompt_update=True,
1502
+ return_mm_hashes=return_mm_hashes,
1503
)
1504
1365
- mm_maybe_cached_kw_items = {
1366
- modality: [
1367
- cache.get(model_id, modality, item, hf_processor_mm_kwargs)
1368
- for item in items
1369
- ]
1370
- for modality, items in mm_data_items.items()
1371
- }
1372
-
1373
- mm_missing_idxs = {
1374
- modality:
1375
- [idx for idx, item in enumerate(kw_items) if item is None]
1376
- for modality, kw_items in mm_maybe_cached_kw_items.items()
1377
- }
1378
- mm_missing_data = {
1379
- modality: [mm_data_items[modality][idx] for idx in idxs]
1380
- for modality, idxs in mm_missing_idxs.items()
1381
- }
1382
- mm_missing_data_items = self._to_mm_items(mm_missing_data)
1505
+ (
1506
+ mm_cache_items,
1507
+ mm_missing_data,
1508
+ ) = self._get_cache_missing_items(
1509
+ cache=cache,
1510
+ mm_data_items=mm_data_items,
1511
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1512
+ )
1513
1514
# NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1515
# so we can't apply prompt updates until the new multimodal
@@ -1390,48 +1520,29 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1520
is_update_applied,
1521
) = self._apply_hf_processor_main(
1522
prompt=prompt,
1393
- mm_items=mm_missing_data_items,
1523
+ mm_items=self._to_mm_items(mm_missing_data),
1524
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1525
enable_hf_prompt_update=False,
1526
)
1527
1398
- mm_missing_next_idx = {
1399
- modality: 0
1400
- for modality in mm_missing_data_items
1401
- }
1402
-
1403
- merged_kw_items = list[MultiModalKwargsItem]()
1404
- for modality, kw_items in mm_maybe_cached_kw_items.items():
1405
- for idx, kw_item in enumerate(kw_items):
1406
- if kw_item is None:
1407
- kw_item = mm_missing_kwargs.get_item(
1408
- modality,
1409
- mm_missing_next_idx[modality],
1410
- )
1411
-
1412
- cache.put(
1413
- model_id,
1414
- modality,
1415
- mm_data_items[modality][idx],
1416
- hf_processor_mm_kwargs,
1417
- kw_item,
1418
- )
1419
-
1420
- mm_missing_next_idx[modality] += 1
1421
-
1422
- merged_kw_items.append(kw_item)
1528
+ mm_cache_items_merged = self._merge_mm_kwargs(
1529
+ cache,
1530
+ mm_cache_items=mm_cache_items,
1531
+ mm_missing_data=mm_missing_data,
1532
+ mm_missing_kwargs=mm_missing_kwargs,
1533
+ )
1534
1424
- if self.enable_sanity_checks:
1425
- mm_missing_counts = mm_missing_data_items.get_all_counts()
1426
- assert all(
1427
- item_count == mm_missing_counts[modality]
1428
- for modality, item_count in mm_missing_next_idx.items()), dict(
1429
- mm_missing_next_idx=mm_missing_next_idx,
1430
- mm_missing_counts=mm_missing_counts)
1535
+ mm_kwargs = MultiModalKwargs.from_items([
1536
+ item.value for cache_items in mm_cache_items_merged.values()
1537
+ for item in cache_items
1538
+ ])
1539
1432
- mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
1540
+ mm_hashes = {
1541
+ modality: [item.key for item in cache_items]
1542
+ for modality, cache_items in mm_cache_items_merged.items()
1543
+ } if return_mm_hashes else None
1544
1434
- return prompt_ids, mm_kwargs, is_update_applied
1545
+ return prompt_ids, mm_kwargs, mm_hashes, is_update_applied
1546
1547
def _bind_and_group_updates(
1548
self,
@@ -1569,27 +1680,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1680
"model (usually arising from an inconsistency between "
1681
"`_call_hf_processor` and `_get_prompt_updates`).")
1682
1572
- def _hash_mm_items(
1573
- self,
1574
- mm_items: MultiModalDataItems,
1575
- hf_processor_mm_kwargs: Mapping[str, object],
1576
- ) -> dict[str, list[str]]:
1577
- """Create MM hashes to be returned (only used in V1)."""
1578
-
1579
- # TODO: Use these hash keys for caching operations in apply_hf_processor
1580
- # instead of rehashing.
1581
- model_id = self.info.model_id
1582
-
1583
- return {
1584
- modality: [
1585
- MultiModalHasher.hash_kwargs(model_id=model_id,
1586
- **{modality: item},
1587
- **hf_processor_mm_kwargs)
1588
- for item in items
1589
- ]
1590
- for modality, items in mm_items.items()
1591
- }
1592
-
1683
def _maybe_apply_prompt_updates(
1684
self,
1685
mm_items: MultiModalDataItems,
@@ -1655,17 +1745,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1745
"""
1746
mm_items = self._to_mm_items(mm_data)
1747
1658
- mm_hashes = (self._hash_mm_items(mm_items, hf_processor_mm_kwargs)
1659
- if return_mm_hashes else None)
1660
-
1748
(
1749
prompt_ids,
1750
mm_kwargs,
1751
+ mm_hashes,
1752
is_update_applied,
1753
) = self._cached_apply_hf_processor(
1754
prompt,
1755
mm_items,
1756
hf_processor_mm_kwargs,
1757
+ return_mm_hashes=return_mm_hashes,
1758
)
1759
1760
prompt_ids, prompt, mm_placeholders = self._maybe_apply_prompt_updates(
@@ -1717,28 +1806,12 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1806
"""Create input prompt for the decoder."""
1807
return prompt
1808
1720
- def apply(
1809
+ def _get_enc_dec_inputs(
1810
self,
1811
prompt: Union[str, list[int]],
1812
mm_data: MultiModalDataDict,
1724
- hf_processor_mm_kwargs: Mapping[str, object],
1725
- return_mm_hashes: bool = False,
1726
- ) -> MultiModalEncDecInputs:
1727
- """
1728
- Process multi-modal inputs to be used in vLLM.
1729
- The main processing steps are modified to fit encoder-decoder model:
1730
- 1. Create encoder prompt from input prompt text.
1731
- 2. Apply the HF processor on encoder prompt.
1732
- 3. Copy the input prompt text as decoder prompt inputs.
1733
- """
1734
- encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
1735
- encoder_inputs = super().apply(
1736
- encoder_prompt,
1737
- mm_data,
1738
- hf_processor_mm_kwargs,
1739
- return_mm_hashes,
1740
- )
1741
-
1813
+ encoder_inputs: MultiModalInputs,
1814
+ ):
1815
tokenizer = self.info.get_tokenizer()
1816
decoder_prompt = self.create_decoder_prompt(prompt, mm_data)
1817
if isinstance(decoder_prompt, str):
@@ -1758,3 +1831,31 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
1831
"prompt_token_ids": decoder_prompt_ids
1832
})
1833
return mm_inputs
1834
+
1835
+ def apply(
1836
+ self,
1837
+ prompt: Union[str, list[int]],
1838
+ mm_data: MultiModalDataDict,
1839
+ hf_processor_mm_kwargs: Mapping[str, object],
1840
+ return_mm_hashes: bool = False,
1841
+ ) -> MultiModalEncDecInputs:
1842
+ """
1843
+ Process multi-modal inputs to be used in vLLM.
1844
+ The main processing steps are modified to fit encoder-decoder model:
1845
+ 1. Create encoder prompt from input prompt text.
1846
+ 2. Apply the HF processor on encoder prompt.
1847
+ 3. Copy the input prompt text as decoder prompt inputs.
1848
+ """
1849
+ encoder_prompt = self.create_encoder_prompt(prompt, mm_data)
1850
+ encoder_inputs = super().apply(
1851
+ encoder_prompt,
1852
+ mm_data,
1853
+ hf_processor_mm_kwargs,
1854
+ return_mm_hashes,
1855
+ )
1856
+
1857
+ return self._get_enc_dec_inputs(
1858
+ prompt=prompt,
1859
+ mm_data=mm_data,
1860
+ encoder_inputs=encoder_inputs,
1861
+ )