A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Clean up multi-modal processor (#11207)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 15, 2024 at 14:30 UTC
b10609e6a11554be61976981304984510a0469c9
3 files changed
+32
-38
examples/offline_inference_vision_language.py
+1
-4
@@ -92,10 +92,7 @@ def run_fuyu(question: str, modality: str):
92
def run_phi3v(question: str, modality: str):
93
assert modality == "image"
94
95
- prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501
96
- # Note: The default setting of max_num_seqs (256) and
97
- # max_model_len (128k) for this model may cause OOM.
98
- # You may lower either to run this example on lower-end GPUs.
95
+ prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
96
97
# num_crops is an override kwarg to the multimodal image processor;
98
# For some models, e.g., Phi-3.5-vision-instruct, it is recommended
tests/multimodal/test_processing.py
+8
-9
@@ -2,10 +2,9 @@ from typing import cast
2
3
import pytest
4
5
-from vllm.multimodal.processing import (MultiModalDataItems, PromptReplacement,
6
- _PlaceholderInfo, find_text_matches,
7
- find_token_matches, iter_placeholders,
8
- iter_token_matches,
5
+from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
6
+ find_text_matches, find_token_matches,
7
+ iter_placeholders, iter_token_matches,
8
replace_text_matches,
9
replace_token_matches)
10
from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -314,8 +313,8 @@ def test_find_replace_text(
313
result = replace_text_matches(
314
prompt,
315
matches,
317
- MultiModalDataItems({key: [None] * mm_count
318
- for key in repl_by_key}),
316
+ {key: mm_count
317
+ for key in repl_by_key},
318
)
319
320
# Only displayed on error
@@ -380,8 +379,8 @@ def test_find_replace_tokens(
379
result = replace_token_matches(
380
prompt,
381
matches,
383
- MultiModalDataItems({key: [None] * mm_count
384
- for key in repl_by_key}),
382
+ {key: mm_count
383
+ for key in repl_by_key},
384
)
385
386
# Only displayed on error
@@ -476,7 +475,7 @@ def test_iter_placeholders(
475
prompt_repls,
476
prompt,
477
# Effectively match all occurrences in the prompt
479
- MultiModalDataItems({key: [None] * 3 for key in repl_by_key}),
478
+ {key: 3 for key in repl_by_key},
479
))
480
481
# Only displayed on error
vllm/multimodal/processing.py
+23
-25
@@ -403,18 +403,17 @@ def _resolve_matches(
403
def _replace_matches(
404
prompt: _S,
405
matches: Sequence[_PromptReplacementMatch],
406
- mm_items: MultiModalDataItems,
406
+ mm_item_counts: Mapping[str, int],
407
) -> list[_S]:
408
out_seqs = list[_S]()
409
prev_end_idx = 0
410
- next_idx_by_modality = {modality: 0 for modality in mm_items}
410
+ next_idx_by_modality = {modality: 0 for modality in mm_item_counts}
411
412
for match in _resolve_matches(prompt, matches):
413
modality = match.modality
414
- modal_items = mm_items[modality]
414
415
item_idx = next_idx_by_modality[modality]
417
- if item_idx >= len(modal_items):
416
+ if item_idx >= mm_item_counts[modality]:
417
continue
418
419
start_idx = match.start_idx
@@ -441,13 +440,13 @@ def _replace_matches(
440
def replace_token_matches(
441
prompt: list[int],
442
matches: Sequence[_PromptReplacementTokenMatch],
444
- mm_items: MultiModalDataItems,
443
+ mm_item_counts: Mapping[str, int],
444
) -> list[int]:
445
"""Apply :code:`prompt_repls` to :code:`prompt`."""
446
if not matches:
447
return prompt
448
450
- token_id_seqs = _replace_matches(prompt, matches, mm_items)
449
+ token_id_seqs = _replace_matches(prompt, matches, mm_item_counts)
450
451
return flatten_2d_lists(token_id_seqs)
452
@@ -455,13 +454,13 @@ def replace_token_matches(
454
def replace_text_matches(
455
prompt: str,
456
matches: Sequence[_PromptReplacementTextMatch],
458
- mm_items: MultiModalDataItems,
457
+ mm_item_counts: Mapping[str, int],
458
) -> str:
459
"""Apply :code:`prompt_repls` to :code:`prompt`."""
460
if not matches:
461
return prompt
462
464
- texts = _replace_matches(prompt, matches, mm_items)
463
+ texts = _replace_matches(prompt, matches, mm_item_counts)
464
465
return "".join(texts)
466
@@ -470,9 +469,9 @@ def _iter_modality_placeholders(
469
prompt: list[int],
470
modality: str,
471
modality_repls: Sequence[_BoundPromptReplacement],
473
- modal_items: list[Any],
472
+ modal_item_count: int,
473
) -> Iterable[_PlaceholderInfo]:
475
- if len(modal_items) == 0:
474
+ if modal_item_count == 0:
475
return
476
477
prompt_len = len(prompt)
@@ -499,7 +498,7 @@ def _iter_modality_placeholders(
498
)
499
500
item_index += 1
502
- if item_index >= len(modal_items):
501
+ if item_index >= modal_item_count:
502
return
503
504
# Exclude overlapping matches
@@ -514,7 +513,7 @@ def _iter_modality_placeholders(
513
def iter_placeholders(
514
prompt_repls: Sequence[_BoundPromptReplacement],
515
prompt: list[int],
517
- mm_items: MultiModalDataItems,
516
+ mm_item_counts: Mapping[str, int],
517
) -> Iterable[_PlaceholderInfo]:
518
"""
519
Yield each set of placeholder tokens found in :code:`prompt`.
@@ -523,13 +522,13 @@ def iter_placeholders(
522
"""
523
repls_by_modality = dict(full_groupby_modality(prompt_repls))
524
526
- for modality, modal_items in mm_items.items():
525
+ for modality, modal_item_count in mm_item_counts.items():
526
if modality in repls_by_modality:
527
yield from _iter_modality_placeholders(
528
prompt,
529
modality,
530
repls_by_modality[modality],
532
- modal_items,
531
+ modal_item_count,
532
)
533
534
@@ -590,10 +589,10 @@ class BaseMultiModalProcessor(ABC):
589
self,
590
all_prompt_repls: Sequence[_BoundPromptReplacement],
591
new_token_ids: list[int],
593
- mm_items: MultiModalDataItems,
592
+ mm_item_counts: Mapping[str, int],
593
) -> list[_PlaceholderInfo]:
594
return list(
596
- iter_placeholders(all_prompt_repls, new_token_ids, mm_items))
595
+ iter_placeholders(all_prompt_repls, new_token_ids, mm_item_counts))
596
597
def _apply_hf_processor(
598
self,
@@ -655,10 +654,9 @@ class BaseMultiModalProcessor(ABC):
654
655
def _apply_prompt_replacements(
656
self,
658
- mm_items: MultiModalDataItems,
659
- hf_inputs: BatchFeature,
657
token_ids: list[int],
658
prompt_repls: Sequence[_BoundPromptReplacement],
659
+ mm_item_counts: Mapping[str, int],
660
) -> tuple[list[int], str, list[_PlaceholderInfo]]:
661
tokenizer = self._get_tokenizer()
662
@@ -675,13 +673,13 @@ class BaseMultiModalProcessor(ABC):
673
# of the search text in the prompt, we instead perform string
674
# replacement on the decoded token IDs, then encode them back.
675
if all(
678
- len(matches) >= len(mm_items[modality])
676
+ len(matches) >= mm_item_counts[modality]
677
for modality, matches in full_groupby_modality(token_matches)
678
): # yapf: disable
679
token_ids = replace_token_matches(
680
token_ids,
681
token_matches,
684
- mm_items,
682
+ mm_item_counts,
683
)
684
685
text = _decode(tokenizer, token_ids)
@@ -693,14 +691,14 @@ class BaseMultiModalProcessor(ABC):
691
text = replace_text_matches(
692
text,
693
text_matches,
696
- mm_items,
694
+ mm_item_counts,
695
)
696
697
token_ids = _encode(tokenizer, text)
698
matched_repls = [match.prompt_repl for match in text_matches]
699
700
placeholders = self._find_placeholders(matched_repls, token_ids,
703
- mm_items)
701
+ mm_item_counts)
702
703
return token_ids, text, placeholders
704
@@ -737,8 +735,9 @@ class BaseMultiModalProcessor(ABC):
735
736
# If HF processor already inserts placeholder tokens,
737
# there is no need for us to insert them
738
+ mm_item_counts = {m: len(items) for m, items in mm_items.items()}
739
all_placeholders = self._find_placeholders(all_prompt_repls,
741
- prompt_ids, mm_items)
740
+ prompt_ids, mm_item_counts)
741
742
if all_placeholders:
743
prompt_text = _decode(tokenizer, prompt_ids)
@@ -748,10 +747,9 @@ class BaseMultiModalProcessor(ABC):
747
prompt_text,
748
all_placeholders,
749
) = self._apply_prompt_replacements(
751
- mm_items,
752
- hf_inputs,
750
prompt_ids,
751
all_prompt_repls,
752
+ mm_item_counts,
753
)
754
755
mm_placeholders = {