@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b10609e6a1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Clean up multi-modal processor (#11207)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 15, 2024 at 14:30 UTC b10609e6a11554be61976981304984510a0469c9
3 files changed +32 -38
examples/offline_inference_vision_language.py
+1 -4
@@ -92,10 +92,7 @@ def run_fuyu(question: str, modality: str):
92 def run_phi3v(question: str, modality: str):
93 assert modality == "image"
94
95 - prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501
96 - # Note: The default setting of max_num_seqs (256) and
97 - # max_model_len (128k) for this model may cause OOM.
98 - # You may lower either to run this example on lower-end GPUs.
95 + prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
96
97 # num_crops is an override kwarg to the multimodal image processor;
98 # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
tests/multimodal/test_processing.py
+8 -9
@@ -2,10 +2,9 @@ from typing import cast
2
3 import pytest
4
5 -from vllm.multimodal.processing import (MultiModalDataItems, PromptReplacement,
6 - _PlaceholderInfo, find_text_matches,
7 - find_token_matches, iter_placeholders,
8 - iter_token_matches,
5 +from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
6 + find_text_matches, find_token_matches,
7 + iter_placeholders, iter_token_matches,
8 replace_text_matches,
9 replace_token_matches)
10 from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -314,8 +313,8 @@ def test_find_replace_text(
313 result = replace_text_matches(
314 prompt,
315 matches,
317 - MultiModalDataItems({key: [None] * mm_count
318 - for key in repl_by_key}),
316 + {key: mm_count
317 + for key in repl_by_key},
318 )
319
320 # Only displayed on error
@@ -380,8 +379,8 @@ def test_find_replace_tokens(
379 result = replace_token_matches(
380 prompt,
381 matches,
383 - MultiModalDataItems({key: [None] * mm_count
384 - for key in repl_by_key}),
382 + {key: mm_count
383 + for key in repl_by_key},
384 )
385
386 # Only displayed on error
@@ -476,7 +475,7 @@ def test_iter_placeholders(
475 prompt_repls,
476 prompt,
477 # Effectively match all occurrences in the prompt
479 - MultiModalDataItems({key: [None] * 3 for key in repl_by_key}),
478 + {key: 3 for key in repl_by_key},
479 ))
480
481 # Only displayed on error
vllm/multimodal/processing.py
+23 -25
@@ -403,18 +403,17 @@ def _resolve_matches(
403 def _replace_matches(
404 prompt: _S,
405 matches: Sequence[_PromptReplacementMatch],
406 - mm_items: MultiModalDataItems,
406 + mm_item_counts: Mapping[str, int],
407 ) -> list[_S]:
408 out_seqs = list[_S]()
409 prev_end_idx = 0
410 - next_idx_by_modality = {modality: 0 for modality in mm_items}
410 + next_idx_by_modality = {modality: 0 for modality in mm_item_counts}
411
412 for match in _resolve_matches(prompt, matches):
413 modality = match.modality
414 - modal_items = mm_items[modality]
414
415 item_idx = next_idx_by_modality[modality]
417 - if item_idx >= len(modal_items):
416 + if item_idx >= mm_item_counts[modality]:
417 continue
418
419 start_idx = match.start_idx
@@ -441,13 +440,13 @@ def _replace_matches(
440 def replace_token_matches(
441 prompt: list[int],
442 matches: Sequence[_PromptReplacementTokenMatch],
444 - mm_items: MultiModalDataItems,
443 + mm_item_counts: Mapping[str, int],
444 ) -> list[int]:
445 """Apply :code:`prompt_repls` to :code:`prompt`."""
446 if not matches:
447 return prompt
448
450 - token_id_seqs = _replace_matches(prompt, matches, mm_items)
449 + token_id_seqs = _replace_matches(prompt, matches, mm_item_counts)
450
451 return flatten_2d_lists(token_id_seqs)
452
@@ -455,13 +454,13 @@ def replace_token_matches(
454 def replace_text_matches(
455 prompt: str,
456 matches: Sequence[_PromptReplacementTextMatch],
458 - mm_items: MultiModalDataItems,
457 + mm_item_counts: Mapping[str, int],
458 ) -> str:
459 """Apply :code:`prompt_repls` to :code:`prompt`."""
460 if not matches:
461 return prompt
462
464 - texts = _replace_matches(prompt, matches, mm_items)
463 + texts = _replace_matches(prompt, matches, mm_item_counts)
464
465 return "".join(texts)
466
@@ -470,9 +469,9 @@ def _iter_modality_placeholders(
469 prompt: list[int],
470 modality: str,
471 modality_repls: Sequence[_BoundPromptReplacement],
473 - modal_items: list[Any],
472 + modal_item_count: int,
473 ) -> Iterable[_PlaceholderInfo]:
475 - if len(modal_items) == 0:
474 + if modal_item_count == 0:
475 return
476
477 prompt_len = len(prompt)
@@ -499,7 +498,7 @@ def _iter_modality_placeholders(
498 )
499
500 item_index += 1
502 - if item_index >= len(modal_items):
501 + if item_index >= modal_item_count:
502 return
503
504 # Exclude overlapping matches
@@ -514,7 +513,7 @@ def _iter_modality_placeholders(
513 def iter_placeholders(
514 prompt_repls: Sequence[_BoundPromptReplacement],
515 prompt: list[int],
517 - mm_items: MultiModalDataItems,
516 + mm_item_counts: Mapping[str, int],
517 ) -> Iterable[_PlaceholderInfo]:
518 """
519 Yield each set of placeholder tokens found in :code:`prompt`.
@@ -523,13 +522,13 @@ def iter_placeholders(
522 """
523 repls_by_modality = dict(full_groupby_modality(prompt_repls))
524
526 - for modality, modal_items in mm_items.items():
525 + for modality, modal_item_count in mm_item_counts.items():
526 if modality in repls_by_modality:
527 yield from _iter_modality_placeholders(
528 prompt,
529 modality,
530 repls_by_modality[modality],
532 - modal_items,
531 + modal_item_count,
532 )
533
534
@@ -590,10 +589,10 @@ class BaseMultiModalProcessor(ABC):
589 self,
590 all_prompt_repls: Sequence[_BoundPromptReplacement],
591 new_token_ids: list[int],
593 - mm_items: MultiModalDataItems,
592 + mm_item_counts: Mapping[str, int],
593 ) -> list[_PlaceholderInfo]:
594 return list(
596 - iter_placeholders(all_prompt_repls, new_token_ids, mm_items))
595 + iter_placeholders(all_prompt_repls, new_token_ids, mm_item_counts))
596
597 def _apply_hf_processor(
598 self,
@@ -655,10 +654,9 @@ class BaseMultiModalProcessor(ABC):
654
655 def _apply_prompt_replacements(
656 self,
658 - mm_items: MultiModalDataItems,
659 - hf_inputs: BatchFeature,
657 token_ids: list[int],
658 prompt_repls: Sequence[_BoundPromptReplacement],
659 + mm_item_counts: Mapping[str, int],
660 ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
661 tokenizer = self._get_tokenizer()
662
@@ -675,13 +673,13 @@ class BaseMultiModalProcessor(ABC):
673 # of the search text in the prompt, we instead perform string
674 # replacement on the decoded token IDs, then encode them back.
675 if all(
678 - len(matches) >= len(mm_items[modality])
676 + len(matches) >= mm_item_counts[modality]
677 for modality, matches in full_groupby_modality(token_matches)
678 ): # yapf: disable
679 token_ids = replace_token_matches(
680 token_ids,
681 token_matches,
684 - mm_items,
682 + mm_item_counts,
683 )
684
685 text = _decode(tokenizer, token_ids)
@@ -693,14 +691,14 @@ class BaseMultiModalProcessor(ABC):
691 text = replace_text_matches(
692 text,
693 text_matches,
696 - mm_items,
694 + mm_item_counts,
695 )
696
697 token_ids = _encode(tokenizer, text)
698 matched_repls = [match.prompt_repl for match in text_matches]
699
700 placeholders = self._find_placeholders(matched_repls, token_ids,
703 - mm_items)
701 + mm_item_counts)
702
703 return token_ids, text, placeholders
704
@@ -737,8 +735,9 @@ class BaseMultiModalProcessor(ABC):
735
736 # If HF processor already inserts placeholder tokens,
737 # there is no need for us to insert them
738 + mm_item_counts = {m: len(items) for m, items in mm_items.items()}
739 all_placeholders = self._find_placeholders(all_prompt_repls,
741 - prompt_ids, mm_items)
740 + prompt_ids, mm_item_counts)
741
742 if all_placeholders:
743 prompt_text = _decode(tokenizer, prompt_ids)
@@ -748,10 +747,9 @@ class BaseMultiModalProcessor(ABC):
747 prompt_text,
748 all_placeholders,
749 ) = self._apply_prompt_replacements(
751 - mm_items,
752 - hf_inputs,
750 prompt_ids,
751 all_prompt_repls,
752 + mm_item_counts,
753 )
754
755 mm_placeholders = {