@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb354e6b2d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix various bugs in multi-modal processor (#12031)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 14, 2025 at 20:16 UTC bb354e6b2dd4f8154c39c33b7eee77fa452b7703
3 files changed +62 -51
tests/multimodal/test_processing.py
+19
@@ -421,6 +421,8 @@ def test_find_replace_tokens(
421 "pattern_1": [32000, 32000],
422 "pattern_2": [],
423 "pattern_3": [1550, 918, 1550],
424 + # Test different modalities having the same tokens (32000)
425 + "pattern_4": [32000],
426 },
427 ],
428 )
@@ -438,6 +440,14 @@ def test_find_replace_tokens(
440 replacement=[32000, 32000],
441 ),
442 ],
443 + "pattern_4": [
444 + PlaceholderInfo(
445 + modality="pattern_4",
446 + item_idx=0,
447 + start_idx=3,
448 + replacement=[32000],
449 + ),
450 + ],
451 }
452
453 ),
@@ -466,6 +476,7 @@ def test_find_replace_tokens(
476 replacement=[1550, 918, 1550],
477 ),
478 ],
479 + # No match for pattern_4 as it has lower priority than pattern_1
480 }
481 ),
482 (
@@ -485,6 +496,14 @@ def test_find_replace_tokens(
496 replacement=[32000, 32000],
497 ),
498 ],
499 + "pattern_4": [
500 + PlaceholderInfo(
501 + modality="pattern_4",
502 + item_idx=0,
503 + start_idx=5,
504 + replacement=[32000],
505 + ),
506 + ],
507 "pattern_3": [
508 PlaceholderInfo(
509 modality="pattern_3",
vllm/multimodal/processing.py
+39 -50
@@ -404,73 +404,62 @@ def replace_text_matches(
404 return "".join(texts)
405
406
407 -def _iter_modality_placeholders(
407 +def _iter_placeholders(
408 + mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
409 prompt: list[int],
409 - modality: str,
410 - modality_repls: Sequence[BoundPromptReplacement],
411 - modal_item_count: int,
410 + mm_item_counts: Mapping[str, int],
411 ) -> Iterable[PlaceholderInfo]:
413 - if modal_item_count == 0:
414 - return
412 + """
413 + Yield each set of placeholder tokens found in :code:`prompt`.
414 +
415 + Matches are exclusive even when multiple modalities share
416 + the same placeholder tokens. In that case, the modality that
417 + appears earlier in `mm_prompt_repls` takes priority.
418
419 + Note that empty matches are ignored.
420 + """
421 prompt_len = len(prompt)
417 - item_idx = 0
422 + item_idx_by_modality = defaultdict[str, int](lambda: 0)
423
424 start_idx = 0
425 while start_idx < prompt_len:
426 found = False
427
423 - for repl_info in modality_repls:
424 - replacement = repl_info.get_replacement(item_idx)
425 - repl_tokens = replacement.token_ids
426 - repl_len = len(repl_tokens)
427 - end_idx = start_idx + repl_len
428 -
429 - if repl_len == 0 or end_idx > prompt_len:
428 + for modality, modality_repls in mm_prompt_repls.items():
429 + item_idx = item_idx_by_modality[modality]
430 + if item_idx >= mm_item_counts.get(modality, 0):
431 continue
432
432 - if prompt[start_idx:end_idx] == repl_tokens:
433 - yield PlaceholderInfo(
434 - modality=modality,
435 - item_idx=item_idx,
436 - start_idx=start_idx,
437 - replacement=repl_tokens,
438 - )
433 + for repl_info in modality_repls:
434 + replacement = repl_info.get_replacement(item_idx)
435 + repl_tokens = replacement.token_ids
436 + repl_len = len(repl_tokens)
437 + end_idx = start_idx + repl_len
438 +
439 + if repl_len == 0 or end_idx > prompt_len:
440 + continue
441 +
442 + if prompt[start_idx:end_idx] == repl_tokens:
443 + yield PlaceholderInfo(
444 + modality=modality,
445 + item_idx=item_idx,
446 + start_idx=start_idx,
447 + replacement=repl_tokens,
448 + )
449
440 - item_idx += 1
441 - if item_idx >= modal_item_count:
442 - return
450 + # Exclude overlapping matches
451 + start_idx = end_idx
452 + item_idx_by_modality[modality] += 1
453 + found = True
454 + break
455
444 - # Exclude overlapping matches
445 - start_idx = end_idx
446 - found = True
447 - break
456 + if found:
457 + break # Go back to the outer while loop
458
459 if not found:
460 start_idx += 1
461
462
453 -def _iter_placeholders(
454 - mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
455 - prompt: list[int],
456 - mm_item_counts: Mapping[str, int],
457 -) -> Iterable[PlaceholderInfo]:
458 - """
459 - For each modality, yield each set of placeholder tokens found in
460 - :code:`prompt`.
461 -
462 - Note that empty matches are ignored.
463 - """
464 - for modality, modal_item_count in mm_item_counts.items():
465 - if modality in mm_prompt_repls:
466 - yield from _iter_modality_placeholders(
467 - prompt,
468 - modality,
469 - mm_prompt_repls[modality],
470 - modal_item_count,
471 - )
472 -
473 -
463 def find_mm_placeholders(
464 mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
465 prompt: list[int],
@@ -1156,7 +1145,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1145
1146 # If HF processor already inserts placeholder tokens,
1147 # there is no need for us to insert them
1159 - if all(len(repls) == 0 for repls in mm_missing_repls.items()):
1148 + if all(len(repls) == 0 for repls in mm_missing_repls.values()):
1149 tokenizer = self.info.get_tokenizer()
1150 prompt = decode_tokens(tokenizer, prompt_ids)
1151 mm_placeholders = hf_mm_placeholders
vllm/multimodal/registry.py
+4 -1
@@ -259,7 +259,10 @@ class MultiModalRegistry:
259 This is currently directly used only in V1.
260 """
261 if self.has_processor(model_config):
262 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
262 + tokenizer = cached_get_tokenizer(
263 + model_config.tokenizer,
264 + trust_remote_code=model_config.trust_remote_code,
265 + )
266 processor = self.create_processor(model_config, tokenizer)
267 seq_len = model_config.max_model_len
268 return processor.info.get_mm_max_tokens_per_item(seq_len)