A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix various bugs in multi-modal processor (#12031)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 14, 2025 at 20:16 UTC
bb354e6b2dd4f8154c39c33b7eee77fa452b7703
3 files changed
+62
-51
tests/multimodal/test_processing.py
+19
@@ -421,6 +421,8 @@ def test_find_replace_tokens(
421
"pattern_1": [32000, 32000],
422
"pattern_2": [],
423
"pattern_3": [1550, 918, 1550],
424
+ # Test different modalities having the same tokens (32000)
425
+ "pattern_4": [32000],
426
},
427
],
428
)
@@ -438,6 +440,14 @@ def test_find_replace_tokens(
440
replacement=[32000, 32000],
441
),
442
],
443
+ "pattern_4": [
444
+ PlaceholderInfo(
445
+ modality="pattern_4",
446
+ item_idx=0,
447
+ start_idx=3,
448
+ replacement=[32000],
449
+ ),
450
+ ],
451
}
452
453
),
@@ -466,6 +476,7 @@ def test_find_replace_tokens(
476
replacement=[1550, 918, 1550],
477
),
478
],
479
+ # No match for pattern_4 as it has lower priority than pattern_1
480
}
481
),
482
(
@@ -485,6 +496,14 @@ def test_find_replace_tokens(
496
replacement=[32000, 32000],
497
),
498
],
499
+ "pattern_4": [
500
+ PlaceholderInfo(
501
+ modality="pattern_4",
502
+ item_idx=0,
503
+ start_idx=5,
504
+ replacement=[32000],
505
+ ),
506
+ ],
507
"pattern_3": [
508
PlaceholderInfo(
509
modality="pattern_3",
vllm/multimodal/processing.py
+39
-50
@@ -404,73 +404,62 @@ def replace_text_matches(
404
return "".join(texts)
405
406
407
-def _iter_modality_placeholders(
407
+def _iter_placeholders(
408
+ mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
409
prompt: list[int],
409
- modality: str,
410
- modality_repls: Sequence[BoundPromptReplacement],
411
- modal_item_count: int,
410
+ mm_item_counts: Mapping[str, int],
411
) -> Iterable[PlaceholderInfo]:
413
- if modal_item_count == 0:
414
- return
412
+ """
413
+ Yield each set of placeholder tokens found in :code:`prompt`.
414
+
415
+ Matches are exclusive even when multiple modalities share
416
+ the same placeholder tokens. In that case, the modality that
417
+ appears earlier in `mm_prompt_repls` takes priority.
418
419
+ Note that empty matches are ignored.
420
+ """
421
prompt_len = len(prompt)
417
- item_idx = 0
422
+ item_idx_by_modality = defaultdict[str, int](lambda: 0)
423
424
start_idx = 0
425
while start_idx < prompt_len:
426
found = False
427
423
- for repl_info in modality_repls:
424
- replacement = repl_info.get_replacement(item_idx)
425
- repl_tokens = replacement.token_ids
426
- repl_len = len(repl_tokens)
427
- end_idx = start_idx + repl_len
428
-
429
- if repl_len == 0 or end_idx > prompt_len:
428
+ for modality, modality_repls in mm_prompt_repls.items():
429
+ item_idx = item_idx_by_modality[modality]
430
+ if item_idx >= mm_item_counts.get(modality, 0):
431
continue
432
432
- if prompt[start_idx:end_idx] == repl_tokens:
433
- yield PlaceholderInfo(
434
- modality=modality,
435
- item_idx=item_idx,
436
- start_idx=start_idx,
437
- replacement=repl_tokens,
438
- )
433
+ for repl_info in modality_repls:
434
+ replacement = repl_info.get_replacement(item_idx)
435
+ repl_tokens = replacement.token_ids
436
+ repl_len = len(repl_tokens)
437
+ end_idx = start_idx + repl_len
438
+
439
+ if repl_len == 0 or end_idx > prompt_len:
440
+ continue
441
+
442
+ if prompt[start_idx:end_idx] == repl_tokens:
443
+ yield PlaceholderInfo(
444
+ modality=modality,
445
+ item_idx=item_idx,
446
+ start_idx=start_idx,
447
+ replacement=repl_tokens,
448
+ )
449
440
- item_idx += 1
441
- if item_idx >= modal_item_count:
442
- return
450
+ # Exclude overlapping matches
451
+ start_idx = end_idx
452
+ item_idx_by_modality[modality] += 1
453
+ found = True
454
+ break
455
444
- # Exclude overlapping matches
445
- start_idx = end_idx
446
- found = True
447
- break
456
+ if found:
457
+ break # Go back to the outer while loop
458
459
if not found:
460
start_idx += 1
461
462
453
-def _iter_placeholders(
454
- mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
455
- prompt: list[int],
456
- mm_item_counts: Mapping[str, int],
457
-) -> Iterable[PlaceholderInfo]:
458
- """
459
- For each modality, yield each set of placeholder tokens found in
460
- :code:`prompt`.
461
-
462
- Note that empty matches are ignored.
463
- """
464
- for modality, modal_item_count in mm_item_counts.items():
465
- if modality in mm_prompt_repls:
466
- yield from _iter_modality_placeholders(
467
- prompt,
468
- modality,
469
- mm_prompt_repls[modality],
470
- modal_item_count,
471
- )
472
-
473
-
463
def find_mm_placeholders(
464
mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
465
prompt: list[int],
@@ -1156,7 +1145,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1145
1146
# If HF processor already inserts placeholder tokens,
1147
# there is no need for us to insert them
1159
- if all(len(repls) == 0 for repls in mm_missing_repls.items()):
1148
+ if all(len(repls) == 0 for repls in mm_missing_repls.values()):
1149
tokenizer = self.info.get_tokenizer()
1150
prompt = decode_tokens(tokenizer, prompt_ids)
1151
mm_placeholders = hf_mm_placeholders
vllm/multimodal/registry.py
+4
-1
@@ -259,7 +259,10 @@ class MultiModalRegistry:
259
This is currently directly used only in V1.
260
"""
261
if self.has_processor(model_config):
262
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
262
+ tokenizer = cached_get_tokenizer(
263
+ model_config.tokenizer,
264
+ trust_remote_code=model_config.trust_remote_code,
265
+ )
266
processor = self.create_processor(model_config, tokenizer)
267
seq_len = model_config.max_model_len
268
return processor.info.get_mm_max_tokens_per_item(seq_len)