@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8e3e7f2713
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model][Pixtral] Optimizations for input_processor_for_pixtral_hf (#9514)

Michael Goin committed Oct 19, 2024 at 10:44 UTC 8e3e7f271326e8cdb32c8f9581b2f98013a567c7
1 file changed +41 -40
vllm/model_executor/models/pixtral.py
+41 -40
@@ -701,63 +701,64 @@ def input_processor_for_pixtral_hf(
701 new_prompt = inputs.get("prompt")
702 new_token_ids = inputs["prompt_token_ids"]
703
704 + image_token = processor.image_token
705 + image_break_token = processor.image_break_token
706 + image_end_token = processor.image_end_token
707 +
708 # Update new_prompt if present
709 if new_prompt:
706 - replace_strings = []
707 - for image in image_data:
708 - w, h = image.size
710 + parts = new_prompt.split(image_token)
711 + assert len(parts) - 1 == len(image_data)
712 + new_parts = [parts[0]] # Start with the part before any image tokens
713
714 + for image, next_part in zip(image_data, parts[1:]):
715 + w, h = image.size
716 (num_width_tokens,
717 num_height_tokens) = get_pixtral_hf_image_feature_size(
718 hf_config, image_width=w, image_height=h)
719
714 - replace_tokens = [[processor.image_token] * num_width_tokens +
715 - [processor.image_break_token]
716 - ] * num_height_tokens
717 - # Flatten list
718 - replace_tokens = [
719 - item for sublist in replace_tokens for item in sublist
720 + replace_tokens = [image_token] * num_width_tokens + [
721 + image_break_token
722 ]
721 - replace_tokens[-1] = processor.image_end_token
722 - replace_str = "".join(replace_tokens)
723 - replace_strings.append(replace_str)
724 - new_prompt = new_prompt.replace(processor.image_token,
725 - "<placeholder>", 1)
723 + replace_tokens = replace_tokens * num_height_tokens
724 + replace_tokens[-1] = image_end_token
725
727 - while "<placeholder>" in new_prompt:
728 - replace_str = replace_strings.pop(0)
729 - new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
726 + new_parts.append("".join(replace_tokens))
727 + new_parts.append(next_part)
728 +
729 + new_prompt = "".join(new_parts)
730
731 # Update new_token_ids
732 - image_token_id = 10
733 - image_break_id = 12
734 - image_end_id = 13
732 + convert_tokens_to_ids = processor.tokenizer.convert_tokens_to_ids
733 + image_token_id = convert_tokens_to_ids(image_token)
734 + image_break_id = convert_tokens_to_ids(image_break_token)
735 + image_end_id = convert_tokens_to_ids(image_end_token)
736 placeholder_token_id = -999
737 + # Find all image token indices at once
738 + placeholder_indices = [
739 + idx for idx, token_id in enumerate(new_token_ids)
740 + if token_id == image_token_id
741 + ]
742 + assert len(placeholder_indices) == len(image_data)
743 replace_tokens_list = []
737 - for image in image_data:
738 - w, h = image.size
744 + for placeholder_idx, image in zip(placeholder_indices, image_data):
745 + new_token_ids[placeholder_idx] = placeholder_token_id
746
740 - num_width_tokens, num_height_tokens = get_pixtral_hf_image_feature_size(
741 - hf_config, image_width=w, image_height=h)
747 + w, h = image.size
748 + (num_width_tokens,
749 + num_height_tokens) = get_pixtral_hf_image_feature_size(hf_config,
750 + image_width=w,
751 + image_height=h)
752
743 - replace_tokens = [[image_token_id] * num_width_tokens +
744 - [image_break_id]] * num_height_tokens
745 - # Flatten list
746 - replace_tokens = [
747 - item for sublist in replace_tokens for item in sublist
748 - ]
753 + replace_tokens = [image_token_id] * num_width_tokens + [image_break_id]
754 + replace_tokens = replace_tokens * num_height_tokens
755 replace_tokens[-1] = image_end_id
756 replace_tokens_list.append(replace_tokens)
751 - # Replace image id with placeholder id
752 - next_image_index = new_token_ids.index(image_token_id)
753 - new_token_ids[next_image_index] = placeholder_token_id
754 -
755 - while placeholder_token_id in new_token_ids:
756 - replace_tokens = replace_tokens_list.pop(0)
757 - next_image_index = new_token_ids.index(placeholder_token_id)
758 - prefix = new_token_ids[:next_image_index]
759 - postfix = new_token_ids[next_image_index + 1:]
760 - new_token_ids = prefix + replace_tokens + postfix
757 +
758 + # Backward iteration for replacement without affecting known indices
759 + for placeholder_idx, replace_tokens in zip(reversed(placeholder_indices),
760 + reversed(replace_tokens_list)):
761 + new_token_ids[placeholder_idx:placeholder_idx + 1] = replace_tokens
762
763 # NOTE: Create a defensive copy of the original inputs
764 return token_inputs(prompt_token_ids=new_token_ids,