A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model][Pixtral] Optimizations for input_processor_for_pixtral_hf (#9514)
Michael Goin committed
Oct 19, 2024 at 10:44 UTC
8e3e7f271326e8cdb32c8f9581b2f98013a567c7
1 file changed
+41
-40
vllm/model_executor/models/pixtral.py
+41
-40
@@ -701,63 +701,64 @@ def input_processor_for_pixtral_hf(
701
new_prompt = inputs.get("prompt")
702
new_token_ids = inputs["prompt_token_ids"]
703
704
+ image_token = processor.image_token
705
+ image_break_token = processor.image_break_token
706
+ image_end_token = processor.image_end_token
707
+
708
# Update new_prompt if present
709
if new_prompt:
706
- replace_strings = []
707
- for image in image_data:
708
- w, h = image.size
710
+ parts = new_prompt.split(image_token)
711
+ assert len(parts) - 1 == len(image_data)
712
+ new_parts = [parts[0]] # Start with the part before any image tokens
713
714
+ for image, next_part in zip(image_data, parts[1:]):
715
+ w, h = image.size
716
(num_width_tokens,
717
num_height_tokens) = get_pixtral_hf_image_feature_size(
718
hf_config, image_width=w, image_height=h)
719
714
- replace_tokens = [[processor.image_token] * num_width_tokens +
715
- [processor.image_break_token]
716
- ] * num_height_tokens
717
- # Flatten list
718
- replace_tokens = [
719
- item for sublist in replace_tokens for item in sublist
720
+ replace_tokens = [image_token] * num_width_tokens + [
721
+ image_break_token
722
]
721
- replace_tokens[-1] = processor.image_end_token
722
- replace_str = "".join(replace_tokens)
723
- replace_strings.append(replace_str)
724
- new_prompt = new_prompt.replace(processor.image_token,
725
- "<placeholder>", 1)
723
+ replace_tokens = replace_tokens * num_height_tokens
724
+ replace_tokens[-1] = image_end_token
725
727
- while "<placeholder>" in new_prompt:
728
- replace_str = replace_strings.pop(0)
729
- new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
726
+ new_parts.append("".join(replace_tokens))
727
+ new_parts.append(next_part)
728
+
729
+ new_prompt = "".join(new_parts)
730
731
# Update new_token_ids
732
- image_token_id = 10
733
- image_break_id = 12
734
- image_end_id = 13
732
+ convert_tokens_to_ids = processor.tokenizer.convert_tokens_to_ids
733
+ image_token_id = convert_tokens_to_ids(image_token)
734
+ image_break_id = convert_tokens_to_ids(image_break_token)
735
+ image_end_id = convert_tokens_to_ids(image_end_token)
736
placeholder_token_id = -999
737
+ # Find all image token indices at once
738
+ placeholder_indices = [
739
+ idx for idx, token_id in enumerate(new_token_ids)
740
+ if token_id == image_token_id
741
+ ]
742
+ assert len(placeholder_indices) == len(image_data)
743
replace_tokens_list = []
737
- for image in image_data:
738
- w, h = image.size
744
+ for placeholder_idx, image in zip(placeholder_indices, image_data):
745
+ new_token_ids[placeholder_idx] = placeholder_token_id
746
740
- num_width_tokens, num_height_tokens = get_pixtral_hf_image_feature_size(
741
- hf_config, image_width=w, image_height=h)
747
+ w, h = image.size
748
+ (num_width_tokens,
749
+ num_height_tokens) = get_pixtral_hf_image_feature_size(hf_config,
750
+ image_width=w,
751
+ image_height=h)
752
743
- replace_tokens = [[image_token_id] * num_width_tokens +
744
- [image_break_id]] * num_height_tokens
745
- # Flatten list
746
- replace_tokens = [
747
- item for sublist in replace_tokens for item in sublist
748
- ]
753
+ replace_tokens = [image_token_id] * num_width_tokens + [image_break_id]
754
+ replace_tokens = replace_tokens * num_height_tokens
755
replace_tokens[-1] = image_end_id
756
replace_tokens_list.append(replace_tokens)
751
- # Replace image id with placeholder id
752
- next_image_index = new_token_ids.index(image_token_id)
753
- new_token_ids[next_image_index] = placeholder_token_id
754
-
755
- while placeholder_token_id in new_token_ids:
756
- replace_tokens = replace_tokens_list.pop(0)
757
- next_image_index = new_token_ids.index(placeholder_token_id)
758
- prefix = new_token_ids[:next_image_index]
759
- postfix = new_token_ids[next_image_index + 1:]
760
- new_token_ids = prefix + replace_tokens + postfix
757
+
758
+ # Backward iteration for replacement without affecting known indices
759
+ for placeholder_idx, replace_tokens in zip(reversed(placeholder_indices),
760
+ reversed(replace_tokens_list)):
761
+ new_token_ids[placeholder_idx:placeholder_idx + 1] = replace_tokens
762
763
# NOTE: Create a defensive copy of the original inputs
764
return token_inputs(prompt_token_ids=new_token_ids,