@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a0f7d53beb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Cleanup Pixtral HF code (#11333)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 19, 2024 at 21:22 UTC a0f7d53beb176034546c6deb328a3d49e94e1f6d
1 file changed +14 -141
vllm/model_executor/models/pixtral.py
+14 -141
@@ -10,12 +10,12 @@ from mistral_common.protocol.instruct.messages import ImageChunk
10 from PIL import Image
11 from transformers import PixtralVisionConfig
12 from transformers.models.pixtral.image_processing_pixtral import (
13 - _num_image_tokens)
13 + _num_image_tokens as _get_pixtral_hf_num_image_tokens)
14 from transformers.models.pixtral.modeling_pixtral import (
15 PixtralRotaryEmbedding, apply_rotary_pos_emb, position_ids_in_meshgrid)
16
17 from vllm.attention import AttentionMetadata
18 -from vllm.config import ModelConfig, VllmConfig
18 +from vllm.config import VllmConfig
19 from vllm.distributed import divide, get_tensor_model_parallel_world_size
20 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 InputContext, token_inputs)
@@ -27,7 +27,6 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
27 from vllm.model_executor.layers.quantization import QuantizationConfig
28 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 -from vllm.model_executor.models.utils import merge_multimodal_embeddings
30 from vllm.model_executor.sampling_metadata import SamplingMetadata
31 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
32 from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
@@ -35,11 +34,10 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
34 consecutive_placeholder_ranges,
35 resolve_visual_encoder_outputs)
36 from vllm.sequence import IntermediateTensors, SequenceData
38 -from vllm.transformers_utils.processor import cached_get_processor
39 -from vllm.utils import is_list_of
37
38 from .interfaces import SupportsMultiModal, SupportsPP
42 -from .utils import init_vllm_registered_model, maybe_prefix
39 +from .utils import (init_vllm_registered_model, maybe_prefix,
40 + merge_multimodal_embeddings)
41
42 try:
43 from xformers import ops as xops
@@ -699,37 +697,14 @@ def get_pixtral_hf_num_patches(*, image_size: int, patch_size: int) -> int:
697 return grid_length * grid_length
698
699
702 -def get_max_pixtral_hf_image_feature_size(
703 - hf_config: PixtralVisionConfig) -> int:
704 - return get_pixtral_hf_num_patches(image_size=hf_config.image_size,
705 - patch_size=hf_config.patch_size)
706 -
707 -
700 def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:
709 - return get_max_pixtral_hf_image_feature_size(hf_config)
701 + grid_length = get_pixtral_hf_patch_grid_length(
702 + image_size=hf_config.image_size,
703 + patch_size=hf_config.patch_size,
704 + )
705
711 -
712 -def dummy_seq_data_for_pixtral_hf(
713 - hf_config: PixtralVisionConfig,
714 - seq_len: int,
715 - num_images: int,
716 - *,
717 - image_token_id: int,
718 - image_feature_size_override: Optional[int] = None,
719 - mm_key: str = "image"):
720 - if image_feature_size_override is None:
721 - image_feature_size = get_max_pixtral_hf_image_feature_size(hf_config)
722 - else:
723 - image_feature_size = image_feature_size_override
724 -
725 - return SequenceData.from_prompt_token_counts(
726 - (image_token_id, image_feature_size * num_images),
727 - (0, seq_len - image_feature_size * num_images),
728 - ), {
729 - mm_key:
730 - consecutive_placeholder_ranges(num_items=num_images,
731 - item_size=image_feature_size)
732 - }
706 + # Consider the image_break_token
707 + return (grid_length + 1) * grid_length
708
709
710 def dummy_image_for_pixtral_hf(
@@ -763,116 +738,14 @@ def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
738 image_width = int(numpy.ceil(image_width / ratio))
739 image_height = int(numpy.ceil(image_height / ratio))
740
766 - num_height_tokens, num_width_tokens = _num_image_tokens(
767 - (image_height, image_width), (patch_height, patch_width))
741 + num_height_tokens, num_width_tokens = _get_pixtral_hf_num_image_tokens(
742 + (image_height, image_width),
743 + (patch_height, patch_width),
744 + )
745
746 return num_width_tokens, num_height_tokens
747
748
772 -def input_processor_for_pixtral_hf(
773 - model_config: ModelConfig,
774 - hf_config: PixtralVisionConfig,
775 - inputs: DecoderOnlyInputs,
776 - *,
777 - image_token_id: int,
778 - image_feature_size_override: Optional[Union[int, List[int]]] = None,
779 -) -> DecoderOnlyInputs:
780 - assert image_feature_size_override is None, (
781 - "image_feature_size_override is not supported for Pixtral")
782 -
783 - multi_modal_data = inputs.get("multi_modal_data")
784 - if multi_modal_data is None or "image" not in multi_modal_data:
785 - return inputs
786 -
787 - processor = cached_get_processor(model_config.model)
788 -
789 - image_data = multi_modal_data["image"]
790 - if isinstance(image_data, Image.Image):
791 - image_data = [image_data]
792 - elif not is_list_of(image_data, Image.Image):
793 - raise TypeError(f"Invalid image type: {type(image_data)}")
794 -
795 - new_prompt = inputs.get("prompt")
796 - new_token_ids = inputs["prompt_token_ids"]
797 -
798 - image_token = processor.image_token
799 - image_break_token = processor.image_break_token
800 - image_end_token = processor.image_end_token
801 -
802 - # Update new_prompt if present
803 - if new_prompt:
804 - parts = new_prompt.split(image_token)
805 - assert len(parts) - 1 == len(image_data)
806 - new_parts = [parts[0]] # Start with the part before any image tokens
807 -
808 - for image, next_part in zip(image_data, parts[1:]):
809 - w, h = image.size
810 - (num_width_tokens,
811 - num_height_tokens) = get_pixtral_hf_image_feature_size(
812 - hf_config, image_width=w, image_height=h)
813 -
814 - replace_tokens = [image_token] * num_width_tokens + [
815 - image_break_token
816 - ]
817 - replace_tokens = replace_tokens * num_height_tokens
818 - replace_tokens[-1] = image_end_token
819 -
820 - new_parts.append("".join(replace_tokens))
821 - new_parts.append(next_part)
822 -
823 - new_prompt = "".join(new_parts)
824 -
825 - # Update new_token_ids
826 - convert_tokens_to_ids = processor.tokenizer.convert_tokens_to_ids
827 - image_token_id = convert_tokens_to_ids(image_token)
828 - image_break_id = convert_tokens_to_ids(image_break_token)
829 - image_end_id = convert_tokens_to_ids(image_end_token)
830 - placeholder_token_id = -999
831 - # Find all image token indices at once
832 - placeholder_indices = [
833 - idx for idx, token_id in enumerate(new_token_ids)
834 - if token_id == image_token_id
835 - ]
836 - assert len(placeholder_indices) == len(image_data)
837 - replace_tokens_list = []
838 - for placeholder_idx, image in zip(placeholder_indices, image_data):
839 - new_token_ids[placeholder_idx] = placeholder_token_id
840 -
841 - w, h = image.size
842 - (num_width_tokens,
843 - num_height_tokens) = get_pixtral_hf_image_feature_size(hf_config,
844 - image_width=w,
845 - image_height=h)
846 -
847 - replace_tokens = [image_token_id] * num_width_tokens + [image_break_id]
848 - replace_tokens = replace_tokens * num_height_tokens
849 - replace_tokens[-1] = image_end_id
850 - replace_tokens_list.append(replace_tokens)
851 -
852 - reverse_offsets: List[int] = []
853 - # Backward iteration for replacement without affecting known indices
854 - for placeholder_idx, replace_tokens in zip(reversed(placeholder_indices),
855 - reversed(replace_tokens_list)):
856 - reverse_offsets.append(
857 - len(new_token_ids) - placeholder_idx + len(replace_tokens))
858 - new_token_ids[placeholder_idx:placeholder_idx + 1] = replace_tokens
859 -
860 - placeholder_ranges: List[PlaceholderRange] = []
861 - for reverse_offset, replace_tokens in zip(reversed(reverse_offsets),
862 - replace_tokens_list):
863 - placeholder_ranges.append(
864 - PlaceholderRange(
865 - offset=len(new_token_ids) - reverse_offset,
866 - length=len(replace_tokens),
867 - ))
868 -
869 - # NOTE: Create a defensive copy of the original inputs
870 - return token_inputs(prompt_token_ids=new_token_ids,
871 - prompt=new_prompt,
872 - multi_modal_data=multi_modal_data,
873 - multi_modal_placeholders={"image": placeholder_ranges})
874 -
875 -
749 class PixtralHFMLP(nn.Module):
750
751 def __init__(