@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7cd7409142
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

PaliGemma 2 support (#11142)

Jani Monoses committed Dec 13, 2024 at 09:40 UTC 7cd7409142ff97aee1a13568753db9263fcf8f6b
3 files changed +25 -3
docs/source/models/supported_models.rst
+2 -2
@@ -664,9 +664,9 @@ Text Generation (``--task generate``)
664 - ✅︎
665 - ✅︎
666 * - :code:`PaliGemmaForConditionalGeneration`
667 - - PaliGemma
667 + - PaliGemma, PaliGemma 2
668 - T + I\ :sup:`E`
669 - - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, etc.
669 + - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, :code:`google/paligemma2-3b-ft-docci-448`, etc.
670 -
671 - ✅︎
672 -
examples/offline_inference_vision_language.py
+13
@@ -137,6 +137,18 @@ def run_paligemma(question: str, modality: str):
137 return llm, prompt, stop_token_ids
138
139
140 +# PaliGemma 2
141 +def run_paligemma2(question: str, modality: str):
142 + assert modality == "image"
143 +
144 + # PaliGemma 2 has special prompt format for VQA
145 + prompt = "caption en"
146 + llm = LLM(model="google/paligemma2-3b-ft-docci-448",
147 + mm_cache_preprocessor=args.mm_cache_preprocessor)
148 + stop_token_ids = None
149 + return llm, prompt, stop_token_ids
150 +
151 +
152 # Chameleon
153 def run_chameleon(question: str, modality: str):
154 assert modality == "image"
@@ -473,6 +485,7 @@ model_example_map = {
485 "fuyu": run_fuyu,
486 "phi3_v": run_phi3v,
487 "paligemma": run_paligemma,
488 + "paligemma2": run_paligemma2,
489 "chameleon": run_chameleon,
490 "minicpmv": run_minicpmv,
491 "blip-2": run_blip2,
vllm/model_executor/models/paligemma.py
+10 -1
@@ -105,6 +105,11 @@ def input_processor_for_paligemma(ctx: InputContext,
105 orig_prompt_ids.remove(hf_config.image_token_index)
106
107 new_prompt = f"{image_token_str_pad}{bos_token}{orig_prompt}\n"
108 +
109 + # The PaliGemma 2 tokenizer does not include a starting BOS token
110 + if orig_prompt_ids[0] != hf_config.bos_token_id:
111 + orig_prompt_ids = [hf_config.bos_token_id] + orig_prompt_ids
112 +
113 new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline
114
115 # NOTE: Create a defensive copy of the original inputs
@@ -149,7 +154,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
154 projection_dim=config.vision_config.projection_dim)
155
156 self.quant_config = quant_config
152 - config.text_config.architectures = ["GemmaForCausalLM"]
157 +
158 + if config.text_config.model_type == "gemma":
159 + config.text_config.architectures = ["GemmaForCausalLM"]
160 + else:
161 + config.text_config.architectures = ["Gemma2ForCausalLM"]
162 self.language_model = init_vllm_registered_model(
163 vllm_config=vllm_config,
164 hf_config=config.text_config,