A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
PaliGemma 2 support (#11142)
Jani Monoses committed
Dec 13, 2024 at 09:40 UTC
7cd7409142ff97aee1a13568753db9263fcf8f6b
3 files changed
+25
-3
docs/source/models/supported_models.rst
+2
-2
@@ -664,9 +664,9 @@ Text Generation (``--task generate``)
664
- ✅︎
665
- ✅︎
666
* - :code:`PaliGemmaForConditionalGeneration`
667
- - PaliGemma
667
+ - PaliGemma, PaliGemma 2
668
- T + I\ :sup:`E`
669
- - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, etc.
669
+ - :code:`google/paligemma-3b-pt-224`, :code:`google/paligemma-3b-mix-224`, :code:`google/paligemma2-3b-ft-docci-448`, etc.
670
-
671
- ✅︎
672
-
examples/offline_inference_vision_language.py
+13
@@ -137,6 +137,18 @@ def run_paligemma(question: str, modality: str):
137
return llm, prompt, stop_token_ids
138
139
140
+# PaliGemma 2
141
+def run_paligemma2(question: str, modality: str):
142
+ assert modality == "image"
143
+
144
+ # PaliGemma 2 has special prompt format for VQA
145
+ prompt = "caption en"
146
+ llm = LLM(model="google/paligemma2-3b-ft-docci-448",
147
+ mm_cache_preprocessor=args.mm_cache_preprocessor)
148
+ stop_token_ids = None
149
+ return llm, prompt, stop_token_ids
150
+
151
+
152
# Chameleon
153
def run_chameleon(question: str, modality: str):
154
assert modality == "image"
@@ -473,6 +485,7 @@ model_example_map = {
485
"fuyu": run_fuyu,
486
"phi3_v": run_phi3v,
487
"paligemma": run_paligemma,
488
+ "paligemma2": run_paligemma2,
489
"chameleon": run_chameleon,
490
"minicpmv": run_minicpmv,
491
"blip-2": run_blip2,
vllm/model_executor/models/paligemma.py
+10
-1
@@ -105,6 +105,11 @@ def input_processor_for_paligemma(ctx: InputContext,
105
orig_prompt_ids.remove(hf_config.image_token_index)
106
107
new_prompt = f"{image_token_str_pad}{bos_token}{orig_prompt}\n"
108
+
109
+ # The PaliGemma 2 tokenizer does not include a starting BOS token
110
+ if orig_prompt_ids[0] != hf_config.bos_token_id:
111
+ orig_prompt_ids = [hf_config.bos_token_id] + orig_prompt_ids
112
+
113
new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline
114
115
# NOTE: Create a defensive copy of the original inputs
@@ -149,7 +154,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
154
projection_dim=config.vision_config.projection_dim)
155
156
self.quant_config = quant_config
152
- config.text_config.architectures = ["GemmaForCausalLM"]
157
+
158
+ if config.text_config.model_type == "gemma":
159
+ config.text_config.architectures = ["GemmaForCausalLM"]
160
+ else:
161
+ config.text_config.architectures = ["Gemma2ForCausalLM"]
162
self.language_model = init_vllm_registered_model(
163
vllm_config=vllm_config,
164
hf_config=config.text_config,