@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 5dc36a4fa5

[Model] Remove Tarsier, Tarsier2 (#47143)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Jun 30, 2026 at 14:20 UTC 5dc36a4fa507740c94ad30c8dcde3d7620e766d3

12 files changed +2 -872

docs/models/supported_models.md

-2

@@ -626,8 +626,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen

626 | `Step3VLForConditionalGeneration` | Step3-VL | T + I+ | `stepfun-ai/step3` | | ✅︎ |

627 | `StepVLForConditionalGeneration` | Step3-VL-10B | T + I+ | `stepfun-ai/Step3-VL-10B` | | ✅︎ |

628 | `Step3p7ForConditionalGeneration` | Step-3.7-Flash | T + I+ | `stepfun-ai/Step-3.7-Flash` | | ✅︎ |

629 -| `TarsierForConditionalGeneration` | Tarsier | T + IE+ | `omni-search/Tarsier-7b`, `omni-search/Tarsier-34b` | | ✅︎ |

629 | `UltravoxModel` | Ultravox | T + AE+ | `fixie-ai/ultravox-v0_5-llama-3_2-1b` | ✅︎ | ✅︎ |

630 | `UnlimitedOCRForCausalLM` | Unlimited-OCR | T + I+ | `baidu/Unlimited-OCR`, etc. | ✅︎ | ✅︎ |

631

examples/generate/multimodal/vision_language_multi_image_offline.py

-51

@@ -1275,55 +1275,6 @@ def load_step_vl(question: str, image_urls: list[str]) -> ModelRequestData:

1275 )

1276

1277

1278 -def load_tarsier(question: str, image_urls: list[str]) -> ModelRequestData:

1279 - model_name = "omni-research/Tarsier-7b"

1280 -

1281 - engine_args = EngineArgs(

1282 - model=model_name,

1283 - trust_remote_code=True,

1284 - max_model_len=4096,

1285 - limit_mm_per_prompt={"image": len(image_urls)},

1286 - )

1287 -

1288 - prompt = f"USER: {'<image>' * len(image_urls)}\n{question}\n ASSISTANT:"

1289 - image_data = [fetch_image(url) for url in image_urls]

1290 -

1291 - return ModelRequestData(

1292 - engine_args=engine_args,

1293 - prompt=prompt,

1294 - image_data=image_data,

1295 - )

1296 -

1297 -

1298 -def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:

1299 - model_name = "omni-research/Tarsier2-Recap-7b"

1300 -

1301 - engine_args = EngineArgs(

1302 - model=model_name,

1303 - trust_remote_code=True,

1304 - max_model_len=32768,

1305 - limit_mm_per_prompt={"image": len(image_urls)},

1306 - hf_overrides={

1307 - "architectures": ["Tarsier2ForConditionalGeneration"],

1308 - "model_type": "tarsier2",

1309 - },

1310 - )

1311 -

1312 - prompt = (

1313 - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

1314 - f"<|im_start|>user\n<|vision_start|>{'<|image_pad|>' * len(image_urls)}"

1315 - f"<|vision_end|>{question}<|im_end|>\n"

1316 - "<|im_start|>assistant\n"

1317 - )

1318 - image_data = [fetch_image(url) for url in image_urls]

1319 -

1320 - return ModelRequestData(

1321 - engine_args=engine_args,

1322 - prompt=prompt,

1323 - image_data=image_data,

1324 - )

1325 -

1326 -

1278 # GLM-4.1V

1279 def load_glm4_1v(question: str, image_urls: list[str]) -> ModelRequestData:

1280 model_name = "zai-org/GLM-4.1V-9B-Thinking"

@@ -1507,8 +1458,6 @@ model_example_map = {

1458 "smolvlm": load_smolvlm,

1459 "step3": load_step3,

1460 "stepvl": load_step_vl,

1510 - "tarsier": load_tarsier,

1511 - "tarsier2": load_tarsier2,

1461 "glm4_1v": load_glm4_1v,

1462 "glm4_5v": load_glm4_5v,

1463 "glm4_5v_fp8": load_glm4_5v_fp8,

examples/generate/multimodal/vision_language_offline.py

-61

@@ -2347,65 +2347,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:

2347 )

2348

2349

2350 -# omni-research/Tarsier-7b

2351 -def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:

2352 - assert modality == "image"

2353 - model_name = "omni-research/Tarsier-7b"

2354 -

2355 - engine_args = EngineArgs(

2356 - model=model_name,

2357 - trust_remote_code=True,

2358 - max_model_len=4096,

2359 - limit_mm_per_prompt={modality: 1},

2360 - )

2361 - prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]

2362 -

2363 - return ModelRequestData(

2364 - engine_args=engine_args,

2365 - prompts=prompts,

2366 - )

2367 -

2368 -

2369 -def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:

2370 - model_name = "omni-research/Tarsier2-Recap-7b"

2371 -

2372 - mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}

2373 - engine_args = EngineArgs(

2374 - model=model_name,

2375 - max_model_len=4096,

2376 - hf_overrides={

2377 - "architectures": ["Tarsier2ForConditionalGeneration"],

2378 - "model_type": "tarsier2",

2379 - },

2380 - limit_mm_per_prompt=mm_limit,

2381 - )

2382 -

2383 - image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"

2384 - video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"

2385 -

2386 - if modality == "image":

2387 - placeholder = image_placeholder

2388 - elif modality == "video":

2389 - placeholder = video_placeholder

2390 - elif modality == "image+video":

2391 - placeholder = image_placeholder + video_placeholder

2392 -

2393 - prompts = [

2394 - (

2395 - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

2396 - f"<|im_start|>user\n{placeholder}"

2397 - f"{question}<|im_end|>\n"

2398 - "<|im_start|>assistant\n"

2399 - )

2400 - for question in questions

2401 - ]

2402 -

2403 - return ModelRequestData(

2404 - engine_args=engine_args,

2405 - prompts=prompts,

2406 - )

2407 -

2408 -

2350 model_example_map = {

2351 "aria": run_aria,

2352 "aya_vision": run_aya_vision,

@@ -2479,8 +2420,6 @@ model_example_map = {

2420 "smolvlm": run_smolvlm,

2421 "step3": run_step3,

2422 "stepvl": run_step_vl,

2482 - "tarsier": run_tarsier,

2483 - "tarsier2": run_tarsier2,

2423 }

2424

2425

tests/models/multimodal/generation/test_common.py

-25

@@ -993,31 +993,6 @@ VLM_TEST_SETTINGS = {

993 hf_output_post_proc=model_utils.smolvlm_trunc_hf_output,

994 num_logprobs=10,

995 ),

996 - "tarsier": VLMTestInfo(

997 - models=["omni-research/Tarsier-7b"],

998 - test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

999 - prompt_formatter=lambda img_prompt: f"USER: {img_prompt} ASSISTANT:",

1000 - max_model_len=4096,

1001 - max_num_seqs=2,

1002 - auto_cls=AutoModelForImageTextToText,

1003 - patch_hf_runner=model_utils.tarsier_patch_hf_runner,

1004 - ),

1005 - "tarsier2": VLMTestInfo(

1006 - models=["omni-research/Tarsier2-Recap-7b"],

1007 - test_type=(

1008 - VLMTestType.IMAGE,

1009 - VLMTestType.MULTI_IMAGE,

1010 - VLMTestType.VIDEO,

1011 - ),

1012 - prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501

1013 - img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",

1014 - video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",

1015 - max_model_len=4096,

1016 - max_num_seqs=2,

1017 - auto_cls=AutoModelForImageTextToText,

1018 - image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],

1019 - marks=[pytest.mark.skip("Model initialization hangs")],

1020 - ),

996 ### Tensor parallel / multi-gpu broadcast tests

997 "chameleon-broadcast": VLMTestInfo(

998 models=["facebook/chameleon-7b"],

tests/models/multimodal/generation/vlm_utils/model_utils.py

-12

@@ -1220,18 +1220,6 @@ def qwen3_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

1220 return hf_model

1221

1222

1223 -def tarsier_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

1224 - from vllm.model_executor.models.tarsier import get_vision_encoder_info

1225 -

1226 - vision_encoder_info = get_vision_encoder_info(hf_model.config)

1227 -

1228 - hf_processor = hf_model.processor

1229 - if hf_processor.patch_size is None:

1230 - hf_processor.patch_size = vision_encoder_info.get_patch_size()

1231 -

1232 - return hf_model

1233 -

1234 -

1223 def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":

1224 """Patch HfRunner for Voxtral's conversation-based processor.

1225

tests/models/registry.py

-16

@@ -1369,22 +1369,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {

1369 "fixie-ai/ultravox-v0_5-llama-3_2-1b",

1370 trust_remote_code=True,

1371 ),

1372 - "TarsierForConditionalGeneration": _HfExamplesInfo("omni-research/Tarsier-7b"),

1373 - "Tarsier2ForConditionalGeneration": _HfExamplesInfo(

1374 - "omni-research/Tarsier2-Recap-7b",

1375 - hf_overrides={

1376 - "architectures": ["Tarsier2ForConditionalGeneration"],

1377 - "model_type": "tarsier2",

1378 - },

1379 - max_transformers_version="5.3",

1380 - transformers_version_reason={

1381 - "vllm": (

1382 - "Qwen2VLConfig was split into Qwen2VLConfig + "

1383 - "Qwen2VLTextConfig in transformers v5, breaking "

1384 - "attribute access (num_attention_heads, hidden_size, etc.)"

1385 - )

1386 - },

1387 - ),

1372 "VoxtralForConditionalGeneration": _HfExamplesInfo(

1373 "mistralai/Voxtral-Mini-3B-2507",

1374 tokenizer_mode="mistral",

vllm/model_executor/models/qwen2_vl.py

-82

@@ -41,7 +41,6 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (

41 Qwen2VLVisionConfig,

42 )

43 from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize

44 -from transformers.models.qwen2_vl.video_processing_qwen2_vl import Qwen2VLVideoProcessor

45 from vllm.config import VllmConfig

46 from vllm.config.multimodal import BaseDummyOptions

@@ -86,7 +85,6 @@ from vllm.multimodal.processing import (

85 PromptUpdate,

86 )

87 from vllm.sequence import IntermediateTensors

89 -from vllm.tokenizers import TokenizerLike

88 from vllm.utils.tensor_schema import TensorSchema, TensorShape

89 from vllm.v1.attention.backends.registry import AttentionBackendEnum

90 from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers

@@ -1755,83 +1753,3 @@ class Qwen2VLForConditionalGeneration(

1753 vision_config = hf_config.vision_config

1754 merge_size = vision_config.spatial_merge_size

1755 return num_vision_tokens // merge_size**2

1758 -

1759 -

1760 -class Tarsier2MultiModalProcessor(Qwen2VLMultiModalProcessor):

1761 - pass

1762 -

1763 -

1764 -class Tarsier2ImageProcessor(Qwen2VLImageProcessor):

1765 - def __init__(

1766 - self,

1767 - size: dict[str, int] | None = None,

1768 - **kwargs,

1769 - ) -> None:

1770 - if size is not None and "min_pixels" in size and "max_pixels" in size:

1771 - # Remap if Tarsier2-specific format is provided

1772 - remapped_size = {

1773 - "shortest_edge": size["min_pixels"],

1774 - "longest_edge": size["max_pixels"],

1775 - }

1776 - super().__init__(size=remapped_size, **kwargs)

1777 - else:

1778 - super().__init__(size=size, **kwargs)

1779 -

1780 -

1781 -class Tarsier2Processor(Qwen2VLProcessor):

1782 - def __init__(

1783 - self,

1784 - image_processor: Tarsier2ImageProcessor,

1785 - tokenizer: TokenizerLike,

1786 - video_processor: Qwen2VLVideoProcessor,

1787 - **kwargs,

1788 - ):

1789 - super().__init__(

1790 - image_processor=image_processor,

1791 - tokenizer=tokenizer,

1792 - video_processor=video_processor,

1793 - chat_template=None,

1794 - **kwargs,

1795 - )

1796 -

1797 -

1798 -class Tarsier2ProcessingInfo(Qwen2VLProcessingInfo):

1799 - def get_hf_config(self) -> Qwen2VLConfig:

1800 - model_path = self.ctx.model_config.model

1801 - correct_config = Qwen2VLConfig.from_pretrained(model_path)

1802 -

1803 - return correct_config

1804 -

1805 - def get_hf_processor(self, **kwargs: object) -> Tarsier2Processor:

1806 - vision_config = self.ctx.get_hf_image_processor_config()

1807 - image_processor = Tarsier2ImageProcessor(**vision_config)

1808 - video_processor = Qwen2VLVideoProcessor(**vision_config)

1809 - return Tarsier2Processor(

1810 - image_processor=image_processor,

1811 - video_processor=video_processor,

1812 - tokenizer=self.get_tokenizer(),

1813 - **kwargs,

1814 - )

1815 -

1816 - def get_image_processor(self) -> Tarsier2ImageProcessor:

1817 - return Tarsier2ImageProcessor(**self.ctx.get_hf_image_processor_config())

1818 -

1819 -

1820 -@MULTIMODAL_REGISTRY.register_processor(

1821 - Tarsier2MultiModalProcessor,

1822 - info=Tarsier2ProcessingInfo,

1823 - dummy_inputs=Qwen2VLDummyInputsBuilder,

1824 -)

1825 -class Tarsier2ForConditionalGeneration(Qwen2VLForConditionalGeneration):

1826 - hf_to_vllm_mapper = WeightsMapper(

1827 - orig_to_new_prefix={

1828 - "vision_tower.": "visual.",

1829 - }

1830 - )

1831 -

1832 - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

1833 - skip_prefixes = []

1834 - if self.visual is None:

1835 - skip_prefixes.extend(["visual."])

1836 - loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)

1837 - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

vllm/model_executor/models/registry.py

+2 -5

@@ -571,11 +571,6 @@ _MULTIMODAL_MODELS = {

571 "StepVLForConditionalGeneration": ("step_vl", "StepVLForConditionalGeneration"),

572 "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"),

573 "Step3p7ForConditionalGeneration": ("step3p7", "Step3p7ForConditionalGeneration"),

574 - "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"),

575 - "Tarsier2ForConditionalGeneration": (

576 - "qwen2_vl",

577 - "Tarsier2ForConditionalGeneration",

578 - ),

574 "UltravoxModel": ("ultravox", "UltravoxModel"),

575 "VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"),

576 "VoxtralRealtimeGeneration": ("voxtral_realtime", "VoxtralRealtimeGeneration"),

@@ -736,6 +731,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {

731 "AquilaForCausalLM": "0.24.0",

732 "Grok1ModelForCausalLM": "0.24.0",

733 "Grok1ForCausalLM": "0.24.0",

734 + "TarsierForConditionalGeneration": "0.24.0",

735 + "Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4

736 }

737

738 _OOT_SUPPORTED_MODELS = {

vllm/model_executor/models/tarsier.py deleted

-591

@@ -1,591 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import math

5 -from collections.abc import Iterable, Mapping, Sequence

6 -from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar

7 -

8 -import torch

9 -import torch.nn as nn

10 -from transformers import (

11 - BatchFeature,

12 - CLIPVisionConfig,

13 - PretrainedConfig,

14 - SiglipVisionConfig,

15 -)

16 -from transformers import LlavaConfig as HfLlavaConfig

17 -from transformers.image_utils import ImageInput, get_image_size, to_numpy_array

18 -from transformers.models.llava import LlavaProcessor

19 -from transformers.processing_utils import ProcessingKwargs, Unpack

20 -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

21 -

22 -from vllm.config import VllmConfig

23 -from vllm.model_executor.layers.activation import get_act_fn

24 -from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear

25 -from vllm.model_executor.layers.quantization import QuantizationConfig

26 -from vllm.model_executor.models.llava import LlavaDummyInputsBuilder

27 -from vllm.multimodal import MULTIMODAL_REGISTRY

28 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems

29 -from vllm.multimodal.parse import (

30 - ImageEmbeddingItems,

31 - ImageProcessorItems,

32 - ImageSize,

33 - MultiModalDataItems,

34 -)

35 -from vllm.multimodal.processing import (

36 - BaseMultiModalProcessor,

37 - BaseProcessingInfo,

38 - PromptReplacement,

39 - PromptUpdate,

40 -)

41 -from vllm.sequence import IntermediateTensors

42 -from vllm.utils.tensor_schema import TensorSchema, TensorShape

43 -

44 -from .clip import CLIPVisionModel

45 -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

46 -from .siglip import SiglipVisionModel

47 -from .utils import (

48 - AutoWeightsLoader,

49 - get_layer_index,

50 - init_vllm_registered_model,

51 - maybe_prefix,

52 -)

53 -from .vision import (

54 - VisionEncoderInfo,

55 - get_num_selected_vision_tokens,

56 - get_vision_encoder_info,

57 -)

58 -

59 -

60 -class TarsierImagePixelInputs(TensorSchema):

61 - """

62 - Dimensions:

63 - - bn: Batch size * number of images

64 - - c: Number of channels (3)

65 - - h: Height

66 - - w: Width

67 - """

68 -

69 - type: Literal["pixel_values"] = "pixel_values"

70 - pixel_values: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]

71 -

72 -

73 -class TarsierImageEmbeddingInputs(TensorSchema):

74 - """

75 - Dimensions:

76 - - bn: Batch size * number of images

77 - - ifs: Image feature size

78 - - hs: Hidden size (must match the hidden size of language model

79 - backbone)

80 - """

81 -

82 - type: Literal["image_embeds"] = "image_embeds"

83 - data: Annotated[torch.Tensor, TensorShape("bn", "ifs", "hs")]

84 -

85 -

86 -TarsierImageInputs: TypeAlias = TarsierImagePixelInputs | TarsierImageEmbeddingInputs

87 -

88 -

89 -class TarsierHfConfig(Protocol): # Based on the Tarsier's LlavaConfig

90 - vision_config: Final[PretrainedConfig]

91 - text_config: Final[PretrainedConfig] # Added from Tarsier's LlavaConfig

92 - image_token_index: Final[int]

93 - vision_feature_select_strategy: Final[str]

94 - vision_feature_layer: Final[int | list[int]]

95 - projector_hidden_act: Final[str]

96 - image_newline_idx: Final[int]

97 - image_new_idx: Final[int]

98 - multimodal_projector_bias: bool = True

99 -

100 -

101 -class TarsierProcessorKwargs(ProcessingKwargs, total=False):

102 - _defaults = {

103 - "text_kwargs": {

104 - "padding": False,

105 - },

106 - "images_kwargs": {},

107 - }

108 -

109 -

110 -class TarsierProcessor(LlavaProcessor):

111 - def __call__(

112 - self,

113 - images: ImageInput = None,

114 - text: TextInput

115 - | PreTokenizedInput

116 - | list[TextInput]

117 - | list[PreTokenizedInput] = None,

118 - audio=None,

119 - videos=None,

120 - **kwargs: Unpack[TarsierProcessorKwargs],

121 - ) -> BatchFeature:

122 - if images is None and text is None:

123 - raise ValueError("You have to specify at least one of `images` or `text`.")

124 -

125 - output_kwargs = self._merge_kwargs(

126 - TarsierProcessorKwargs,

127 - tokenizer_init_kwargs=self.tokenizer.init_kwargs,

128 - **kwargs,

129 - )

130 - if images is not None:

131 - image_inputs = self.image_processor(

132 - images, **output_kwargs["images_kwargs"]

133 - )

134 - else:

135 - image_inputs = {}

136 -

137 - if isinstance(text, str):

138 - text = [text]

139 - elif not isinstance(text, list) and not isinstance(text[0], str):

140 - raise ValueError(

141 - "Invalid input text. Please provide a string, or a list of strings"

142 - )

143 -

144 - # try to expand inputs in processing if we have the necessary parts

145 - prompt_strings = text

146 - if image_inputs.get("pixel_values") is not None:

147 - # Replace the image token with the expanded image token sequence

148 - pixel_values = image_inputs["pixel_values"]

149 - height, width = get_image_size(to_numpy_array(pixel_values[0]))

150 - num_image_tokens = (

151 - (height // self.patch_size) * (width // self.patch_size + 1)

152 - + self.num_additional_image_tokens

153 - + 1

154 - )

155 - if self.vision_feature_select_strategy == "default":

156 - num_image_tokens -= 1

157 -

158 - prompt_strings = []

159 - for sample in text:

160 - sample = sample.replace(

161 - self.image_token, self.image_token * num_image_tokens

162 - )

163 - prompt_strings.append(sample)

164 -

165 - return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)

166 - text_inputs = self.tokenizer(prompt_strings, **output_kwargs["text_kwargs"])

167 - return BatchFeature(

168 - data={**text_inputs, **image_inputs}, tensor_type=return_tensors

169 - )

170 -

171 -

172 -class TarsierMultiModalProjector(nn.Module):

173 - def __init__(

174 - self,

175 - vision_hidden_size: int,

176 - text_hidden_size: int,

177 - projector_hidden_act: str,

178 - multimodal_projector_bias: bool,

179 - quant_config: QuantizationConfig | None = None,

180 - prefix: str = "",

181 - ):

182 - super().__init__()

183 -

184 - self.linear_1 = ColumnParallelLinear(

185 - vision_hidden_size,

186 - text_hidden_size,

187 - bias=multimodal_projector_bias,

188 - quant_config=quant_config,

189 - prefix=f"{prefix}.linear_1",

190 - )

191 - self.act = get_act_fn(projector_hidden_act)

192 - self.linear_2 = RowParallelLinear(

193 - text_hidden_size,

194 - text_hidden_size,

195 - bias=multimodal_projector_bias,

196 - quant_config=quant_config,

197 - prefix=f"{prefix}.linear_2",

198 - )

199 -

200 - def forward(self, image_features: torch.Tensor) -> torch.Tensor:

201 - hidden_states, _ = self.linear_1(image_features)

202 - hidden_states = self.act(hidden_states)

203 - hidden_states, _ = self.linear_2(hidden_states)

204 - return hidden_states

205 -

206 -

207 -class TarsierProcessingInfo(BaseProcessingInfo):

208 - def get_hf_config(self) -> TarsierHfConfig:

209 - return self.ctx.get_hf_config(HfLlavaConfig)

210 -

211 - def get_vision_encoder_info(self) -> VisionEncoderInfo:

212 - return get_vision_encoder_info(self.get_hf_config())

213 -

214 - def get_hf_processor(self, **kwargs: object) -> TarsierProcessor:

215 - vision_info = self.get_vision_encoder_info()

216 -

217 - kwargs.setdefault("patch_size", vision_info.get_patch_size())

218 -

219 - return self.ctx.get_hf_processor(TarsierProcessor, **kwargs)

220 -

221 - def get_supported_mm_limits(self) -> Mapping[str, int | None]:

222 - return {"image": None}

223 -

224 - def get_num_image_tokens(

225 - self,

226 - *,

227 - image_width: int,

228 - image_height: int,

229 - ) -> int:

230 - hf_config = self.get_hf_config()

231 - vision_encoder_info = self.get_vision_encoder_info()

232 - num_projected_patches = get_num_selected_vision_tokens(

233 - vision_encoder_info.get_num_image_tokens(

234 - image_width=image_width,

235 - image_height=image_height,

236 - ),

237 - hf_config.vision_feature_select_strategy,

238 - )

239 - if num_projected_patches <= 0:

240 - default_size = self.get_image_size_with_most_features()

241 - num_projected_patches_default = get_num_selected_vision_tokens(

242 - vision_encoder_info.get_num_image_tokens(

243 - image_width=default_size.width,

244 - image_height=default_size.height,

245 - ),

246 - hf_config.vision_feature_select_strategy,

247 - )

248 - if num_projected_patches_default <= 0:

249 - raise ValueError("Could not determine a valid number of image patches.")

250 - num_projected_patches = num_projected_patches_default

251 - num_height_patches = int(math.sqrt(num_projected_patches))

252 - total_image_tokens_for_llm = num_projected_patches + num_height_patches + 1

253 - return total_image_tokens_for_llm

254 -

255 - def get_image_size_with_most_features(self) -> ImageSize:

256 - vision_encoder_info = self.get_vision_encoder_info()

257 - width = height = vision_encoder_info.get_image_size()

258 - return ImageSize(width=width, height=height)

259 -

260 - def get_max_image_tokens(self) -> int:

261 - target_width, target_height = self.get_image_size_with_most_features()

262 - return self.get_num_image_tokens(

263 - image_width=target_width,

264 - image_height=target_height,

265 - )

266 -

267 - def get_image_newline_idx(self) -> int:

268 - return self.get_hf_config().image_newline_idx

269 -

270 - def get_image_new_idx(self) -> int:

271 - return self.get_hf_config().image_new_idx

272 -

273 -

274 -_I_Tarsier = TypeVar("_I_Tarsier", bound=TarsierProcessingInfo)

275 -

276 -

277 -class TarsierDummyInputsBuilder(LlavaDummyInputsBuilder[_I_Tarsier]):

278 - pass

279 -

280 -

281 -class TarsierMultiModalProcessor(BaseMultiModalProcessor[_I_Tarsier]):

282 - def _get_mm_fields_config(

283 - self,

284 - hf_inputs: BatchFeature,

285 - hf_processor_mm_kwargs: Mapping[str, object],

286 - ) -> Mapping[str, MultiModalFieldConfig]:

287 - return dict(

288 - pixel_values=MultiModalFieldConfig.batched("image"),

289 - image_embeds=MultiModalFieldConfig.batched("image"),

290 - )

291 -

292 - def _get_prompt_updates(

293 - self,

294 - mm_items: MultiModalDataItems,

295 - hf_processor_mm_kwargs: Mapping[str, object],

296 - out_mm_kwargs: MultiModalKwargsItems,

297 - ) -> Sequence[PromptUpdate]:

298 - hf_config = self.info.get_hf_config()

299 - image_token_id = hf_config.image_token_index # The <IMAGE> token ID

300 -

301 - def get_replacement(item_idx: int):

302 - images = mm_items.get_items(

303 - "image", (ImageEmbeddingItems, ImageProcessorItems)

304 - )

305 -

306 - if isinstance(images, ImageEmbeddingItems):

307 - num_projected_patches = images.get_feature_size(item_idx)

308 - # This assumes num_projected_patches is a perfect square

309 - num_height_patches = int(math.sqrt(num_projected_patches))

310 - num_final_image_tokens = num_projected_patches + num_height_patches + 1

311 - else:

312 - image_size = images.get_image_size(item_idx)

313 - num_final_image_tokens = self.info.get_num_image_tokens(

314 - image_width=image_size.width,

315 - image_height=image_size.height,

316 - )

317 -

318 - return [image_token_id] * num_final_image_tokens

319 -

320 - return [

321 - PromptReplacement(

322 - modality="image",

323 - target=[image_token_id], # Replace each single <IMAGE> token

324 - replacement=get_replacement,

325 - ),

326 - ]

327 -

328 -

329 -def init_vision_tower_for_tarsier(

330 - hf_config: TarsierHfConfig, # Use the Tarsier specific config protocol

331 - quant_config: QuantizationConfig | None,

332 - *,

333 - require_post_norm: bool | None = None,

334 - prefix: str = "",

335 -) -> CLIPVisionModel | SiglipVisionModel:

336 - vision_config = hf_config.vision_config

337 -

338 - feature_layers = hf_config.vision_feature_layer

339 - base_num_hidden_layers = vision_config.num_hidden_layers

340 -

341 - if isinstance(feature_layers, int):

342 - num_hidden_layers_to_init = get_layer_index(

343 - feature_layers, base_num_hidden_layers

344 - )

345 - elif isinstance(feature_layers, (list, tuple)):

346 - num_hidden_layers_to_init = max(

347 - get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers

348 - )

349 - else:

350 - raise TypeError(

351 - f"vision_layer_feature type: {type(feature_layers)} is not supported"

352 - )

353 -

354 - if isinstance(vision_config, CLIPVisionConfig):

355 - return CLIPVisionModel(

356 - vision_config,

357 - quant_config=quant_config,

358 - num_hidden_layers_override=num_hidden_layers_to_init,

359 - require_post_norm=require_post_norm,

360 - prefix=prefix,

361 - )

362 - elif isinstance(vision_config, SiglipVisionConfig):

363 - return SiglipVisionModel(

364 - vision_config,

365 - quant_config=quant_config,

366 - num_hidden_layers_override=num_hidden_layers_to_init,

367 - require_post_norm=require_post_norm,

368 - prefix=prefix,

369 - )

370 -

371 - msg = f"Unsupported vision config for Tarsier: {type(vision_config)}"

372 - raise NotImplementedError(msg)

373 -

374 -

375 -@MULTIMODAL_REGISTRY.register_processor(

376 - TarsierMultiModalProcessor,

377 - info=TarsierProcessingInfo,

378 - dummy_inputs=TarsierDummyInputsBuilder,

379 -)

380 -class TarsierForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

381 - packed_modules_mapping = {

382 - "qkv_proj": ["q_proj", "k_proj", "v_proj"],

383 - "gate_up_proj": ["gate_proj", "up_proj"],

384 - }

385 -

386 - @classmethod

387 - def get_placeholder_str(cls, modality: str, i: int) -> str | None:

388 - if modality.startswith("image"):

389 - return "<image>"

390 -

391 - raise ValueError("Only image modality is supported")

392 -

393 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:

394 - super().__init__()

395 -

396 - config: TarsierHfConfig = vllm_config.model_config.hf_config

397 - quant_config = vllm_config.quant_config

398 -

399 - self.config = config # Storing the Tarsier-specific HF config

400 -

401 - with self._mark_tower_model(vllm_config, "image"):

402 - self.vision_tower = init_vision_tower_for_tarsier(

403 - config,

404 - quant_config=quant_config,

405 - require_post_norm=False,

406 - prefix=maybe_prefix(prefix, "vision_tower"),

407 - )

408 - projector_bias = getattr(config, "multimodal_projector_bias", True)

409 -

410 - self.multi_modal_projector = TarsierMultiModalProjector(

411 - vision_hidden_size=config.vision_config.hidden_size,

412 - text_hidden_size=config.text_config.hidden_size,

413 - projector_hidden_act=config.projector_hidden_act,

414 - multimodal_projector_bias=projector_bias,

415 - quant_config=quant_config,

416 - prefix=maybe_prefix(prefix, "multi_modal_projector"),

417 - )

418 - self.register_buffer(

419 - "image_newline_idx_tensor",

420 - torch.tensor([config.image_newline_idx], dtype=torch.long),

421 - persistent=False,

422 - )

423 - self.register_buffer(

424 - "image_new_idx_tensor",

425 - torch.tensor([config.image_new_idx], dtype=torch.long),

426 - persistent=False,

427 - )

428 -

429 - with self._mark_language_model(vllm_config):

430 - self.language_model = init_vllm_registered_model(

431 - vllm_config=vllm_config,

432 - # Use text_config from Tarsier's main config

433 - hf_config=config.text_config,

434 - prefix=maybe_prefix(prefix, "language_model"),

435 - )

436 -

437 - self.make_empty_intermediate_tensors = (

438 - self.language_model.make_empty_intermediate_tensors

439 - )

440 -

441 - def _parse_and_validate_image_input(

442 - self, **kwargs: object

443 - ) -> TarsierImageInputs | None:

444 - pixel_values = kwargs.pop("pixel_values", None)

445 - image_embeds = kwargs.pop("image_embeds", None)

446 -

447 - if pixel_values is None and image_embeds is None:

448 - return None

449 -

450 - if pixel_values is not None:

451 - return TarsierImagePixelInputs(

452 - type="pixel_values",

453 - pixel_values=pixel_values,

454 - )

455 -

456 - if image_embeds is not None:

457 - return TarsierImageEmbeddingInputs(

458 - type="image_embeds",

459 - data=image_embeds,

460 - )

461 -

462 - raise AssertionError("This line should be unreachable.")

463 -

464 - def _image_pixels_to_features(

465 - self,

466 - vision_tower: CLIPVisionModel | SiglipVisionModel,

467 - pixel_values: torch.Tensor | list[torch.Tensor],

468 - ) -> torch.Tensor | tuple[torch.Tensor, ...]:

469 - # From vLLM LLaVA, vision tower output handling

470 - return vision_tower(

471 - pixel_values,

472 - feature_select_strategy=self.config.vision_feature_select_strategy,

473 - )

474 -

475 - def _add_tarsier_split_tokens(

476 - self, projected_image_features: torch.Tensor

477 - ) -> torch.Tensor:

478 - """

479 - Implements Tarsier's `add_split_tokens` logic.

480 - """

481 - num_images, num_projected_patches, embed_dim = projected_image_features.shape

482 - num_height_patches = int(math.sqrt(num_projected_patches))

483 - num_width_patches = num_projected_patches // num_height_patches

484 - device = projected_image_features.device

485 - embedding_layer = self.language_model.model.embed_tokens

486 - image_newline_emb = embedding_layer(

487 - self.image_newline_idx_tensor.to(device)

488 - ).squeeze(0)

489 - image_new_emb = embedding_layer(self.image_new_idx_tensor.to(device)).squeeze(0)

490 - try:

491 - current_image_features_grid = projected_image_features.view(

492 - num_images, num_height_patches, num_width_patches, embed_dim

493 - )

494 - except RuntimeError as e:

495 - raise RuntimeError(

496 - "Cannot reshape projected_image_features"

497 - f" with shape {projected_image_features.shape} "

498 - f"to ({num_images}, {num_height_patches},"

499 - f" {num_width_patches}, {embed_dim}). "

500 - "Ensure num_projected_patches is compatible"

501 - " with a grid structure. "

502 - f"num_projected_patches={num_projected_patches}, "

503 - f"derived num_height_patches={num_height_patches}. "

504 - ) from e

505 -

506 - image_newline_expanded = image_newline_emb.expand(

507 - (num_images, num_height_patches, 1, embed_dim)

508 - )

509 - features_with_newlines = torch.cat(

510 - [current_image_features_grid, image_newline_expanded],

511 - dim=2, # Concatenate along width dim

512 - )

513 - new_num_patches_after_newline = num_projected_patches + num_height_patches

514 - features_with_newlines_flat = features_with_newlines.view(

515 - num_images, new_num_patches_after_newline, embed_dim

516 - )

517 - image_new_expanded = image_new_emb.expand((num_images, 1, embed_dim))

518 - final_image_features = torch.cat(

519 - [features_with_newlines_flat, image_new_expanded],

520 - dim=1, # Concatenate along patch sequence dim

521 - )

522 - return final_image_features

523 -

524 - def _process_image_pixels(

525 - self,

526 - inputs: TarsierImagePixelInputs,

527 - ) -> torch.Tensor | tuple[torch.Tensor, ...]:

528 - pixel_values = inputs["pixel_values"]

529 - image_features_selected = self._image_pixels_to_features(

530 - self.vision_tower, pixel_values

531 - ) # type: ignore

532 - if isinstance(image_features_selected, torch.Tensor):

533 - projected_features = self.multi_modal_projector(image_features_selected)

534 - final_features = self._add_tarsier_split_tokens(projected_features)

535 - return final_features

536 - else:

537 - raise TypeError(

538 - f"_image_pixels_to_features type:"

539 - f" {type(image_features_selected)} is not supported"

540 - )

541 -

542 - def _process_image_input(

543 - self,

544 - image_input: TarsierImageInputs,

545 - ) -> torch.Tensor | tuple[torch.Tensor, ...]:

546 - if image_input["type"] == "image_embeds":

547 - projected_features = image_input["data"]

548 - if isinstance(projected_features, torch.Tensor):

549 - return self._add_tarsier_split_tokens(projected_features)

550 - else:

551 - raise ValueError(

552 - "Incorrect type of image_embeds. "

553 - f"Got type: {type(projected_features)}. "

554 - )

555 -

556 - return self._process_image_pixels(image_input)

557 -

558 - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:

559 - image_input = self._parse_and_validate_image_input(**kwargs)

560 - if image_input is None:

561 - return []

562 - return self._process_image_input(image_input)

563 -

564 - def forward(

565 - self,

566 - input_ids: torch.Tensor | None,

567 - positions: torch.Tensor,

568 - intermediate_tensors: IntermediateTensors | None = None,

569 - inputs_embeds: torch.Tensor | None = None,

570 - **kwargs: object,

571 - ) -> torch.Tensor | IntermediateTensors:

572 - if intermediate_tensors is not None:

573 - inputs_embeds = None

574 -

575 - hidden_states = self.language_model.model(

576 - input_ids=input_ids,

577 - positions=positions,

578 - intermediate_tensors=intermediate_tensors,

579 - inputs_embeds=inputs_embeds,

580 - )

581 - return hidden_states

582 -

583 - def compute_logits(

584 - self,

585 - hidden_states: torch.Tensor,

586 - ) -> torch.Tensor | None:

587 - return self.language_model.compute_logits(hidden_states)

588 -

589 - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:

590 - loader = AutoWeightsLoader(self)

591 - return loader.load_weights(weights)

vllm/transformers_utils/config.py

-1

@@ -123,7 +123,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(

123 qwen3_5_moe="Qwen3_5MoeConfig",

124 laguna="LagunaConfig",

125 lfm2_moe="Lfm2MoeConfig",

126 - tarsier2="Tarsier2Config",

126 **{"unlimited-ocr": "UnlimitedOCRConfig"},

127 )

128

vllm/transformers_utils/configs/__init__.py

-2

@@ -86,7 +86,6 @@ _CLASS_TO_MODULE: dict[str, str] = {

86 "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",

87 "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",

88 "Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",

89 - "Tarsier2Config": "vllm.transformers_utils.configs.tarsier2",

89 # Special case: DeepseekV3Config is from HuggingFace Transformers

90 "DeepseekV3Config": "transformers",

91 }

@@ -161,7 +160,6 @@ __all__ = [

160 "Qwen3_5TextConfig",

161 "Qwen3_5MoeConfig",

162 "Qwen3_5MoeTextConfig",

164 - "Tarsier2Config",

163 ]

164

165

vllm/transformers_utils/configs/tarsier2.py deleted

-24

@@ -1,24 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from transformers import Qwen2VLConfig

4 -

5 -

6 -class Tarsier2Config(Qwen2VLConfig):

7 - """

8 - Tarsier2's config.json is written such that AutoConfig.from_pretrained will create

9 - a deeply nested config consisting of:

10 -

11 - - LlavaConfig

12 - - Qwen2VLConfig

13 - - Qwen2VLTextConfig

14 - - Qwen2VLVisionConfig

15 - - Qwen2VLConfig

16 - - Qwen2VLTextConfig

17 - - Qwen2VLVisionConfig

18 -

19 - When it should really just be a single Qwen2VLConfig.

20 -

21 - This class is a hack to stop AutoConfig from creating the nested config structure.

22 - """

23 -

24 - model_type = "tarsier2"