@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 88c8304104

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 12:15 UTC

[Model] Refactor Ovis2 to support original tokenizer (#17537)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed May 2, 2025 at 02:00 UTC 88c830410468fa3549638cf78d6c354b13046921

5 files changed +102 -48

examples/offline_inference/vision_language.py

-2

@@ -730,11 +730,9 @@ def run_ovis2(questions: list[str], modality: str) -> ModelRequestData:

730 assert modality == "image"

731

732 model_name = "AIDC-AI/Ovis2-1B"

733 - tokenizer = "Isotr0py/Ovis2-tokenizer"

733

734 engine_args = EngineArgs(

735 model=model_name,

737 - tokenizer=tokenizer,

736 max_model_len=4096,

737 max_num_seqs=2,

738 trust_remote_code=True,

examples/offline_inference/vision_language_multi_image.py

-2

@@ -439,11 +439,9 @@ def load_nvlm_d(question: str, image_urls: list[str]) -> ModelRequestData:

439 # Ovis2

440 def load_ovis2(question: str, image_urls: list[str]) -> ModelRequestData:

441 model_name = "AIDC-AI/Ovis2-1B"

442 - tokenizer = "Isotr0py/Ovis2-tokenizer"

442

443 engine_args = EngineArgs(

444 model=model_name,

446 - tokenizer=tokenizer,

445 max_model_len=8192,

446 max_num_seqs=2,

447 trust_remote_code=True,

tests/models/registry.py

-1

@@ -349,7 +349,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {

349 transformers_version_reason="Use of deprecated imports which have been removed.", # noqa: E501

350 extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"}), # noqa: E501

351 "Ovis2ForConditionalGeneration": _HfExamplesInfo("AIDC-AI/Ovis2-1B",

352 - tokenizer="Isotr0py/Ovis2-tokenizer",

352 trust_remote_code=True,

353 hf_overrides={"architectures": ["Ovis2ForConditionalGeneration"]}), # noqa: E501

354 "Phi4MMForCausalLM": _HfExamplesInfo("microsoft/Phi-4-multimodal-instruct",

vllm/model_executor/models/ovis2.py

+66 -9

@@ -46,8 +46,7 @@ from .utils import merge_multimodal_embeddings

47 # Cannot find the following number from hf config.

48 IMAGE_TOKEN = "<image>"

49 -IMAGE_ATOM_TOKEN_ID = 151666

50 -IMAGE_PAD_TOKEN_ID = 151672

49 +IMAGE_PAD_TOKEN_ID = 151655

50 NUMBER_OF_TOKEN_TO_RESERVE_FOR_SEGMENT = 256

@@ -59,6 +58,12 @@ class Ovis2ImagePatchInputs(TypedDict):

58 `(batch_size * num_patches, patch_size_x * patch_size_y * num_channels)`

59 """

61 + inducator_tokens: torch.Tensor

62 + """

63 + Shape:

64 + `(batch_size * (num_patches + 1))`

65 + """

66 +

67 patches_per_image: List[int]

68 """

69 List of number of total patches for each image in the batch.

@@ -138,6 +143,21 @@ class Ovis2DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2ProcessingInfo]):

143

144 class Ovis2MultiModalProcessor(BaseMultiModalProcessor[Ovis2ProcessingInfo]):

145

146 + def image_indicators_to_visual_tokens(

147 + self,

148 + image_indicators: list[int],

149 + ) -> list[int]:

150 + """

151 + Filter image indicators placeholders and convert them to corresponding

152 + tokens in visual tokenizer.

153 + For example, [-301, -300, -302, -300, -303, -300, -304, -300, -305]

154 + should return [vocab_size-1, vocab_size-2, ..., vocab_size-5]

155 + """

156 + hf_config = self.info.get_hf_config()

157 + vte_vocab_size = hf_config.visual_tokenizer_config.vocab_size

158 + # -300 is image_atom token, filter them out

159 + return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]

160 +

161 def _call_hf_processor(

162 self,

163 prompt: str,

@@ -156,6 +176,16 @@ class Ovis2MultiModalProcessor(BaseMultiModalProcessor[Ovis2ProcessingInfo]):

176 mm_kwargs=mm_kwargs,

177 )

178

179 + hf_processor = self.info.get_hf_processor()

180 + image_indicators = [

181 + hf_processor.construct_image_indicators(grid)

182 + for grid in processed_outputs["grids"]

183 + ]

184 + indicator_tokens = [

185 + self.image_indicators_to_visual_tokens(indicator)

186 + for indicator in image_indicators

187 + ]

188 + processed_outputs["indicator_tokens"] = indicator_tokens

189 return processed_outputs

190

191 def _apply_hf_processor_tokens_only(

@@ -171,7 +201,8 @@ class Ovis2MultiModalProcessor(BaseMultiModalProcessor[Ovis2ProcessingInfo]):

201 hf_processor_mm_kwargs: Mapping[str, object],

202 ) -> Mapping[str, MultiModalFieldConfig]:

203 return dict(pixel_values=MultiModalFieldConfig.batched("image"),

174 - grids=MultiModalFieldConfig.batched("image"))

204 + grids=MultiModalFieldConfig.batched("image"),

205 + indicator_tokens=MultiModalFieldConfig.batched("image"))

206

207 def _get_prompt_updates(

208 self,

@@ -230,20 +261,28 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):

261 def _parse_and_validate_image_input(

262 self, **kwargs: object) -> Optional[Ovis2ImagePatchInputs]:

263 pixel_values = kwargs.pop("pixel_values", None)

233 - if pixel_values is None:

264 + indicator_tokens = kwargs.pop("indicator_tokens", None)

265 +

266 + if pixel_values is None and indicator_tokens is None:

267 return None

268

236 - if pixel_values is not None:

269 + if pixel_values is not None and indicator_tokens is not None:

270 if not isinstance(pixel_values, (torch.Tensor, list)):

271 raise ValueError("Incorrect type of pixel values. "

272 f"Got type: {type(pixel_values)}")

273

274 + if not isinstance(indicator_tokens, (torch.Tensor, list)):

275 + raise ValueError("Incorrect type of indicator_tokens. "

276 + f"Got type: {type(pixel_values)}")

277 +

278 return Ovis2ImagePatchInputs(

279 type="image_patches",

280 flat_data=flatten_bn(flatten_bn(pixel_values), concat=True),

281 patches_per_image=[

282 x.shape[0] for x in flatten_bn(pixel_values)

283 ],

284 + indicator_tokens=flatten_bn(flatten_bn(indicator_tokens),

285 + concat=True),

286 )

287

288 raise AssertionError("This line should be unreachable.")

@@ -252,15 +291,33 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):

291 self, image_input: Ovis2ImagePatchInputs) -> MultiModalEmbeddings:

292 image_patches_flat = image_input["flat_data"]

293 patches_per_image = image_input["patches_per_image"]

294 + indicator_tokens = image_input["indicator_tokens"]

295 +

296 + indicator_per_image = list(

297 + map(lambda x: x + 1 if x > 1 else x + 2, patches_per_image))

298

299 target_dtype = self.visual_tokenizer.dtype

300 visual_tokens = self.visual_tokenizer(

301 image_patches_flat.to(target_dtype))

302 visual_embeds = self.vte(visual_tokens) # 1:1 numeric eq.

303

261 - return tuple(

262 - x.flatten(0, 1)

263 - for x in visual_embeds.split(patches_per_image, dim=0))

304 + indicator_embeds = self.vte(indicator_tokens)

305 + indicator_embeds_per_image = indicator_embeds.split(

306 + indicator_per_image)

307 +

308 + visual_embeds_per_image = visual_embeds.split(patches_per_image, dim=0)

309 + vision_embeddings = []

310 + for indicator, visual in zip(indicator_embeds_per_image,

311 + visual_embeds_per_image):

312 + vision_embeddings_per_image = []

313 + for i in range(visual.shape[0]):

314 + vision_embeddings_per_image.append(

315 + torch.cat([indicator[i:i + 1], visual[i]], dim=0))

316 + vision_embeddings_per_image.append(indicator[i + 1:])

317 + vision_embeddings.append(

318 + torch.cat(vision_embeddings_per_image, dim=0))

319 +

320 + return tuple(vision_embeddings)

321

322 def get_multimodal_embeddings(

323 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

@@ -281,7 +338,7 @@ class Ovis2ForConditionalGeneration(nn.Module, SupportsMultiModal):

338 if multimodal_embeddings is not None:

339 inputs_embeds = merge_multimodal_embeddings(

340 input_ids, inputs_embeds, multimodal_embeddings,

284 - [IMAGE_ATOM_TOKEN_ID, IMAGE_PAD_TOKEN_ID])

341 + [IMAGE_PAD_TOKEN_ID])

342 return inputs_embeds

343

344 def forward(

vllm/transformers_utils/processors/ovis2.py

+36 -34

@@ -69,20 +69,21 @@ class OvisProcessor(ProcessorMixin):

69 image_processor_class = "AutoImageProcessor"

70 tokenizer_class = "Qwen2Tokenizer"

72 - def __init__(self, image_processor=None, tokenizer=None, chat_template=None, **kwargs):

73 - self.image_token = "<|image_pad|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token

74 - self.video_token = "<|video_pad|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token

72 + def __init__(self, image_processor=None, tokenizer=None, chat_template=None, image_pad_token=None, **kwargs):

73 + self.image_token = "<image>"

74 + self.image_pad_token = "<|image_pad|>" if image_pad_token is None else image_pad_token

75 super().__init__(image_processor, tokenizer, chat_template=chat_template)

77 + self.image_pad_token_id = self.tokenizer.get_vocab()[self.image_pad_token]

78 self.extra_special_tokens = {

78 - "image_token": "<image>",

79 - "image_atom": "<image_atom>",

80 - "image_start": "<img>",

81 - "image_prefix": "<pre>",

82 - "image_col_sep": "<col>",

83 - "image_row_sep": "<row>",

84 - "image_end": "</img>",

85 - 'image_pad': '<image_pad>',

79 + "image_token": -200,

80 + "image_atom": -300,

81 + "image_start": -301,

82 + "image_prefix": -302,

83 + "image_col_sep": -303,

84 + "image_row_sep": -304,

85 + "image_end": -305,

86 + 'image_pad': self.image_pad_token_id,

87 }

89 def __call__(

@@ -157,58 +158,44 @@ class OvisProcessor(ProcessorMixin):

158 if not isinstance(text, list):

159 text = [text]

160

160 - tokenized_batched_text = self.tokenizer.batch_encode_plus(

161 - text,

162 - **output_kwargs["text_kwargs"]

163 - )

161 + tokenized_batched_text = self._tokenize_with_image_symbol(text)

162 image_token_id = self.get_token_value("image_token")

163 replaced_ids_list = []

166 - replaced_attn_mask_list = []

164 idx = 0

168 - for ids_tensor, attn_mask in zip(tokenized_batched_text['input_ids'],

169 - tokenized_batched_text['attention_mask']):

165 + for ids_tensor in tokenized_batched_text:

166 if image_token_id in ids_tensor and "image_placeholders" in image_features:

167 if idx < len(image_features["image_placeholders"]):

168 # Converts in list for ease of use

169 ids_list = ids_tensor.tolist()

174 - attn_list = attn_mask.tolist()

170

171 new_ids = []

177 - new_attn = []

172

173 # replace placeholders

174 for i, token_id in enumerate(ids_list):

175 if token_id == image_token_id:

176 placeholder_ids = image_features["image_placeholders"][idx]

177 new_ids.extend(placeholder_ids)

184 - new_attn.extend([1] * len(placeholder_ids))

178 idx += 1

179 else:

180 new_ids.append(token_id)

188 - new_attn.append(attn_list[i])

181

182 # Converts back to tensors

183 ids_tensor = torch.tensor(new_ids, dtype=torch.long)

192 - attn_mask = torch.tensor(new_attn, dtype=torch.long)

184 else:

185 raise RuntimeError(

186 'Mismatch between the images you provided and the number of placeholder present in the text')

187

188 replaced_ids_list.append(ids_tensor)

198 - replaced_attn_mask_list.append(attn_mask)

189

190 if replaced_ids_list:

191 replaced_and_tokenized_ids = torch.stack(replaced_ids_list)

202 - replaced_and_tokenized_attn_mask = torch.stack(replaced_attn_mask_list)

192 else:

193 replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long)

205 - replaced_and_tokenized_attn_mask = torch.tensor([], dtype=torch.long)

194

195 # Create the output with text features

196 output = BatchFeature(

197 data={

198 "input_ids": replaced_and_tokenized_ids,

211 - "attention_mask": replaced_and_tokenized_attn_mask,

199 }

200 )

201

@@ -219,10 +206,22 @@ class OvisProcessor(ProcessorMixin):

206

207 return output

208

222 -

209 # If only images were provided

210 return BatchFeature(data=image_features)

211

212 + def _tokenize_with_image_symbol(self, text_list: list[str]) -> torch.LongTensor:

213 + batch_token_ids = []

214 + for text in text_list:

215 + text_chunks = [self.tokenizer(chunk, add_special_tokens=False).input_ids for chunk in

216 + text.split(self.image_token)]

217 + token_ids = []

218 + num_chuck = len(text_chunks)

219 + for i, chunk in enumerate(text_chunks):

220 + token_ids.extend(chunk)

221 + if i < num_chuck - 1:

222 + token_ids.append(self.get_token_value("image_token"))

223 + batch_token_ids.append(token_ids)

224 + return torch.tensor(batch_token_ids, dtype=torch.long)

225

226 def get_image_size(self):

227 height = self.image_processor.crop_size["height"]

@@ -230,10 +229,9 @@ class OvisProcessor(ProcessorMixin):

229 return height, width

230

231 def get_token_value(self, tok):

233 - return self.tokenizer.get_vocab()[self.extra_special_tokens[tok]]

234 -

235 - def construct_image_placeholders(self, grid):

232 + return self.extra_special_tokens[tok]

233

234 + def construct_image_indicators(self, grid):

235 image_placeholders = [self.get_token_value('image_start'),

236 self.get_token_value('image_atom'),

237 self.get_token_value('image_prefix')]

@@ -246,7 +244,11 @@ class OvisProcessor(ProcessorMixin):

244 if r < grid[0] - 1:

245 image_placeholders.append(self.get_token_value('image_row_sep'))

246 image_placeholders.append(self.get_token_value('image_end'))

249 - # return image_placeholders

247 + return image_placeholders

248 +

249 + def construct_image_placeholders(self, grid):

250 +

251 + image_placeholders = self.construct_image_indicators(grid)

252

253 image_atom_token_id = self.get_token_value('image_atom')

254 # Extract the padding token ID from tokenizer

@@ -255,7 +257,7 @@ class OvisProcessor(ProcessorMixin):

257 # Create a new list with padding tokens inserted

258 padded_placeholder_tokens = []

259 for token in image_placeholders:

258 - padded_placeholder_tokens.append(token)

260 + padded_placeholder_tokens.append(image_padding_token_id)

261 if token == image_atom_token_id:

262 # Add 255 padding tokens after each image atom token

263 padded_placeholder_tokens.extend([image_padding_token_id] * 255)

@@ -394,4 +396,4 @@ class OvisProcessor(ProcessorMixin):

396 return names_from_processor + ["second_per_grid_ts"]

397

398

397 -AutoProcessor.register("OvisProcessor", OvisProcessor)

\ No newline at end of file

399 +AutoProcessor.register("OvisProcessor", OvisProcessor)