| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Common utility functions relating to different models that are useful |
| 4 | for manipulating the input / output of HF & vLLM test runners, which are |
| 5 | typically specific to a small subset of models. |
| 6 | """ |
| 7 | |
| 8 | import types |
| 9 | |
| 10 | import numpy as np |
| 11 | import numpy.typing as npt |
| 12 | import PIL.Image |
| 13 | import pytest |
| 14 | import regex as re |
| 15 | import torch |
| 16 | from PIL.Image import Image |
| 17 | from transformers import ( |
| 18 | AutoConfig, |
| 19 | AutoTokenizer, |
| 20 | BatchFeature, |
| 21 | GenerationConfig, |
| 22 | GenerationMixin, |
| 23 | ) |
| 24 | from transformers.masking_utils import create_causal_mask |
| 25 | from transformers.video_utils import VideoMetadata |
| 26 | |
| 27 | from vllm.logprobs import SampleLogprobs |
| 28 | from vllm.platforms import current_platform |
| 29 | from vllm.utils.collection_utils import is_list_of |
| 30 | |
| 31 | from .....conftest import HfRunner, ImageTestAssets |
| 32 | from .types import RunnerOutput |
| 33 | |
| 34 | |
| 35 | ####### vLLM output processors functions |
| 36 | def blip2_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput: |
| 37 | """Sanitize vllm output [blip2 models] to be comparable with hf output.""" |
| 38 | _, output_str, out_logprobs = vllm_output |
| 39 | |
| 40 | hf_output_str = output_str + "\n" |
| 41 | |
| 42 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 43 | hf_output_ids = tokenizer.encode(hf_output_str) |
| 44 | assert hf_output_ids[0] == tokenizer.bos_token_id |
| 45 | hf_output_ids = hf_output_ids[1:] |
| 46 | |
| 47 | return hf_output_ids, hf_output_str, out_logprobs |
| 48 | |
| 49 | |
| 50 | def qwen2_vllm_to_hf_output( |
| 51 | vllm_output: RunnerOutput, model: str |
| 52 | ) -> tuple[list[int], str, SampleLogprobs | None]: |
| 53 | """Sanitize vllm output [qwen2 models] to be comparable with hf output.""" |
| 54 | output_ids, output_str, out_logprobs = vllm_output |
| 55 | |
| 56 | hf_output_str = output_str + "<|im_end|>" |
| 57 | |
| 58 | return output_ids, hf_output_str, out_logprobs |
| 59 | |
| 60 | |
| 61 | def kimiv_vl_vllm_to_hf_output( |
| 62 | vllm_output: RunnerOutput, model: str |
| 63 | ) -> tuple[list[int], str, SampleLogprobs | None]: |
| 64 | """Sanitize vllm output [kimi_vl models] to be comparable with hf output.""" |
| 65 | output_ids, output_str, out_logprobs = vllm_output |
| 66 | |
| 67 | hf_output_str = output_str + "<|im_end|>[EOS]" |
| 68 | |
| 69 | return output_ids, hf_output_str, out_logprobs |
| 70 | |
| 71 | |
| 72 | def llava_image_vllm_to_hf_output( |
| 73 | vllm_output: RunnerOutput, model: str |
| 74 | ) -> RunnerOutput: |
| 75 | config = AutoConfig.from_pretrained(model) |
| 76 | mm_token_id = config.image_token_index |
| 77 | return _llava_vllm_to_hf_output(vllm_output, model, mm_token_id) |
| 78 | |
| 79 | |
| 80 | def llava_video_vllm_to_hf_output( |
| 81 | vllm_output: RunnerOutput, model: str |
| 82 | ) -> tuple[list[int], str, SampleLogprobs | None]: |
| 83 | config = AutoConfig.from_pretrained(model) |
| 84 | mm_token_id = config.video_token_index |
| 85 | return _llava_vllm_to_hf_output(vllm_output, model, mm_token_id) |
| 86 | |
| 87 | |
| 88 | def _llava_vllm_to_hf_output( |
| 89 | vllm_output: RunnerOutput, model: str, mm_token_id: int |
| 90 | ) -> RunnerOutput: |
| 91 | """Sanitize vllm output [Llava models] to be comparable with hf output.""" |
| 92 | output_ids, output_str, out_logprobs = vllm_output |
| 93 | |
| 94 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 95 | eos_token_id = tokenizer.eos_token_id |
| 96 | |
| 97 | hf_output_ids = [ |
| 98 | token_id |
| 99 | for idx, token_id in enumerate(output_ids) |
| 100 | if token_id != mm_token_id or output_ids[idx - 1] != mm_token_id |
| 101 | ] |
| 102 | |
| 103 | # output_str[0] is not " " in some cases, e.g., Granite Vision, |
| 104 | # but for most llava based models, this is the case |
| 105 | hf_output_str = output_str[1:] if output_str[0] == " " else output_str |
| 106 | |
| 107 | if hf_output_ids[-1] == eos_token_id: |
| 108 | hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) |
| 109 | |
| 110 | return hf_output_ids, hf_output_str, out_logprobs |
| 111 | |
| 112 | |
| 113 | def llava_onevision_hf_model_kwargs(model: str) -> dict: |
| 114 | """Workaround to fix the sliding window issue in llava_onevision.""" |
| 115 | config = AutoConfig.from_pretrained(model) |
| 116 | config.text_config.sliding_window = None |
| 117 | return config.to_dict() |
| 118 | |
| 119 | |
| 120 | def llava_onevision_vllm_to_hf_output( |
| 121 | vllm_output: RunnerOutput, model: str |
| 122 | ) -> RunnerOutput: |
| 123 | """Sanitize vllm output [llava-onevision] to compare with hf output.""" |
| 124 | output_ids, output_str, out_logprobs = vllm_output |
| 125 | |
| 126 | config = AutoConfig.from_pretrained(model) |
| 127 | video_token_id = config.video_token_index |
| 128 | |
| 129 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 130 | eos_token_id = tokenizer.eos_token_id |
| 131 | |
| 132 | hf_output_ids = [ |
| 133 | token_id |
| 134 | for idx, token_id in enumerate(output_ids) |
| 135 | if token_id != video_token_id or output_ids[idx - 1] != video_token_id |
| 136 | ] |
| 137 | |
| 138 | hf_output_str = output_str |
| 139 | if hf_output_ids[-1] == eos_token_id: |
| 140 | hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) |
| 141 | |
| 142 | return hf_output_ids, hf_output_str, out_logprobs |
| 143 | |
| 144 | |
| 145 | def phi3v_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput: |
| 146 | """Sanitize vllm output [phi3v] to be comparable with hf output.""" |
| 147 | _, output_str, out_logprobs = vllm_output |
| 148 | |
| 149 | output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str) |
| 150 | assert output_str_without_image[0] == " " |
| 151 | output_str_without_image = output_str_without_image[1:] |
| 152 | |
| 153 | hf_output_str = output_str_without_image + "<|end|><|endoftext|>" |
| 154 | |
| 155 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 156 | hf_output_ids = tokenizer.encode(output_str_without_image) |
| 157 | assert hf_output_ids[0] == 1 |
| 158 | hf_output_ids = hf_output_ids[1:] |
| 159 | |
| 160 | return hf_output_ids, hf_output_str, out_logprobs |
| 161 | |
| 162 | |
| 163 | def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput: |
| 164 | """Sanitize vllm output to be comparable with hf output.""" |
| 165 | output_ids, output_str, out_logprobs = vllm_output |
| 166 | |
| 167 | config = AutoConfig.from_pretrained(model) |
| 168 | image_token_id = config.image_token_index |
| 169 | |
| 170 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 171 | eos_token_id = tokenizer.eos_token_id |
| 172 | |
| 173 | hf_output_ids = [ |
| 174 | token_id |
| 175 | for idx, token_id in enumerate(output_ids) |
| 176 | if token_id != image_token_id or output_ids[idx - 1] != image_token_id |
| 177 | ] |
| 178 | |
| 179 | hf_output_str = output_str |
| 180 | |
| 181 | if hf_output_ids[-1] == eos_token_id: |
| 182 | hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) |
| 183 | |
| 184 | return hf_output_ids, hf_output_str, out_logprobs |
| 185 | |
| 186 | |
| 187 | ####### Post-processors for HF outputs |
| 188 | def deepseekvl2_trunc_hf_output(hf_output: RunnerOutput, model: str) -> RunnerOutput: |
| 189 | output_ids, output_str, out_logprobs = hf_output |
| 190 | if output_str.endswith("<｜end▁of▁sentence｜>"): |
| 191 | output_str = output_str.split("<｜end▁of▁sentence｜>")[0] |
| 192 | return output_ids, output_str, out_logprobs |
| 193 | |
| 194 | |
| 195 | def idefics3_trunc_hf_output(hf_output: RunnerOutput, model: str) -> RunnerOutput: |
| 196 | output_ids, output_str, out_logprobs = hf_output |
| 197 | if output_str.endswith("<end_of_utterance>"): |
| 198 | output_str = output_str.split("<end_of_utterance>")[0] |
| 199 | return output_ids, output_str, out_logprobs |
| 200 | |
| 201 | |
| 202 | def smolvlm_trunc_hf_output(hf_output: RunnerOutput, model: str) -> RunnerOutput: |
| 203 | # Based on Idefics3 |
| 204 | return idefics3_trunc_hf_output(hf_output, model) |
| 205 | |
| 206 | |
| 207 | def minicpmv_trunc_hf_output(hf_output: RunnerOutput, model: str) -> RunnerOutput: |
| 208 | output_ids, output_str, out_logprobs = hf_output |
| 209 | if output_str.endswith("<|eot_id|>"): |
| 210 | output_str = output_str.split("<|eot_id|>")[0] |
| 211 | return output_ids, output_str, out_logprobs |
| 212 | |
| 213 | |
| 214 | def ultravox_trunc_hf_output(hf_output: RunnerOutput, model: str) -> RunnerOutput: |
| 215 | output_ids, output_str, out_logprobs = hf_output |
| 216 | |
| 217 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 218 | eos_token_id = tokenizer.eos_token_id |
| 219 | eos_token = tokenizer.decode(eos_token_id) |
| 220 | if output_str.endswith(eos_token): |
| 221 | output_str = output_str.split(eos_token)[0] |
| 222 | return output_ids, output_str, out_logprobs |
| 223 | |
| 224 | |
| 225 | ####### Functions for converting image assets to embeddings |
| 226 | def get_llava_embeddings(image_assets: ImageTestAssets): |
| 227 | return [asset.image_embeds for asset in image_assets] |
| 228 | |
| 229 | |
| 230 | ####### Model-specific HuggingFace runner patchers |
| 231 | def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 232 | """Patches and returns an instance of the HfRunner to use for GLM4.""" |
| 233 | hf_processor = hf_model.processor |
| 234 | |
| 235 | def processor(*args, text="", images=None, **kwargs): |
| 236 | if isinstance(images, Image): |
| 237 | images = [images] |
| 238 | # inputs is a custom class instead of dict or BatchFeature |
| 239 | inputs = hf_processor( |
| 240 | *args, |
| 241 | prompt=text, |
| 242 | images=images, |
| 243 | **kwargs, |
| 244 | ) |
| 245 | inputs = { |
| 246 | k: inputs[k] |
| 247 | for k in inputs.keys() # noqa |
| 248 | if k not in ("seq_lens", "sft_format") |
| 249 | } |
| 250 | return BatchFeature(data=inputs, tensor_type="pt") |
| 251 | |
| 252 | hf_model.processor = processor |
| 253 | hf_model.model.get_output_embeddings = ( |
| 254 | lambda: hf_model.model.language.model.embed_tokens |
| 255 | ) |
| 256 | return hf_model |
| 257 | |
| 258 | |
| 259 | def gemma3_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 260 | """Patches and returns an instance of the HfRunner to use for Gemma 3.""" |
| 261 | hf_processor = hf_model.processor |
| 262 | |
| 263 | def processor(*args, **kwargs): |
| 264 | return hf_processor(*args, do_pan_and_scan=True, **kwargs) |
| 265 | |
| 266 | hf_model.processor = processor |
| 267 | |
| 268 | return hf_model |
| 269 | |
| 270 | |
| 271 | def gemma3_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput: |
| 272 | """Sanitize vllm output [gemma-3] to compare with hf output.""" |
| 273 | output_ids, output_str, out_logprobs = vllm_output |
| 274 | |
| 275 | config = AutoConfig.from_pretrained(model) |
| 276 | image_token_id = config.image_token_id |
| 277 | |
| 278 | tokenizer = AutoTokenizer.from_pretrained(model) |
| 279 | eos_token_id = tokenizer.eos_token_id |
| 280 | |
| 281 | hf_output_ids = [ |
| 282 | token_id |
| 283 | for idx, token_id in enumerate(output_ids) |
| 284 | if token_id != image_token_id |
| 285 | ] |
| 286 | |
| 287 | hf_output_str = output_str |
| 288 | if hf_output_ids[-1] == eos_token_id: |
| 289 | hf_output_str = hf_output_str + tokenizer.decode(eos_token_id) |
| 290 | |
| 291 | return hf_output_ids, hf_output_str, out_logprobs |
| 292 | |
| 293 | |
| 294 | def glm4v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 295 | """Patches and returns an instance of the HfRunner to use for GLM4V.""" |
| 296 | if current_platform.is_rocm(): |
| 297 | import types |
| 298 | |
| 299 | config = hf_model.model.config |
| 300 | if hasattr(config, "num_layers") and not hasattr(config, "num_hidden_layers"): |
| 301 | config.num_hidden_layers = config.num_layers |
| 302 | config.output_hidden_states = True |
| 303 | |
| 304 | def patched_prepare_cache( |
| 305 | self, generation_config, model_kwargs, *args, **kwargs |
| 306 | ): |
| 307 | model_kwargs["past_key_values"] = None |
| 308 | model_kwargs["use_cache"] = False |
| 309 | return model_kwargs |
| 310 | |
| 311 | hf_model.model._prepare_cache_for_generation = types.MethodType( |
| 312 | patched_prepare_cache, hf_model.model |
| 313 | ) |
| 314 | original_generate = hf_model.model.generate |
| 315 | |
| 316 | def patched_generate(*args, **kwargs): |
| 317 | kwargs["output_hidden_states"] = True |
| 318 | kwargs["return_dict_in_generate"] = True |
| 319 | return original_generate(*args, **kwargs) |
| 320 | |
| 321 | hf_model.model.generate = patched_generate |
| 322 | original_forward = hf_model.model.forward |
| 323 | |
| 324 | def patched_forward(*args, **kwargs): |
| 325 | kwargs["output_hidden_states"] = True |
| 326 | return original_forward(*args, **kwargs) |
| 327 | |
| 328 | hf_model.model.forward = patched_forward |
| 329 | |
| 330 | hf_processor = hf_model.processor |
| 331 | |
| 332 | def processor(*args, text="", images=None, **kwargs): |
| 333 | if images is None: |
| 334 | return hf_processor(*args, **kwargs) |
| 335 | |
| 336 | images = [images] if isinstance(images, Image) else images |
| 337 | |
| 338 | contents = re.findall( |
| 339 | r"<\|begin_of_image\|><\|endoftext\|><\|end_of_image\|>(.*?)<\|assistant\|>", |
| 340 | text, |
| 341 | ) |
| 342 | assert len(contents) == len(images) |
| 343 | |
| 344 | return hf_processor.apply_chat_template( |
| 345 | [ |
| 346 | {"role": "user", "image": image, "content": content} |
| 347 | for image, content in zip(images, contents) |
| 348 | ], |
| 349 | add_generation_prompt=True, |
| 350 | tokenize=True, |
| 351 | return_dict=True, |
| 352 | **kwargs, |
| 353 | ) |
| 354 | |
| 355 | hf_model.processor = processor |
| 356 | hf_model.model.get_output_embeddings = ( |
| 357 | lambda: hf_model.model.transformer.output_layer |
| 358 | ) |
| 359 | return hf_model |
| 360 | |
| 361 | |
| 362 | def glm4_1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 363 | """Patches and returns an instance of the HfRunner to use for GLM4.1V.""" |
| 364 | hf_processor = hf_model.processor |
| 365 | |
| 366 | def processor(*args, videos=None, **kwargs): |
| 367 | if videos is not None and is_list_of(videos, tuple): |
| 368 | # If videos is a list of tuples, we assume each tuple contains |
| 369 | # (video_array, metadata) as in the case of GLM4.1V. |
| 370 | # Filter out 'do_sample_frames' as it's not a valid VideoMetadata arg |
| 371 | video_metadata = [ |
| 372 | [ |
| 373 | VideoMetadata( |
| 374 | **{k: v for k, v in video[1].items() if k != "do_sample_frames"} |
| 375 | ) |
| 376 | ] |
| 377 | for video in videos |
| 378 | ] |
| 379 | videos = [[video[0]] for video in videos] |
| 380 | else: |
| 381 | video_metadata = None |
| 382 | |
| 383 | return hf_processor( |
| 384 | *args, videos=videos, video_metadata=video_metadata, **kwargs |
| 385 | ) |
| 386 | |
| 387 | hf_model.processor = processor |
| 388 | return hf_model |
| 389 | |
| 390 | |
| 391 | def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 392 | """Patches and returns an instance of the HfRunner to use for H2OVL.""" |
| 393 | |
| 394 | class H2OVLProcessor: |
| 395 | """A simple processor for H2OVL models.""" |
| 396 | |
| 397 | def __init__(self, hf_runner: HfRunner): |
| 398 | self.num_image_token = hf_runner.model.num_image_token |
| 399 | self.tokenizer = hf_runner.tokenizer |
| 400 | |
| 401 | self.config = AutoConfig.from_pretrained( |
| 402 | hf_runner.model_name, trust_remote_code=True |
| 403 | ) |
| 404 | self.vision_config = self.config.vision_config |
| 405 | self.use_thumbnail = self.config.use_thumbnail |
| 406 | self.use_msac = self.config.use_msac |
| 407 | self.min_num = self.config.min_dynamic_patch |
| 408 | self.max_num = self.config.max_dynamic_patch |
| 409 | self.image_size = self.vision_config.image_size |
| 410 | |
| 411 | def __call__(self, text: str, images: Image | list[Image], **kwargs): |
| 412 | from vllm.transformers_utils.processors.h2ovl import ( |
| 413 | image_to_pixel_values_h2ovl, |
| 414 | ) |
| 415 | |
| 416 | IMG_START = "<img>" |
| 417 | IMG_END = "</img>" |
| 418 | IMG_CONTEXT = "<IMG_CONTEXT>" |
| 419 | |
| 420 | images = [images] if isinstance(images, Image) else images |
| 421 | pixel_values = [ |
| 422 | image_to_pixel_values_h2ovl( |
| 423 | image, |
| 424 | input_size=self.image_size, |
| 425 | min_num=self.min_num, |
| 426 | max_num=self.max_num, |
| 427 | use_thumbnail=self.use_thumbnail, |
| 428 | use_msac=self.use_msac, |
| 429 | ) |
| 430 | for image in images |
| 431 | ] |
| 432 | num_patches_list = [pixel_value.shape[0] for pixel_value in pixel_values] |
| 433 | pixel_values = torch.cat(pixel_values, dim=0) |
| 434 | for num_patches in num_patches_list: |
| 435 | context_tokens = IMG_CONTEXT * self.num_image_token * num_patches |
| 436 | image_tokens = IMG_START + context_tokens + IMG_END |
| 437 | text = text.replace("<image>", image_tokens, 1) |
| 438 | prompt = self.tokenizer(text, return_tensors="pt") |
| 439 | prompt.update({"pixel_values": pixel_values}) |
| 440 | return prompt |
| 441 | |
| 442 | img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>") |
| 443 | hf_model.model.img_context_token_id = img_context_token_id |
| 444 | hf_model.processor = H2OVLProcessor(hf_model) |
| 445 | hf_model.model.get_output_embeddings = ( |
| 446 | lambda: hf_model.model.language_model.get_output_embeddings() |
| 447 | ) |
| 448 | hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) |
| 449 | return hf_model |
| 450 | |
| 451 | |
| 452 | def isaac_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 453 | """Patch HF runner for Isaac: |
| 454 | 1) Move processor outputs to model device |
| 455 | 2) Ensure IsaacModel.forward returns hidden_states |
| 456 | for compatibility with hidden_states_to_seq_logprobs() |
| 457 | """ |
| 458 | from perceptron.tensorstream import TextType |
| 459 | from perceptron.tensorstream.ops import compute_mrope_pos_tensor, modality_mask |
| 460 | from transformers.modeling_outputs import BaseModelOutputWithPast |
| 461 | |
| 462 | def compute_position_ids_input_ids(input_ids: torch.Tensor) -> torch.Tensor: |
| 463 | """Create 3D positional indices for token input.""" |
| 464 | batch_size, seq_length = input_ids.shape |
| 465 | position_ids = torch.arange(seq_length, device=input_ids.device) |
| 466 | position_ids = position_ids.view(1, -1).expand(batch_size, -1) |
| 467 | position_ids = position_ids.unsqueeze(2).expand(-1, -1, 3) # Add 3D for MRoPE |
| 468 | return position_ids |
| 469 | |
| 470 | model_device = next(hf_model.model.parameters()).device |
| 471 | |
| 472 | # ---------------------------- |
| 473 | # 1) Patch processor: move BatchFeature input_ids and TensorStream to model device |
| 474 | # ---------------------------- |
| 475 | original_processor = hf_model.processor |
| 476 | |
| 477 | def patched_processor(*args, **kwargs): |
| 478 | result = original_processor(*args, **kwargs) |
| 479 | for k, v in result.data.items(): |
| 480 | result[k] = v.to(model_device) |
| 481 | return result |
| 482 | |
| 483 | hf_model.processor = patched_processor |
| 484 | |
| 485 | tokenizer = AutoTokenizer.from_pretrained( |
| 486 | hf_model.model_name, trust_remote_code=True |
| 487 | ) |
| 488 | |
| 489 | original_generate = hf_model.model.generate |
| 490 | |
| 491 | def patched_generate(*args, **kwargs): |
| 492 | kwargs["pad_token_id"] = tokenizer.eos_token_id |
| 493 | kwargs["eos_token_id"] = tokenizer.eos_token_id |
| 494 | return original_generate(*args, **kwargs) |
| 495 | |
| 496 | hf_model.model.generate = patched_generate |
| 497 | |
| 498 | # ---------------------------- |
| 499 | # 2) Patch IsaacModel.forward: add hidden_states to the output |
| 500 | # ---------------------------- |
| 501 | isaac_model = hf_model.model.model |
| 502 | |
| 503 | def patched_forward( |
| 504 | self, |
| 505 | input_ids=None, |
| 506 | tensor_stream=None, |
| 507 | attention_mask=None, |
| 508 | position_ids=None, |
| 509 | modality_tensor=None, |
| 510 | past_key_values=None, |
| 511 | inputs_embeds=None, |
| 512 | use_cache=None, |
| 513 | output_hidden_states=None, |
| 514 | return_dict=None, |
| 515 | cache_position=None, |
| 516 | **kwargs, |
| 517 | ): |
| 518 | """Forward pass with MRoPE position embeddings. |
| 519 | Computes position embeddings once and passes them through all layers. |
| 520 | """ |
| 521 | output_hidden_states = ( |
| 522 | output_hidden_states |
| 523 | if output_hidden_states is not None |
| 524 | else self.config.output_hidden_states |
| 525 | ) |
| 526 | use_cache = use_cache if use_cache is not None else self.config.use_cache |
| 527 | return_dict = ( |
| 528 | return_dict if return_dict is not None else self.config.use_return_dict |
| 529 | ) |
| 530 | |
| 531 | # Get inputs |
| 532 | if tensor_stream is not None and inputs_embeds is not None: |
| 533 | raise ValueError("You cannot specify both tensor_stream and inputs_embeds") |
| 534 | elif tensor_stream is not None: |
| 535 | # Embed TensorStream directly |
| 536 | inputs_embeds = self.embed_stream(tensor_stream) |
| 537 | # Create modality tensor if not provided |
| 538 | if modality_tensor is None: |
| 539 | modality_tensor = modality_mask(tensor_stream) |
| 540 | elif input_ids is not None and inputs_embeds is not None: |
| 541 | raise ValueError( |
| 542 | "You cannot specify both input_ids and inputs_embeds at the same time" |
| 543 | ) |
| 544 | elif input_ids is not None: |
| 545 | inputs_embeds = self.embed_tokens(input_ids) |
| 546 | # Create text modality tensor if not provided |
| 547 | if modality_tensor is None: |
| 548 | batch_size, seq_length = input_ids.shape |
| 549 | modality_tensor = torch.full( |
| 550 | (batch_size, seq_length), |
| 551 | TextType.text.value, |
| 552 | device=input_ids.device, |
| 553 | dtype=torch.long, |
| 554 | ) |
| 555 | elif inputs_embeds is None: |
| 556 | raise ValueError( |
| 557 | "You have to specify either tensor_stream, input_ids or inputs_embeds" |
| 558 | ) |
| 559 | |
| 560 | # Create default position_ids if not provided |
| 561 | if position_ids is None: |
| 562 | if tensor_stream is not None: |
| 563 | position_ids = compute_mrope_pos_tensor(tensor_stream) # (B,L,3) |
| 564 | else: |
| 565 | position_ids = compute_position_ids_input_ids(input_ids) |
| 566 | |
| 567 | # Compute MRoPE position embeddings if we have custom rotary_emb |
| 568 | cos, sin = self.rotary_emb(position_ids, modality_tensor) |
| 569 | cos = cos.to(inputs_embeds.dtype) |
| 570 | sin = sin.to(inputs_embeds.dtype) |
| 571 | |
| 572 | # Prepare attention mask |
| 573 | attention_mask = create_causal_mask( |
| 574 | config=self.config, |
| 575 | input_embeds=inputs_embeds, |
| 576 | attention_mask=attention_mask, |
| 577 | past_key_values=past_key_values, |
| 578 | position_ids=position_ids, |
| 579 | cache_position=cache_position, |
| 580 | ) |
| 581 | |
| 582 | # Initialize and collect hidden states |
| 583 | hidden_states = inputs_embeds |
| 584 | hidden_states_list: list[torch.Tensor] = [] |
| 585 | |
| 586 | if output_hidden_states: |
| 587 | hidden_states_list.append(hidden_states) |
| 588 | |
| 589 | for decoder_layer in self.layers: |
| 590 | layer_outputs = decoder_layer( |
| 591 | hidden_states, |
| 592 | attention_mask=attention_mask, |
| 593 | position_ids=position_ids, |
| 594 | past_key_value=past_key_values, |
| 595 | use_cache=use_cache, |
| 596 | cache_position=cache_position, |
| 597 | position_embeddings=(cos, sin), |
| 598 | **kwargs, |
| 599 | ) |
| 600 | |
| 601 | hidden_states = ( |
| 602 | layer_outputs[0] if isinstance(layer_outputs, tuple) else layer_outputs |
| 603 | ) |
| 604 | |
| 605 | if output_hidden_states: |
| 606 | hidden_states_list.append(hidden_states) |
| 607 | |
| 608 | # Final layer norm |
| 609 | hidden_states = self.norm(hidden_states) |
| 610 | |
| 611 | if output_hidden_states: |
| 612 | hidden_states_list.append(hidden_states) |
| 613 | |
| 614 | # Convert to tuple or None |
| 615 | all_hidden_states = tuple(hidden_states_list) if output_hidden_states else None |
| 616 | |
| 617 | # Include hidden_states for compatibility with hidden_states_to_seq_logprobs() |
| 618 | return BaseModelOutputWithPast( |
| 619 | last_hidden_state=hidden_states, |
| 620 | past_key_values=past_key_values, |
| 621 | hidden_states=all_hidden_states, |
| 622 | ) |
| 623 | |
| 624 | isaac_model.forward = types.MethodType(patched_forward, isaac_model) |
| 625 | |
| 626 | return hf_model |
| 627 | |
| 628 | |
| 629 | def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 630 | """Patches and returns an instance of the HfRunner to use for SkyworkR1V.""" |
| 631 | |
| 632 | class SkyworkR1VProcessor: |
| 633 | """A simple processor for SkyworkR1V.""" |
| 634 | |
| 635 | def __init__(self, hf_runner: HfRunner): |
| 636 | self.num_image_token = hf_runner.model.num_image_token |
| 637 | self.tokenizer = hf_runner.tokenizer |
| 638 | |
| 639 | self.config = AutoConfig.from_pretrained( |
| 640 | hf_runner.model_name, trust_remote_code=True |
| 641 | ) |
| 642 | self.vision_config = self.config.vision_config |
| 643 | self.use_thumbnail = self.config.use_thumbnail |
| 644 | self.min_num = self.config.min_dynamic_patch |
| 645 | self.max_num = self.config.max_dynamic_patch |
| 646 | self.image_size = self.vision_config.image_size |
| 647 | |
| 648 | def __call__(self, text: str, images: Image | list[Image], **kwargs): |
| 649 | from vllm.transformers_utils.processors.internvl import ( |
| 650 | image_to_pixel_values_internvl, |
| 651 | ) |
| 652 | |
| 653 | IMG_START = "<img>" |
| 654 | IMG_END = "</img>" |
| 655 | IMG_CONTEXT = "<IMG_CONTEXT>" |
| 656 | |
| 657 | images = [images] if isinstance(images, Image) else images |
| 658 | pixel_values = [ |
| 659 | image_to_pixel_values_internvl( |
| 660 | image, |
| 661 | input_size=self.image_size, |
| 662 | min_num=self.min_num, |
| 663 | max_num=self.max_num, |
| 664 | use_thumbnail=self.use_thumbnail, |
| 665 | ) |
| 666 | for image in images |
| 667 | ] |
| 668 | num_patches_list = [pixel_value.shape[0] for pixel_value in pixel_values] |
| 669 | pixel_values = torch.cat(pixel_values, dim=0) |
| 670 | for num_patches in num_patches_list: |
| 671 | context_tokens = IMG_CONTEXT * self.num_image_token * num_patches |
| 672 | image_tokens = IMG_START + context_tokens + IMG_END |
| 673 | text = text.replace("<image>", image_tokens, 1) |
| 674 | prompt = self.tokenizer(text, return_tensors="pt") |
| 675 | prompt.update({"pixel_values": pixel_values}) |
| 676 | return prompt |
| 677 | |
| 678 | img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>") |
| 679 | hf_model.model.img_context_token_id = img_context_token_id |
| 680 | hf_model.processor = SkyworkR1VProcessor(hf_model) |
| 681 | hf_model.model.get_output_embeddings = ( |
| 682 | lambda: hf_model.model.language_model.get_output_embeddings() |
| 683 | ) |
| 684 | hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) |
| 685 | return hf_model |
| 686 | |
| 687 | |
| 688 | def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 689 | """Patches and returns an instance of the HfRunner to use for InternVL.""" |
| 690 | |
| 691 | class InternVLProcessor: |
| 692 | """A simple processor for InternVL2 which misses a processor.""" |
| 693 | |
| 694 | def __init__(self, hf_runner: HfRunner): |
| 695 | self.num_image_token = hf_runner.model.num_image_token |
| 696 | self.tokenizer = hf_runner.tokenizer |
| 697 | |
| 698 | self.config = AutoConfig.from_pretrained( |
| 699 | hf_runner.model_name, trust_remote_code=True |
| 700 | ) |
| 701 | self.vision_config = self.config.vision_config |
| 702 | self.use_thumbnail = self.config.use_thumbnail |
| 703 | self.min_num = self.config.min_dynamic_patch |
| 704 | self.max_num = self.config.max_dynamic_patch |
| 705 | self.image_size = self.vision_config.image_size |
| 706 | |
| 707 | def __call__( |
| 708 | self, |
| 709 | text: str, |
| 710 | images: Image | list[Image] = None, |
| 711 | videos: npt.NDArray | list[npt.NDArray] = None, |
| 712 | **kwargs, |
| 713 | ): |
| 714 | from vllm.transformers_utils.processors.internvl import ( |
| 715 | image_to_pixel_values_internvl, |
| 716 | video_to_pixel_values_internvl, |
| 717 | ) |
| 718 | |
| 719 | IMG_START = "<img>" |
| 720 | IMG_END = "</img>" |
| 721 | IMG_CONTEXT = "<IMG_CONTEXT>" |
| 722 | |
| 723 | images = [images] if isinstance(images, Image) else images |
| 724 | videos = [videos] if isinstance(videos, np.ndarray) else videos |
| 725 | if images is not None: |
| 726 | pixel_values_images = [ |
| 727 | image_to_pixel_values_internvl( |
| 728 | image, |
| 729 | input_size=self.image_size, |
| 730 | min_num=self.min_num, |
| 731 | max_num=self.max_num, |
| 732 | use_thumbnail=self.use_thumbnail, |
| 733 | ) |
| 734 | for image in images |
| 735 | ] |
| 736 | num_patches_images = [ |
| 737 | pixel_value.shape[0] for pixel_value in pixel_values_images |
| 738 | ] |
| 739 | else: |
| 740 | pixel_values_images, num_patches_images = [], [] |
| 741 | |
| 742 | if videos is not None: |
| 743 | pixel_values_videos = [ |
| 744 | video_to_pixel_values_internvl( |
| 745 | video, |
| 746 | input_size=self.image_size, |
| 747 | min_num=1, |
| 748 | max_num=1, |
| 749 | use_thumbnail=False, |
| 750 | ) |
| 751 | for video in videos |
| 752 | ] |
| 753 | num_patches_videos = [ |
| 754 | pixel_value.shape[0] for pixel_value in pixel_values_videos |
| 755 | ] |
| 756 | else: |
| 757 | pixel_values_videos, num_patches_videos = [], [] |
| 758 | |
| 759 | pixel_values = [] |
| 760 | while ("<image>" in text) or ("<video>" in text): |
| 761 | image_index = text.find("<image>") |
| 762 | video_index = text.find("<video>") |
| 763 | if image_index == -1 or ( |
| 764 | video_index > -1 and video_index < image_index |
| 765 | ): |
| 766 | num_patches = num_patches_videos.pop(0) |
| 767 | pixel_values.append(pixel_values_videos.pop(0)) |
| 768 | context_tokens = ( |
| 769 | IMG_START + IMG_CONTEXT * self.num_image_token + IMG_END |
| 770 | ) |
| 771 | video_tokens = "".join( |
| 772 | [f"Frame{i + 1}: {context_tokens}" for i in range(num_patches)] |
| 773 | ) |
| 774 | text = text.replace("<video>", video_tokens, 1) |
| 775 | else: |
| 776 | num_patches = num_patches_images.pop(0) |
| 777 | pixel_values.append(pixel_values_images.pop(0)) |
| 778 | context_tokens = IMG_CONTEXT * self.num_image_token * num_patches |
| 779 | image_tokens = IMG_START + context_tokens + IMG_END |
| 780 | text = text.replace("<image>", image_tokens, 1) |
| 781 | pixel_values = torch.cat(pixel_values, dim=0) |
| 782 | |
| 783 | prompt = self.tokenizer(text, return_tensors="pt") |
| 784 | prompt.update({"pixel_values": pixel_values}) |
| 785 | return prompt |
| 786 | |
| 787 | img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>") |
| 788 | hf_model.model.img_context_token_id = img_context_token_id |
| 789 | hf_model.processor = InternVLProcessor(hf_model) |
| 790 | hf_model.model.get_output_embeddings = ( |
| 791 | lambda: hf_model.model.language_model.get_output_embeddings() |
| 792 | ) |
| 793 | hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) |
| 794 | return hf_model |
| 795 | |
| 796 | |
| 797 | def _internvl_generate( |
| 798 | self, |
| 799 | pixel_values: torch.FloatTensor, |
| 800 | input_ids: torch.FloatTensor, |
| 801 | attention_mask: torch.LongTensor | None = None, |
| 802 | **generate_kwargs, |
| 803 | ) -> torch.LongTensor: |
| 804 | """Generate method for InternVL2 model without fixed use_cache.""" |
| 805 | assert self.img_context_token_id is not None |
| 806 | target_dtype = next(self.parameters()).dtype |
| 807 | vit_embeds = self.extract_feature(pixel_values.to(target_dtype)) |
| 808 | input_embeds = self.language_model.get_input_embeddings()(input_ids) |
| 809 | B, N, C = input_embeds.shape |
| 810 | input_embeds = input_embeds.reshape(B * N, C) |
| 811 | |
| 812 | input_ids = input_ids.reshape(B * N) |
| 813 | selected = input_ids == self.img_context_token_id |
| 814 | assert selected.sum() != 0 |
| 815 | input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device) |
| 816 | |
| 817 | input_embeds = input_embeds.reshape(B, N, C) |
| 818 | |
| 819 | forward_kwargs = dict( |
| 820 | inputs_embeds=input_embeds, |
| 821 | attention_mask=attention_mask, |
| 822 | ) |
| 823 | if getattr(self, "use_visual_token_mask", False): |
| 824 | visual_token_mask = selected.reshape(B, N, 1).to(input_embeds.dtype) |
| 825 | forward_kwargs["visual_token_mask"] = visual_token_mask |
| 826 | |
| 827 | # e.g. InternVL2-2B |
| 828 | if not isinstance(self.language_model, GenerationMixin): |
| 829 | pytest.skip("HF impl is not compatible with current transformers") |
| 830 | |
| 831 | outputs = self.language_model.generate( |
| 832 | **forward_kwargs, |
| 833 | **generate_kwargs, |
| 834 | ) |
| 835 | |
| 836 | return outputs |
| 837 | |
| 838 | |
| 839 | def _restore_resampler_pos_cache(hf_model: HfRunner) -> None: |
| 840 | """Recompute the resampler's 2D sin/cos position cache after loading. |
| 841 | |
| 842 | `from_pretrained` materializes non-persistent buffers as uninitialized |
| 843 | memory, leaving `pos_embed` as zeros or NaN instead of the values computed |
| 844 | in `__init__`. |
| 845 | """ |
| 846 | restored = 0 |
| 847 | for module in hf_model.model.modules(): |
| 848 | if not hasattr(module, "_set_2d_pos_cache"): |
| 849 | continue |
| 850 | module._set_2d_pos_cache(module.max_size, module.pos_embed.device) |
| 851 | restored += 1 |
| 852 | assert restored, "no resampler pos cache found to restore" |
| 853 | |
| 854 | |
| 855 | def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 856 | orig_generate = hf_model.model.generate |
| 857 | |
| 858 | def _generate( |
| 859 | self, |
| 860 | *args, |
| 861 | input_ids=None, |
| 862 | pixel_values=None, |
| 863 | image_sizes=None, |
| 864 | image_bound=None, |
| 865 | tgt_sizes=None, |
| 866 | **kwargs, |
| 867 | ): |
| 868 | model_inputs = { |
| 869 | "input_ids": input_ids, |
| 870 | "pixel_values": pixel_values, |
| 871 | "image_sizes": image_sizes, |
| 872 | "image_bound": image_bound, |
| 873 | "tgt_sizes": tgt_sizes, |
| 874 | } |
| 875 | for k in list(model_inputs.keys()): |
| 876 | if model_inputs[k] is None: |
| 877 | model_inputs.pop(k) |
| 878 | |
| 879 | return orig_generate(model_inputs, *args, decode_text=False, **kwargs) |
| 880 | |
| 881 | hf_model.model.generate = types.MethodType(_generate, hf_model.model) |
| 882 | |
| 883 | return hf_model |
| 884 | |
| 885 | |
| 886 | def minicpmo_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 887 | orig_generate = hf_model.model.generate |
| 888 | |
| 889 | def _generate(self, *args, image_sizes=None, **kwargs): |
| 890 | return orig_generate(*args, decode_text=False, **kwargs) |
| 891 | |
| 892 | hf_model.model.generate = types.MethodType(_generate, hf_model.model) |
| 893 | |
| 894 | return hf_model |
| 895 | |
| 896 | |
| 897 | def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 898 | _restore_resampler_pos_cache(hf_model) |
| 899 | |
| 900 | orig_generate = hf_model.model.generate |
| 901 | |
| 902 | def _generate(self, *args, image_sizes=None, **kwargs): |
| 903 | return orig_generate(*args, decode_text=False, **kwargs) |
| 904 | |
| 905 | hf_model.model.generate = types.MethodType(_generate, hf_model.model) |
| 906 | |
| 907 | return hf_model |
| 908 | |
| 909 | |
| 910 | def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 911 | """Patches and returns an instance of the HfRunner to use for Molmo.""" |
| 912 | hf_processor = hf_model.processor |
| 913 | |
| 914 | def _processor(*args, **kwargs): |
| 915 | return hf_processor.process(*args, **kwargs) |
| 916 | |
| 917 | hf_model.processor = _processor |
| 918 | |
| 919 | def _generate(self, max_new_tokens=None, do_sample=None, **kwargs): |
| 920 | batch = { |
| 921 | k: kwargs.pop(k).unsqueeze(0) |
| 922 | for k in ("input_ids", "images", "image_input_idx", "image_masks") |
| 923 | if k in kwargs |
| 924 | } |
| 925 | batch = BatchFeature(batch).to(dtype=self.dtype) |
| 926 | |
| 927 | return self.generate_from_batch( |
| 928 | batch, |
| 929 | generation_config=GenerationConfig( |
| 930 | max_new_tokens=max_new_tokens, |
| 931 | stop_strings="<|endoftext|>", |
| 932 | do_sample=do_sample, |
| 933 | ), |
| 934 | **kwargs, |
| 935 | ) |
| 936 | |
| 937 | hf_model.model.generate = types.MethodType(_generate, hf_model.model) |
| 938 | |
| 939 | return hf_model |
| 940 | |
| 941 | |
| 942 | def ovis_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 943 | """Patches and returns an instance of the HfRunner to use for Ovis2.""" |
| 944 | hf_model.model.get_output_embeddings = ( |
| 945 | lambda: hf_model.model.llm.get_output_embeddings() |
| 946 | ) |
| 947 | |
| 948 | def processor(*args, text="", images=None, **kwargs): |
| 949 | text_tokenizer = hf_model.model.get_text_tokenizer() |
| 950 | images = [images] if isinstance(images, Image) else images |
| 951 | |
| 952 | prompt_start_and_end = { |
| 953 | "qwen2": ("<|im_start|>user\n", "<|im_end|>\n"), |
| 954 | "llama": ("<|start_header_id|>user<|end_header_id|>\n\n", "<|eot_id|>"), |
| 955 | "gemma2": ("<start_of_turn>user\n", "<end_of_turn>\n"), |
| 956 | } |
| 957 | for start, end in prompt_start_and_end.values(): |
| 958 | if start in text and end in text: |
| 959 | text = text.split(start)[1].split(end)[0] |
| 960 | break |
| 961 | |
| 962 | prompt, input_ids, pixel_values = hf_model.model.preprocess_inputs( |
| 963 | text_or_conversations=text, images=images |
| 964 | ) |
| 965 | attention_mask = torch.ne(input_ids, text_tokenizer.pad_token_id) |
| 966 | |
| 967 | inputs = { |
| 968 | "inputs": input_ids.unsqueeze(0), |
| 969 | "pixel_values": pixel_values.unsqueeze(0), |
| 970 | "attention_mask": attention_mask.unsqueeze(0), |
| 971 | } |
| 972 | return BatchFeature(data=inputs, tensor_type="pt") |
| 973 | |
| 974 | hf_model.processor = processor |
| 975 | return hf_model |
| 976 | |
| 977 | |
| 978 | def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 979 | """Patches and returns an instance of the HfRunner to use for Ovis2.""" |
| 980 | hf_model.model.get_output_embeddings = ( |
| 981 | lambda: hf_model.model.llm.get_output_embeddings() |
| 982 | ) |
| 983 | |
| 984 | def processor(*args, text="", images=None, videos=None, **kwargs): |
| 985 | if images is None: |
| 986 | images = [] |
| 987 | else: |
| 988 | images = [images] if isinstance(images, Image) else images |
| 989 | if videos is None: |
| 990 | videos = [] |
| 991 | else: |
| 992 | videos = [videos] if isinstance(videos, np.ndarray) else videos |
| 993 | videos = [[PIL.Image.fromarray(frame) for frame in vid] for vid in videos] |
| 994 | |
| 995 | prompt_start_and_end = { |
| 996 | "qwen2": ("<|im_start|>user\n", "<|im_end|>\n"), |
| 997 | "llama": ("<|start_header_id|>user<|end_header_id|>\n\n", "<|eot_id|>"), |
| 998 | "gemma2": ("<start_of_turn>user\n", "<end_of_turn>\n"), |
| 999 | } |
| 1000 | for start, end in prompt_start_and_end.values(): |
| 1001 | if start in text and end in text: |
| 1002 | text = text.split(start)[1].split(end)[0] |
| 1003 | break |
| 1004 | |
| 1005 | images_message = [{"type": "image", "image": img} for img in images] |
| 1006 | videos_message = [{"type": "video", "video": vid} for vid in videos] |
| 1007 | |
| 1008 | messages = [ |
| 1009 | { |
| 1010 | "role": "user", |
| 1011 | "content": [ |
| 1012 | *images_message, |
| 1013 | *videos_message, |
| 1014 | {"type": "text", "text": text}, |
| 1015 | ], |
| 1016 | } |
| 1017 | ] |
| 1018 | |
| 1019 | input_ids, pixel_values, grid_thws = hf_model.model.preprocess_inputs( |
| 1020 | messages=messages, enable_thinking=True |
| 1021 | ) |
| 1022 | inputs = { |
| 1023 | "inputs": input_ids, |
| 1024 | "pixel_values": pixel_values, |
| 1025 | "grid_thws": grid_thws, |
| 1026 | } |
| 1027 | return BatchFeature(data=inputs, tensor_type="pt") |
| 1028 | |
| 1029 | hf_model.processor = processor |
| 1030 | return hf_model |
| 1031 | |
| 1032 | |
| 1033 | def paddleocr_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 1034 | """Patches the HfRunner to fix create_causal_mask API mismatch. |
| 1035 | |
| 1036 | The PaddleOCR-VL HF model passes `inputs_embeds` to create_causal_mask, |
| 1037 | but transformers renamed this parameter to `input_embeds`. |
| 1038 | """ |
| 1039 | import sys |
| 1040 | |
| 1041 | model_module = sys.modules.get(type(hf_model.model.model).__module__) |
| 1042 | if model_module is None: |
| 1043 | return hf_model |
| 1044 | |
| 1045 | original_create_causal_mask = getattr(model_module, "create_causal_mask", None) |
| 1046 | if original_create_causal_mask is None: |
| 1047 | return hf_model |
| 1048 | |
| 1049 | def patched_create_causal_mask(*args, **kwargs): |
| 1050 | if "inputs_embeds" in kwargs: |
| 1051 | kwargs["input_embeds"] = kwargs.pop("inputs_embeds") |
| 1052 | return original_create_causal_mask(*args, **kwargs) |
| 1053 | |
| 1054 | model_module.create_causal_mask = patched_create_causal_mask # type: ignore[attr-defined] |
| 1055 | return hf_model |
| 1056 | |
| 1057 | |
| 1058 | def qwen2_5_omni_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 1059 | """Patches and returns an instance of the HfRunner for Qwen2.5-Omni.""" |
| 1060 | thinker = hf_model.model.thinker |
| 1061 | thinker.get_output_embeddings = lambda: thinker.lm_head |
| 1062 | hf_model.model = thinker |
| 1063 | return hf_model |
| 1064 | |
| 1065 | |
| 1066 | def qwen3_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 1067 | """Patches and returns an instance of the HfRunner to use for GLM4.1V.""" |
| 1068 | hf_processor = hf_model.processor |
| 1069 | |
| 1070 | def processor(*args, videos=None, **kwargs): |
| 1071 | if videos is not None and is_list_of(videos, tuple): |
| 1072 | # batched multi videos |
| 1073 | do_sample_frames = {video[1]["do_sample_frames"] for video in videos} |
| 1074 | assert len(do_sample_frames) == 1 |
| 1075 | if kwargs.get("do_sample_frames") is None: |
| 1076 | kwargs["do_sample_frames"] = do_sample_frames |
| 1077 | video_metadata = [ |
| 1078 | [ |
| 1079 | VideoMetadata( |
| 1080 | **{k: v for k, v in video[1].items() if k != "do_sample_frames"} |
| 1081 | ) |
| 1082 | ] |
| 1083 | for video in videos |
| 1084 | ] |
| 1085 | videos = [[video[0]] for video in videos] |
| 1086 | elif videos is not None and isinstance(videos, tuple): |
| 1087 | # single video |
| 1088 | do_sample_frames = videos[1]["do_sample_frames"] |
| 1089 | if kwargs.get("do_sample_frames") is None: |
| 1090 | kwargs["do_sample_frames"] = do_sample_frames |
| 1091 | video_metadata = [ |
| 1092 | [ |
| 1093 | VideoMetadata( |
| 1094 | **{ |
| 1095 | k: v |
| 1096 | for k, v in videos[1].items() |
| 1097 | if k != "do_sample_frames" |
| 1098 | } |
| 1099 | ) |
| 1100 | ] |
| 1101 | ] |
| 1102 | videos = [[videos[0]]] |
| 1103 | else: |
| 1104 | video_metadata = None |
| 1105 | |
| 1106 | return hf_processor( |
| 1107 | *args, videos=videos, video_metadata=video_metadata, **kwargs |
| 1108 | ) |
| 1109 | |
| 1110 | hf_model.processor = processor |
| 1111 | return hf_model |
| 1112 | |
| 1113 | |
| 1114 | def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner": |
| 1115 | """Patch HfRunner for Voxtral's conversation-based processor. |
| 1116 | |
| 1117 | Two issues in HfRunner require patching: |
| 1118 | |
| 1119 | 1. VoxtralProcessor requires ``apply_chat_template()`` with conversation |
| 1120 | dicts (accepting ``url``, ``path``, or ``base64`` audio) rather than |
| 1121 | the standard ``processor(text=, audio=, sampling_rate=)`` interface. |
| 1122 | 2. HfRunner.get_inputs cannot handle multi-audio per prompt because it |
| 1123 | incorrectly unpacks ``[(arr1, sr1), (arr2, sr2)]`` via a ``len == 2`` check. |
| 1124 | |
| 1125 | We override ``get_inputs`` to build conversation dicts and call |
| 1126 | ``apply_chat_template`` directly, bypassing both issues. We also wrap |
| 1127 | ``model.generate`` to strip prompt tokens before decoding, since |
| 1128 | HfRunner.generate calls batch_decode on the full sequence (prompt + |
| 1129 | generated). |
| 1130 | """ |
| 1131 | import io |
| 1132 | |
| 1133 | import pybase64 as base64 |
| 1134 | import soundfile as sf |
| 1135 | |
| 1136 | processor = hf_model.processor |
| 1137 | |
| 1138 | def _audio_to_base64(audio_array, sample_rate: int) -> str: |
| 1139 | """Encode a numpy audio array as a base64 WAV string.""" |
| 1140 | buf = io.BytesIO() |
| 1141 | sf.write(buf, audio_array, int(sample_rate), format="WAV") |
| 1142 | return base64.b64encode(buf.getvalue()).decode("ascii") |
| 1143 | |
| 1144 | def patched_get_inputs(prompts, images=None, videos=None, audios=None, **kwargs): |
| 1145 | all_inputs = [] |
| 1146 | for i, prompt in enumerate(prompts): |
| 1147 | content: list[dict] = [] |
| 1148 | |
| 1149 | if audios is not None and audios[i] is not None: |
| 1150 | items = audios[i] |
| 1151 | if not isinstance(items, list): |
| 1152 | items = [items] |
| 1153 | for item in items: |
| 1154 | if isinstance(item, (list, tuple)) and len(item) == 2: |
| 1155 | arr, sr = item |
| 1156 | else: |
| 1157 | arr, sr = item, 16_000 |
| 1158 | content.append( |
| 1159 | { |
| 1160 | "type": "audio", |
| 1161 | "base64": _audio_to_base64(arr, sr), |
| 1162 | } |
| 1163 | ) |
| 1164 | |
| 1165 | content.append({"type": "text", "text": prompt}) |
| 1166 | |
| 1167 | inputs = processor.apply_chat_template( |
| 1168 | [{"role": "user", "content": content}] |
| 1169 | ) |
| 1170 | if hasattr(inputs, "to"): |
| 1171 | inputs = inputs.to(dtype=hf_model.dtype) |
| 1172 | all_inputs.append(inputs) |
| 1173 | |
| 1174 | return all_inputs |
| 1175 | |
| 1176 | _orig_generate = hf_model.model.generate |
| 1177 | |
| 1178 | def patched_generate(*args, **kwargs): |
| 1179 | """Strip prompt tokens so only generated tokens are decoded.""" |
| 1180 | input_ids = kwargs.get("input_ids") |
| 1181 | if input_ids is None and args: |
| 1182 | input_ids = args[0] |
| 1183 | prompt_len = input_ids.shape[1] if input_ids is not None else 0 |
| 1184 | |
| 1185 | output = _orig_generate(*args, **kwargs) |
| 1186 | if prompt_len: |
| 1187 | if isinstance(output, torch.Tensor): |
| 1188 | output = output[:, prompt_len:] |
| 1189 | else: |
| 1190 | # GenerateDecoderOnlyOutput - trim sequences but preserve |
| 1191 | # scores/logits so generate_greedy_logprobs_limit can |
| 1192 | # extract per-token logprobs. |
| 1193 | output.sequences = output.sequences[:, prompt_len:] |
| 1194 | return output |
| 1195 | |
| 1196 | hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign] |
| 1197 | hf_model.model.generate = patched_generate # type: ignore[method-assign] |
| 1198 | return hf_model |
| 1199 | |
| 1200 | |
| 1201 | def moondream3_processor(model: str): |
| 1202 | from vllm.transformers_utils.processors.moondream3 import Moondream3Processor |
| 1203 | |
| 1204 | return Moondream3Processor.from_pretrained(model, trust_remote_code=True) |
| 1205 | |
| 1206 | |
| 1207 | def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner: |
| 1208 | """Patch HfRunner for Moondream3.""" |
| 1209 | moondream_processor = hf_model.processor |
| 1210 | |
| 1211 | def processor(*args, text="", images=None, **kwargs): |
| 1212 | if images is None: |
| 1213 | return moondream_processor(text=text, **kwargs) |
| 1214 | |
| 1215 | images_list = [images] if isinstance(images, Image) else images |
| 1216 | return moondream_processor(images=images_list, text=text, **kwargs) |
| 1217 | |
| 1218 | hf_model.processor = processor |
| 1219 | |
| 1220 | # Expose the LM head for logprob extraction. |
| 1221 | hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head |
| 1222 | |
| 1223 | native_model = hf_model.model.model # MoondreamModel instance |
| 1224 | |
| 1225 | from torch.nn import functional as F |
| 1226 | |
| 1227 | from vllm.model_executor.models.moondream3 import reconstruct_from_crops |
| 1228 | |
| 1229 | # Resolve the placeholder tokens from the tokenizer instead of hard-coding. |
| 1230 | image_placeholder_ids = moondream_processor.tokenizer.encode( |
| 1231 | "<image>", add_special_tokens=False |
| 1232 | ) |
| 1233 | |
| 1234 | def _normalize_tiling(tilings): |
| 1235 | """Extract (h, w) tuple from various tiling container formats.""" |
| 1236 | tiling = tilings |
| 1237 | if isinstance(tiling, torch.Tensor): |
| 1238 | tiling = tuple(tiling.squeeze().tolist()) |
| 1239 | elif isinstance(tiling, (list, tuple)): |
| 1240 | t0 = tiling[0] |
| 1241 | if isinstance(t0, torch.Tensor): |
| 1242 | tiling = tuple(t0.tolist()) |
| 1243 | elif isinstance(t0, (list, tuple)): |
| 1244 | tiling = tuple(t0) |
| 1245 | return tiling |
| 1246 | |
| 1247 | def _encode_vision(pixel_values, tilings): |
| 1248 | """Run preprocessed crops through vision encoder + projection.""" |
| 1249 | device = native_model.device |
| 1250 | dtype = native_model.vision.pos_emb.dtype |
| 1251 | config = native_model.config |
| 1252 | |
| 1253 | pv = pixel_values |
| 1254 | while pv.dim() > 4: |
| 1255 | pv = pv.squeeze(0) |
| 1256 | pv = pv.to(device=device, dtype=dtype) |
| 1257 | |
| 1258 | features = native_model._vis_enc(pv) |
| 1259 | grid_size = config.vision.crop_size // config.vision.enc_patch_size |
| 1260 | global_feat = features[0] |
| 1261 | |
| 1262 | if features.shape[0] > 1 and tilings is not None: |
| 1263 | tiling = _normalize_tiling(tilings) |
| 1264 | local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim) |
| 1265 | reconstructed = reconstruct_from_crops( |
| 1266 | local, |
| 1267 | tiling, |
| 1268 | config.vision.overlap_margin, |
| 1269 | patch_size=1, |
| 1270 | ) |
| 1271 | else: |
| 1272 | reconstructed = global_feat.view( |
| 1273 | grid_size, grid_size, config.vision.enc_dim |
| 1274 | ) |
| 1275 | |
| 1276 | return native_model._vis_proj(global_feat, reconstructed) |
| 1277 | |
| 1278 | def _find_subsequence(seq, subseq): |
| 1279 | """Find start index of subseq in seq, or None.""" |
| 1280 | n = len(subseq) |
| 1281 | for i in range(len(seq) - n + 1): |
| 1282 | if seq[i : i + n] == subseq: |
| 1283 | return i |
| 1284 | return None |
| 1285 | |
| 1286 | def _generate( |
| 1287 | self, |
| 1288 | input_ids=None, |
| 1289 | pixel_values=None, |
| 1290 | tilings=None, |
| 1291 | attention_mask=None, |
| 1292 | **kwargs, |
| 1293 | ): |
| 1294 | max_new_tokens = kwargs.get("max_new_tokens", 128) |
| 1295 | return_dict = kwargs.get("return_dict_in_generate", False) |
| 1296 | output_hs = kwargs.get("output_hidden_states", False) |
| 1297 | |
| 1298 | if pixel_values is None: |
| 1299 | sequences = input_ids |
| 1300 | if return_dict: |
| 1301 | return types.SimpleNamespace( |
| 1302 | sequences=sequences, |
| 1303 | hidden_states=() if output_hs else None, |
| 1304 | ) |
| 1305 | return sequences |
| 1306 | |
| 1307 | # Processor may return lists; extract the single element. |
| 1308 | if isinstance(pixel_values, (list, tuple)): |
| 1309 | pixel_values = pixel_values[0] |
| 1310 | if ( |
| 1311 | isinstance(tilings, (list, tuple)) |
| 1312 | and tilings |
| 1313 | and not isinstance(tilings[0], int) |
| 1314 | ): |
| 1315 | tilings = tilings[0] |
| 1316 | |
| 1317 | hf_model.model._setup_caches() |
| 1318 | native_model.use_flex_decoding = False |
| 1319 | |
| 1320 | device = native_model.device |
| 1321 | config = native_model.config |
| 1322 | |
| 1323 | with torch.inference_mode(): |
| 1324 | for block in native_model.text.blocks: |
| 1325 | block.kv_cache.k_cache.zero_() |
| 1326 | block.kv_cache.v_cache.zero_() |
| 1327 | |
| 1328 | img_emb = _encode_vision(pixel_values, tilings) |
| 1329 | |
| 1330 | bos_emb = F.embedding( |
| 1331 | torch.tensor([[config.tokenizer.bos_id]], device=device), |
| 1332 | native_model.text.wte, |
| 1333 | ) |
| 1334 | img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1) |
| 1335 | prefix_len = img_input.size(1) |
| 1336 | |
| 1337 | mask = native_model.attn_mask[:, :, :prefix_len, :] |
| 1338 | pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device) |
| 1339 | native_model._prefill(img_input, mask, pos_ids, None) |
| 1340 | |
| 1341 | ids = input_ids.squeeze(0).tolist() |
| 1342 | img_start = _find_subsequence(ids, image_placeholder_ids) |
| 1343 | |
| 1344 | if img_start is None: |
| 1345 | sequences = input_ids |
| 1346 | if return_dict: |
| 1347 | return types.SimpleNamespace( |
| 1348 | sequences=sequences, |
| 1349 | hidden_states=() if output_hs else None, |
| 1350 | ) |
| 1351 | return sequences |
| 1352 | |
| 1353 | prompt_tokens = ids[img_start + len(image_placeholder_ids) :] |
| 1354 | |
| 1355 | if not prompt_tokens: |
| 1356 | sequences = input_ids |
| 1357 | if return_dict: |
| 1358 | return types.SimpleNamespace( |
| 1359 | sequences=sequences, |
| 1360 | hidden_states=() if output_hs else None, |
| 1361 | ) |
| 1362 | return sequences |
| 1363 | |
| 1364 | prompt_tensor = torch.tensor([prompt_tokens], device=device) |
| 1365 | prompt_emb = F.embedding(prompt_tensor, native_model.text.wte) |
| 1366 | prompt_len = prompt_emb.size(1) |
| 1367 | |
| 1368 | mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :] |
| 1369 | pos_ids = torch.arange( |
| 1370 | prefix_len, |
| 1371 | prefix_len + prompt_len, |
| 1372 | dtype=torch.long, |
| 1373 | device=device, |
| 1374 | ) |
| 1375 | hidden = native_model._prefill(prompt_emb, mask, pos_ids, None) |
| 1376 | pos = prefix_len + prompt_len |
| 1377 | |
| 1378 | hidden_last = native_model.text.post_ln(hidden[:, -1:, :]) |
| 1379 | logits = native_model.text.lm_head(hidden_last.squeeze(1)) |
| 1380 | |
| 1381 | generated = [] |
| 1382 | all_hidden_states = [] |
| 1383 | # Record the hidden state that predicted each generated token. |
| 1384 | prev_hs = hidden_last |
| 1385 | for _ in range(max_new_tokens): |
| 1386 | next_token = logits.argmax(dim=-1).item() |
| 1387 | if next_token == 0: |
| 1388 | break |
| 1389 | generated.append(next_token) |
| 1390 | if output_hs: |
| 1391 | all_hidden_states.append((prev_hs,)) |
| 1392 | |
| 1393 | next_emb = F.embedding( |
| 1394 | torch.tensor([[next_token]], device=device), |
| 1395 | native_model.text.wte, |
| 1396 | ) |
| 1397 | mask = native_model.attn_mask[:, :, pos : pos + 1, :] |
| 1398 | pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device) |
| 1399 | hidden = native_model._prefill(next_emb, mask, pos_ids_step, None) |
| 1400 | hidden_last = native_model.text.post_ln(hidden[:, -1:, :]) |
| 1401 | prev_hs = hidden_last |
| 1402 | logits = native_model.text.lm_head(hidden_last.squeeze(1)) |
| 1403 | pos += 1 |
| 1404 | |
| 1405 | result_ids = ids + generated |
| 1406 | sequences = torch.tensor([result_ids], device=device) |
| 1407 | |
| 1408 | if return_dict: |
| 1409 | return types.SimpleNamespace( |
| 1410 | sequences=sequences, |
| 1411 | hidden_states=tuple(all_hidden_states) if output_hs else None, |
| 1412 | ) |
| 1413 | return sequences |
| 1414 | |
| 1415 | hf_model.model.generate = types.MethodType(_generate, hf_model.model) |
| 1416 | return hf_model |