@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 32c9eff2ff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix][V1] Fix molmo text-only inputs (#11676)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Jan 6, 2025 at 23:22 UTC 32c9eff2fff8ee91a60c9410c69042dc4c1cc5c8
3 files changed +123 -42
tests/models/decoder_only/vision_language/test_models.py
+10
@@ -341,6 +341,16 @@ VLM_TEST_SETTINGS = {
341 ),
342 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
343 ),
344 + "molmo": VLMTestInfo(
345 + models=["allenai/Molmo-7B-D-0924"],
346 + test_type=(VLMTestType.IMAGE),
347 + prompt_formatter=lambda img_prompt:"User: " + img_prompt + " Assistant:", # noqa: E501
348 + max_model_len=4096,
349 + max_num_seqs=2,
350 + image_size_factors=[(),(1.0, 1.0, 1.0)],
351 + patch_hf_runner=model_utils.mlomo_patch_hf_runner,
352 + postprocess_inputs=model_utils.molmo_post_processor,
353 + ),
354 # Tests for phi3v currently live in another file because of a bug in
355 # transformers. Once this issue is fixed, we can enable them here instead.
356 # https://github.com/huggingface/transformers/issues/34307
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+96 -3
@@ -5,17 +5,20 @@ typically specific to a small subset of models.
5 import re
6 import types
7 from pathlib import PosixPath
8 -from typing import Callable, List, Optional, Tuple, Union
8 +from typing import Any, Callable, Dict, List, Optional, Tuple, Union
9
10 import torch
11 from PIL.Image import Image
12 -from transformers import AutoConfig, AutoTokenizer, BatchEncoding
12 +from transformers import (AutoConfig, AutoTokenizer, BatchEncoding,
13 + GenerationConfig)
14
15 from vllm.sequence import SampleLogprobs
16 from vllm.transformers_utils.tokenizer import patch_padding_side
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18
18 -from .....conftest import HfRunner, ImageAsset, _ImageAssets
19 +from .....conftest import (HfRunner, ImageAsset, PromptAudioInput,
20 + PromptImageInput, PromptVideoInput, _ImageAssets)
21 +from ....utils import TokensTextLogprobs
22 from .types import RunnerOutput
23
24
@@ -222,6 +225,11 @@ def wrap_inputs_post_processor(hf_inputs: BatchEncoding, dtype: str):
225 return {"model_inputs": hf_inputs}
226
227
228 +def molmo_post_processor(hf_inputs: BatchEncoding, dtype: str):
229 + hf_inputs = cast_dtype_post_processor("images")(hf_inputs, dtype)
230 + return {k: v.unsqueeze(0) for k, v in hf_inputs.items()}
231 +
232 +
233 ####### Prompt path encoders for models that need models on disk
234 def qwen_prompt_path_encoder(
235 tmp_path: PosixPath, prompt: str, assets: Union[List[ImageAsset],
@@ -451,3 +459,88 @@ def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
459 hf_model.model.generate = types.MethodType(_generate, hf_model.model)
460
461 return hf_model
462 +
463 +
464 +def _generate_greedy_logprobs_limit(
465 + self,
466 + prompts: List[str],
467 + max_tokens: int,
468 + num_logprobs: int,
469 + images: Optional[PromptImageInput] = None,
470 + audios: Optional[PromptAudioInput] = None,
471 + videos: Optional[PromptVideoInput] = None,
472 + **kwargs: Any,
473 +) -> List[TokensTextLogprobs]:
474 + all_inputs = self.get_inputs(prompts,
475 + images=images,
476 + videos=videos,
477 + audios=audios)
478 +
479 + # Process in batches for inference.
480 + if len(all_inputs):
481 + input_ids_lst = []
482 + images_lst = []
483 + images_input_idx_lst = []
484 + imges_masks_lst = []
485 + for inputs in all_inputs:
486 + input_ids_lst.append(inputs["input_ids"])
487 + images_lst.append(inputs["images"])
488 + images_input_idx_lst.append(inputs["image_input_idx"])
489 + imges_masks_lst.append(inputs["image_masks"])
490 + batch_inputs = {}
491 + batch_inputs['input_ids'] = torch.cat(input_ids_lst, dim=0)
492 + batch_inputs['images'] = torch.cat(images_lst, dim=0)
493 + batch_inputs['image_input_idx'] = torch.cat(images_input_idx_lst,
494 + dim=0)
495 + batch_inputs['image_masks'] = torch.cat(imges_masks_lst, dim=0)
496 +
497 + outputs = self.model.generate_from_batch(
498 + batch=self.wrap_device(batch_inputs,
499 + device=self.model.device.type),
500 + generation_config=GenerationConfig(
501 + max_new_tokens=max_tokens,
502 + stop_strings="<|endoftext|>",
503 + do_sample=False,
504 + ),
505 + tokenizer=self.tokenizer,
506 + output_hidden_states=True,
507 + return_dict_in_generate=True,
508 + )
509 +
510 + all_logprobs: List[List[Dict[int, float]]] = []
511 + all_output_ids: List[List[int]] = []
512 + all_output_strs: List[str] = []
513 +
514 + for index in range(len(all_inputs)):
515 + (
516 + seq_logprobs_lst,
517 + output_len,
518 + ) = self._hidden_states_to_logprobs(outputs.hidden_states,
519 + num_logprobs)
520 + all_logprobs.append(seq_logprobs_lst)
521 + seq_ids = outputs.sequences[index]
522 + output_ids = seq_ids[-output_len:]
523 + all_output_ids.append(output_ids.tolist())
524 + all_output_strs.append(self.tokenizer.decode(output_ids))
525 + outputs = zip(all_output_ids, all_output_strs, all_logprobs)
526 + return [(output_ids, output_str, output_logprobs)
527 + for output_ids, output_str, output_logprobs in outputs]
528 +
529 +
530 +####### Molmo-specific HuggingFace runner patchers
531 +def mlomo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
532 + """Patches and returns an instance of the HfRunner to use for Molmo."""
533 + hf_processor = hf_model.processor
534 +
535 + def _processor(*args, **kwargs):
536 + return hf_processor.process(*args, **kwargs)
537 +
538 + hf_model.processor = _processor
539 +
540 + setattr( # noqa: B010
541 + hf_model,
542 + "generate_greedy_logprobs_limit",
543 + types.MethodType(_generate_greedy_logprobs_limit, hf_model),
544 + )
545 +
546 + return hf_model
vllm/model_executor/models/molmo.py
+17 -39
@@ -1081,45 +1081,25 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1081 else:
1082 out = processor.process(None, image, tokens=inputs["prompt_token_ids"])
1083
1084 - image_processor = processor.image_processor
1085 - max_total_crops = 1 + image_processor.max_crops
1086 - if image is not None:
1087 - images, image_input_idx, image_masks = pad_images(
1088 - max_total_crops,
1089 - out["images"],
1090 - out["image_input_idx"],
1091 - out.get("image_masks"),
1092 - )
1093 - else:
1094 - base_image_input_size = image_processor.base_image_input_size
1095 - image_patch_size = image_processor.image_patch_size
1096 - image_num_patch = (
1097 - base_image_input_size[0] // image_patch_size,
1098 - base_image_input_size[1] // image_patch_size,
1099 - )
1100 - n_pixels = image_patch_size * image_patch_size * 3
1101 - n_patches = image_num_patch[0] * image_num_patch[1]
1102 -
1103 - image_length_w = image_processor.image_token_length_w
1104 - image_length_h = image_processor.image_token_length_h
1105 - tokens_per_image = image_length_w * image_length_h
1106 - images = torch.full(
1107 - (max_total_crops, n_patches, n_pixels),
1108 - -1,
1109 - dtype=torch.float32,
1110 - )
1111 - image_input_idx = torch.full(
1112 - (max_total_crops, tokens_per_image),
1113 - -1,
1114 - dtype=torch.int32,
1084 + # If there is no image, return directly.
1085 + if image is None:
1086 + new_prompt_token_ids = out["input_ids"].tolist()
1087 + prompt = inputs.get("prompt")
1088 + if prompt is None:
1089 + prompt = tokenizer.decode(new_prompt_token_ids)
1090 + return token_inputs(
1091 + prompt_token_ids=new_prompt_token_ids,
1092 + prompt=prompt,
1093 )
1116 - if image_processor.image_padding_mask:
1117 - image_masks = torch.full(
1118 - (max_total_crops, n_patches),
1119 - -1,
1120 - dtype=torch.float32,
1121 - )
1094
1095 + image_processor = processor.image_processor
1096 + max_total_crops = 1 + image_processor.max_crops
1097 + images, image_input_idx, image_masks = pad_images(
1098 + max_total_crops,
1099 + out["images"],
1100 + out["image_input_idx"],
1101 + out.get("image_masks"),
1102 + )
1103 image_data = dict(
1104 images=images,
1105 image_input_idx=image_input_idx,
@@ -1143,11 +1123,9 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1123 offset = i
1124 size += 1
1125 image_data["image_start_end"] = (offset, offset + size)
1146 -
1126 prompt = inputs.get("prompt")
1127 if prompt is None:
1128 prompt = tokenizer.decode(new_prompt_token_ids)
1150 -
1129 return token_inputs(
1130 prompt_token_ids=new_prompt_token_ids,
1131 prompt=prompt,