@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 39e227c7ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Update multi-modal processor to support Mantis(LLaVA) model (#10711)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 8, 2024 at 01:10 UTC 39e227c7ae3149eb8345ea1a1ffee672ef76c09a
14 files changed +175 -78
.buildkite/test-pipeline.yaml
+2
@@ -362,6 +362,7 @@ steps:
362 - tests/models/embedding/vision_language
363 - tests/models/encoder_decoder/vision_language
364 commands:
365 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
366 - pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
367 - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
368 - pytest -v -s models/embedding/vision_language -m core_model
@@ -377,6 +378,7 @@ steps:
378 - tests/models/embedding/vision_language
379 - tests/models/encoder_decoder/vision_language
380 commands:
381 + - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
382 - pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
383 # HACK - run phi3v tests separately to sidestep this transformers bug
384 # https://github.com/huggingface/transformers/issues/34307
docs/source/models/supported_models.rst
+5 -1
@@ -555,7 +555,7 @@ Text Generation
555 * - :code:`LlavaForConditionalGeneration`
556 - LLaVA-1.5
557 - T + I\ :sup:`E+`
558 - - :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
558 + - :code:`llava-hf/llava-1.5-7b-hf`, :code:`TIGER-Lab/Mantis-8B-siglip-llama3` (see note), etc.
559 -
560 - ✅︎
561 * - :code:`LlavaNextForConditionalGeneration`
@@ -664,6 +664,10 @@ Text Generation
664 .. note::
665 vLLM currently only supports adding LoRA to the language backbone of multimodal models.
666
667 +.. note::
668 + To use :code:`TIGER-Lab/Mantis-8B-siglip-llama3`, you have to install their GitHub repo (:code:`pip install git+https://github.com/TIGER-AI-Lab/Mantis.git`)
669 + and pass :code:`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
670 +
671 .. note::
672 The official :code:`openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
673 For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
examples/offline_inference_vision_language.py
+17
@@ -419,6 +419,22 @@ def run_aria(question: str, modality: str):
419 return llm, prompt, stop_token_ids
420
421
422 +# Mantis
423 +def run_mantis(question: str, modality: str):
424 + assert modality == "image"
425 +
426 + llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
427 + prompt = llama3_template.format(f"{question}\n<image>")
428 +
429 + llm = LLM(
430 + model="TIGER-Lab/Mantis-8B-siglip-llama3",
431 + max_model_len=4096,
432 + hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
433 + )
434 + stop_token_ids = [128009]
435 + return llm, prompt, stop_token_ids
436 +
437 +
438 model_example_map = {
439 "llava": run_llava,
440 "llava-next": run_llava_next,
@@ -441,6 +457,7 @@ model_example_map = {
457 "glm4v": run_glm4v,
458 "idefics3": run_idefics3,
459 "aria": run_aria,
460 + "mantis": run_mantis,
461 }
462
463
requirements-test.in
-3
@@ -24,9 +24,6 @@ mistral_common[opencv] >= 1.5.0 # required for pixtral test
24 datamodel_code_generator # required for minicpm3 test
25 lm-eval[api]==0.4.4 # required for model evaluation test
26
27 -# TODO: Add this after fully implementing llava(mantis)
28 -# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
29 -
27 # quantization
28 bitsandbytes>=0.44.0
29 buildkite-test-collector==0.1.9
tests/models/decoder_only/vision_language/test_models.py
+22 -8
@@ -34,7 +34,7 @@ COMMON_BROADCAST_SETTINGS = {
34 "dtype": "half",
35 "max_tokens": 5,
36 "tensor_parallel_size": 2,
37 - "model_kwargs": {"device_map": "auto"},
37 + "hf_model_kwargs": {"device_map": "auto"},
38 "image_size_factors": [(.25, 0.5, 1.0)],
39 "distributed_executor_backend": (
40 "ray",
@@ -108,7 +108,7 @@ VLM_TEST_SETTINGS = {
108 "cherry_blossom": "What is in the picture?",
109 }),
110 auto_cls=AutoModelForVision2Seq,
111 - postprocess_inputs=model_utils.get_key_type_post_processor(
111 + postprocess_inputs=model_utils.cast_dtype_post_processor(
112 "pixel_values"
113 ),
114 vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
@@ -151,7 +151,7 @@ VLM_TEST_SETTINGS = {
151 "cherry_blossom": "<vlm_image>Please infer the season with reason.",
152 }),
153 multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
154 - postprocess_inputs=model_utils.get_key_type_post_processor("pixel_values"),
154 + postprocess_inputs=model_utils.cast_dtype_post_processor("pixel_values"),
155 stop_str=["<|im_end|>"],
156 image_size_factors=[(0.10, 0.15)],
157 max_tokens=64,
@@ -177,7 +177,7 @@ VLM_TEST_SETTINGS = {
177 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
178 max_model_len=4096,
179 auto_cls=AutoModelForVision2Seq,
180 - postprocess_inputs=model_utils.get_key_type_post_processor(
180 + postprocess_inputs=model_utils.cast_dtype_post_processor(
181 "pixel_values"
182 ),
183 # For chameleon, we only compare the sequences
@@ -281,7 +281,7 @@ VLM_TEST_SETTINGS = {
281 prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
282 num_video_frames=16,
283 max_model_len=16384,
284 - postprocess_inputs=model_utils.get_key_type_post_processor(
284 + postprocess_inputs=model_utils.cast_dtype_post_processor(
285 "pixel_values_videos"
286 ),
287 auto_cls=AutoModelForVision2Seq,
@@ -306,6 +306,20 @@ VLM_TEST_SETTINGS = {
306 vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
307 image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
308 ),
309 + "mantis": VLMTestInfo(
310 + models=["TIGER-Lab/Mantis-8B-siglip-llama3"],
311 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
312 + prompt_formatter=lambda img_prompt: f"<|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
313 + max_model_len=4096,
314 + postprocess_inputs=model_utils.cast_dtype_post_processor(
315 + "pixel_values"
316 + ),
317 + vllm_runner_kwargs={"hf_overrides": {"architectures": ["MantisForConditionalGeneration"]}}, # noqa: E501
318 + get_stop_token_ids=lambda tok: [128009],
319 + auto_cls=AutoModelForVision2Seq,
320 + vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
321 + patch_hf_runner=model_utils.mantis_patch_hf_runner,
322 + ),
323 "minicpmv_25": VLMTestInfo(
324 models=["openbmb/MiniCPM-Llama3-V-2_5"],
325 test_type=VLMTestType.IMAGE,
@@ -342,7 +356,7 @@ VLM_TEST_SETTINGS = {
356 # max_num_seqs=2,
357 # task="generate",
358 # # use eager mode for hf runner since phi3v didn't work with flash_attn
345 - # model_kwargs={"_attn_implementation": "eager"},
359 + # hf_model_kwargs={"_attn_implementation": "eager"},
360 # use_tokenizer_eos=True,
361 # vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
362 # num_logprobs=10,
@@ -373,7 +387,7 @@ VLM_TEST_SETTINGS = {
387 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
388 max_model_len=4096,
389 auto_cls=AutoModelForVision2Seq,
376 - postprocess_inputs=model_utils.get_key_type_post_processor(
390 + postprocess_inputs=model_utils.cast_dtype_post_processor(
391 "pixel_values"
392 ),
393 vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
@@ -438,7 +452,7 @@ VLM_TEST_SETTINGS = {
452 test_type=VLMTestType.CUSTOM_INPUTS,
453 max_model_len=16384,
454 max_num_seqs=2,
441 - postprocess_inputs=model_utils.get_key_type_post_processor(
455 + postprocess_inputs=model_utils.cast_dtype_post_processor(
456 "pixel_values"
457 ),
458 auto_cls=AutoModelForVision2Seq,
tests/models/decoder_only/vision_language/vlm_utils/core.py
+14 -6
@@ -3,9 +3,11 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
3
4 import torch
5 from PIL.Image import Image
6 -from transformers import AutoTokenizer, BatchEncoding
6 +from transformers import AutoTokenizer, BatchEncoding, PreTrainedTokenizerBase
7 from transformers.models.auto.auto_factory import _BaseAutoModelClass
8
9 +from vllm.config import TaskOption
10 +
11 from .....conftest import HfRunner, VllmRunner
12 from .types import RunnerOutput
13
@@ -28,13 +30,15 @@ def run_test(
30 use_tokenizer_eos: bool,
31 postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
32 comparator: Callable[..., None],
31 - get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]],
33 + get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
34 + List[int]]],
35 stop_str: Optional[List[str]],
36 tokenizer_mode: str,
37 limit_mm_per_prompt: Dict[str, int],
35 - model_kwargs: Optional[Dict[str, Any]],
38 + vllm_runner_kwargs: Optional[Dict[str, Any]],
39 + hf_model_kwargs: Optional[Dict[str, Any]],
40 patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]],
37 - task: str = "auto",
41 + task: TaskOption = "auto",
42 runner_mm_key: str = "images",
43 distributed_executor_backend: Optional[str] = None,
44 tensor_parallel_size: int = 1,
@@ -58,6 +62,9 @@ def run_test(
62 if stop_str:
63 vllm_kwargs["stop"] = stop_str
64
65 + if vllm_runner_kwargs is None:
66 + vllm_runner_kwargs = {}
67 +
68 with vllm_runner(model,
69 tokenizer_mode=tokenizer_mode,
70 max_model_len=max_model_len,
@@ -67,7 +74,8 @@ def run_test(
74 tensor_parallel_size=tensor_parallel_size,
75 distributed_executor_backend=distributed_executor_backend,
76 enforce_eager=enforce_eager,
70 - task=task) as vllm_model:
77 + task=task,
78 + **vllm_runner_kwargs) as vllm_model:
79 for prompts, media in vllm_inputs:
80 vllm_kwargs[runner_mm_key] = media
81 vllm_output = vllm_model.generate_greedy_logprobs(
@@ -78,7 +86,7 @@ def run_test(
86 dtype=dtype,
87 auto_cls=auto_cls,
88 postprocess_inputs=postprocess_inputs,
81 - model_kwargs=model_kwargs)
89 + model_kwargs=hf_model_kwargs)
90
91 # Some models need to patch things like the model processor, e.g., internvl
92 if patch_hf_runner is not None:
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+34 -1
@@ -126,6 +126,16 @@ def llava_onevision_vllm_to_hf_output(vllm_output: RunnerOutput,
126 return hf_output_ids, hf_output_str, out_logprobs
127
128
129 +def mantis_vllm_to_hf_output(vllm_output: RunnerOutput,
130 + model: str) -> RunnerOutput:
131 + """Sanitize vllm output [mantis] to compare with hf output."""
132 + output_ids, output_str, out_logprobs = vllm_output
133 +
134 + hf_output_str = output_str + "<|eot_id|>"
135 +
136 + return output_ids, hf_output_str, out_logprobs
137 +
138 +
139 def phi3v_vllm_to_hf_output(vllm_output: RunnerOutput,
140 model: str) -> RunnerOutput:
141 """Sanitize vllm output [phi3v] to be comparable with hf output."""
@@ -184,7 +194,7 @@ def get_llava_embeddings(image_assets: _ImageAssets):
194
195
196 ####### postprocessors to run on HF BatchEncoding
187 -def get_key_type_post_processor(
197 +def cast_dtype_post_processor(
198 hf_inp_key: str) -> Callable[[BatchEncoding, str], BatchEncoding]:
199 """Gets a handle to a post processor which converts a given key into a
200 target data type."""
@@ -418,3 +428,26 @@ def _internvl_generate(
428 )
429
430 return outputs
431 +
432 +
433 +def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
434 + from mantis.models.mllava import MLlavaProcessor
435 +
436 + hf_model.processor = MLlavaProcessor.from_pretrained(hf_model.model_name)
437 +
438 + orig_generate = hf_model.model.generate
439 + tokenizer = hf_model.processor.tokenizer
440 +
441 + def _generate(self, *args, **kwargs):
442 + return orig_generate(
443 + *args,
444 + **kwargs,
445 + eos_token_id=[
446 + tokenizer.eos_token_id,
447 + tokenizer.convert_tokens_to_ids("<|eot_id|>"),
448 + ],
449 + )
450 +
451 + hf_model.model.generate = types.MethodType(_generate, hf_model.model)
452 +
453 + return hf_model
tests/models/decoder_only/vision_language/vlm_utils/types.py
+12 -7
@@ -7,9 +7,11 @@ from typing import (Any, Callable, Dict, Iterable, List, NamedTuple, Optional,
7 import torch
8 from PIL.Image import Image
9 from pytest import MarkDecorator
10 -from transformers import AutoModelForCausalLM, AutoTokenizer, BatchEncoding
10 +from transformers import (AutoModelForCausalLM, BatchEncoding,
11 + PreTrainedTokenizerBase)
12 from transformers.models.auto.auto_factory import _BaseAutoModelClass
13
14 +from vllm.config import TaskOption
15 from vllm.sequence import SampleLogprobs
16 from vllm.utils import identity
17
@@ -66,7 +68,7 @@ class ImageSizeWrapper(NamedTuple):
68 class VLMTestInfo(NamedTuple):
69 """Holds the configuration for 1+ tests for one model architecture."""
70
69 - models: Union[List[str]]
71 + models: List[str]
72 test_type: Union[VLMTestType, Iterable[VLMTestType]]
73
74 # Should be None only if this is a CUSTOM_INPUTS test
@@ -92,18 +94,20 @@ class VLMTestInfo(NamedTuple):
94 enforce_eager: bool = True
95 max_model_len: int = 1024
96 max_num_seqs: int = 256
95 - task: str = "auto"
97 + task: TaskOption = "auto"
98 tensor_parallel_size: int = 1
99 + vllm_runner_kwargs: Optional[Dict[str, Any]] = None
100
101 # Optional callable which gets a list of token IDs from the model tokenizer
99 - get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]] = None
102 + get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
103 + List[int]]] = None
104 # Optional list of strings to stop generation, useful when stop tokens are
105 # not special tokens in the tokenizer
106 stop_str: Optional[List[str]] = None
107
108 # Exposed options for HF runner
105 - model_kwargs: Optional[Dict[str, Any]] = None
106 - # Indicates we should explicitly pass the EOS from the tokeniezr
109 + hf_model_kwargs: Optional[Dict[str, Any]] = None
110 + # Indicates we should explicitly pass the EOS from the tokenizer
111 use_tokenizer_eos: bool = False
112 auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM
113 # Callable to pass to the HF runner to run on inputs; for now, we also pass
@@ -164,6 +168,7 @@ class VLMTestInfo(NamedTuple):
168 "max_num_seqs": self.max_num_seqs,
169 "task": self.task,
170 "tensor_parallel_size": self.tensor_parallel_size,
171 + "vllm_runner_kwargs": self.vllm_runner_kwargs,
172 "hf_output_post_proc": self.hf_output_post_proc,
173 "vllm_output_post_proc": self.vllm_output_post_proc,
174 "auto_cls": self.auto_cls,
@@ -171,8 +176,8 @@ class VLMTestInfo(NamedTuple):
176 "postprocess_inputs": self.postprocess_inputs,
177 "comparator": self.comparator,
178 "get_stop_token_ids": self.get_stop_token_ids,
179 + "hf_model_kwargs": self.hf_model_kwargs,
180 "stop_str": self.stop_str,
175 - "model_kwargs": self.model_kwargs,
181 "patch_hf_runner": self.patch_hf_runner,
182 "tokenizer_mode": self.tokenizer_mode
183 }
tests/models/registry.py
+1
@@ -176,6 +176,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
176 "LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
177 "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo("llava-hf/LLaVA-NeXT-Video-7B-hf"), # noqa: E501
178 "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
179 + "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3"), # noqa: E501
180 "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
181 trust_remote_code=True),
182 "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+2 -4
@@ -3,16 +3,14 @@ from typing import Optional
3 import torch
4
5 from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6 - create_metadata_for_llava,
7 - dummy_mm_kwargs_for_llava,
6 + LlavaProcessor,
7 get_max_llava_image_tokens)
8 from vllm.model_executor.sampling_metadata import SamplingMetadata
9 from vllm.multimodal import MULTIMODAL_REGISTRY
10
11
12 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
14 -@MULTIMODAL_REGISTRY.register_processor_by_metadata(create_metadata_for_llava,
15 - dummy_mm_kwargs_for_llava)
13 +@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
14 class MyLlava(LlavaForConditionalGeneration):
15
16 def compute_logits(
vllm/model_executor/models/llava.py
+60 -8
@@ -22,10 +22,11 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
23 from vllm.multimodal import MULTIMODAL_REGISTRY
24 from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
25 -from vllm.multimodal.processing import (InputProcessingContext,
25 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
26 + InputProcessingContext,
27 ModalityProcessingMetadata,
28 MultiModalProcessingMetadata,
28 - MultiModalProcessor, PromptReplacement)
29 + PromptReplacement)
30 from vllm.sequence import IntermediateTensors
31
32 from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -163,7 +164,13 @@ def create_metadata_for_llava(
164 }
165
166
166 -class LlavaProcessor(MultiModalProcessor):
167 +class LlavaProcessor(BaseMultiModalProcessor):
168 +
169 + def __init__(self, ctx: InputProcessingContext) -> None:
170 + super().__init__(
171 + ctx=ctx,
172 + metadata=create_metadata_for_llava(ctx),
173 + )
174
175 def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
176 if getattr(hf_processor, "__is_patched__", False):
@@ -193,7 +200,30 @@ class LlavaProcessor(MultiModalProcessor):
200 self,
201 mm_counts: Mapping[str, int],
202 ) -> MultiModalKwargs:
196 - return dummy_mm_kwargs_for_llava(self.ctx, mm_counts)
203 + hf_config = self.ctx.get_hf_config(LlavaConfig)
204 + vision_config = hf_config.vision_config
205 + num_images = mm_counts["image"]
206 +
207 + if isinstance(vision_config, CLIPVisionConfig):
208 + data = dummy_image_for_clip(vision_config, num_images)
209 + elif isinstance(vision_config, SiglipVisionConfig):
210 + data = dummy_image_for_siglip(vision_config, num_images)
211 + elif isinstance(vision_config, PixtralVisionConfig):
212 + data = dummy_image_for_pixtral_hf(vision_config, num_images)
213 + else:
214 + msg = f"Unsupported vision config: {type(vision_config)}"
215 + raise NotImplementedError(msg)
216 +
217 + hf_processor = self._get_hf_processor()
218 + image_processor = hf_processor.image_processor # type: ignore
219 + hf_inputs = image_processor.preprocess(data['image'],
220 + return_tensors="pt")
221 + is_pixtral = isinstance(hf_processor, PixtralProcessor)
222 +
223 + return MultiModalKwargs(
224 + **hf_inputs,
225 + is_pixtral=torch.tensor(is_pixtral),
226 + )
227
228
229 class LlavaLikeConfig(Protocol):
@@ -277,10 +307,7 @@ def init_vision_tower_for_llava(
307
308
309 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
280 -@MULTIMODAL_REGISTRY.register_processor(lambda ctx: LlavaProcessor(
281 - ctx=ctx,
282 - metadata=create_metadata_for_llava(ctx),
283 -))
310 +@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
311 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
312 # BitandBytes specific attributes
313 bitsandbytes_stacked_params_mapping = {
@@ -559,3 +586,28 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
586 torch.Tensor]]) -> Set[str]:
587 loader = AutoWeightsLoader(self)
588 return loader.load_weights(weights)
589 +
590 +
591 +class MantisProcessor(LlavaProcessor):
592 +
593 + def _get_hf_processor(self) -> ProcessorMixin:
594 + try:
595 + from mantis.models.mllava import MLlavaProcessor
596 + except ModuleNotFoundError as exc:
597 + raise ModuleNotFoundError(
598 + "You need to `pip install "
599 + "git+https://github.com/TIGER-AI-Lab/Mantis.git` "
600 + "to use this model") from exc
601 +
602 + processor = MLlavaProcessor.from_pretrained(
603 + self.ctx.model_config.tokenizer)
604 + assert isinstance(processor, ProcessorMixin)
605 + return processor
606 +
607 +
608 +# To use this model, please use
609 +# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
610 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
611 +@MULTIMODAL_REGISTRY.register_processor(MantisProcessor)
612 +class MantisForConditionalGeneration(LlavaForConditionalGeneration):
613 + pass
vllm/model_executor/models/registry.py
+1
@@ -152,6 +152,7 @@ _MULTIMODAL_MODELS = {
152 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
153 "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
154 "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
155 + "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"), # noqa: E501
156 "MiniCPMV": ("minicpmv", "MiniCPMV"),
157 "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
158 "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
vllm/multimodal/processing.py
+2 -2
@@ -529,9 +529,9 @@ def iter_placeholders(
529 yield placeholder
530
531
532 -class MultiModalProcessor(ABC):
532 +class BaseMultiModalProcessor(ABC):
533 """
534 - Helper class to process multi-modal inputs to be used in vLLM.
534 + Abstract base class to process multi-modal inputs to be used in vLLM.
535 """
536
537 def __init__(
vllm/multimodal/registry.py
+3 -38
@@ -15,7 +15,7 @@ from .audio import AudioPlugin
15 from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
16 from .image import ImagePlugin
17 from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18 -from .processing import MultiModalProcessingMetadata, MultiModalProcessor
18 +from .processing import BaseMultiModalProcessor
19 from .video import VideoPlugin
20
21 if TYPE_CHECKING:
@@ -26,7 +26,7 @@ logger = init_logger(__name__)
26 N = TypeVar("N", bound=Type[nn.Module])
27
28 MultiModalProcessorFactory: TypeAlias = Callable[[InputProcessingContext],
29 - MultiModalProcessor]
29 + BaseMultiModalProcessor]
30 """
31 Constructs a :class:`MultiModalProcessor` instance from the context.
32
@@ -311,41 +311,6 @@ class MultiModalRegistry:
311
312 return wrapper
313
314 - def register_processor_by_metadata(
315 - self,
316 - metadata_factory: Callable[[InputProcessingContext],
317 - MultiModalProcessingMetadata],
318 - get_dummy_mm_kwargs: Callable[
319 - [InputProcessingContext, Mapping[str, int]], MultiModalKwargs],
320 - ):
321 - """
322 - Convenience method to register a multi-modal processor to a model class
323 - according to a function that constructs its metadata.
324 -
325 - When the model receives multi-modal data, the provided function is
326 - invoked to transform the data into a dictionary of model inputs.
327 -
328 - See also:
329 - - :ref:`input_processing_pipeline`
330 - - :ref:`enabling_multimodal_inputs`
331 - """
332 -
333 - class ConcreteMultiModalProcessor(MultiModalProcessor):
334 -
335 - def _get_dummy_mm_kwargs(
336 - self,
337 - mm_counts: Mapping[str, int],
338 - ) -> MultiModalKwargs:
339 - return get_dummy_mm_kwargs(self.ctx, mm_counts)
340 -
341 - def factory(ctx: InputProcessingContext):
342 - return ConcreteMultiModalProcessor(
343 - ctx=ctx,
344 - metadata=metadata_factory(ctx),
345 - )
346 -
347 - return self.register_processor(factory)
348 -
314 def has_processor(self, model_config: "ModelConfig") -> bool:
315 """
316 Test whether a multi-modal processor is defined for a specific model.
@@ -360,7 +325,7 @@ class MultiModalRegistry:
325 self,
326 model_config: "ModelConfig",
327 tokenizer: AnyTokenizer,
363 - ) -> MultiModalProcessor:
328 + ) -> BaseMultiModalProcessor:
329 """
330 Create a multi-modal processor for a specific model and tokenizer.
331 """