A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Update multi-modal processor to support Mantis(LLaVA) model (#10711)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 8, 2024 at 01:10 UTC
39e227c7ae3149eb8345ea1a1ffee672ef76c09a
14 files changed
+175
-78
.buildkite/test-pipeline.yaml
+2
@@ -362,6 +362,7 @@ steps:
362
- tests/models/embedding/vision_language
363
- tests/models/encoder_decoder/vision_language
364
commands:
365
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
366
- pytest -v -s models/decoder_only/audio_language -m 'core_model or quant_model'
367
- pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language -m 'core_model or quant_model'
368
- pytest -v -s models/embedding/vision_language -m core_model
@@ -377,6 +378,7 @@ steps:
378
- tests/models/embedding/vision_language
379
- tests/models/encoder_decoder/vision_language
380
commands:
381
+ - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
382
- pytest -v -s models/decoder_only/audio_language -m 'not core_model and not quant_model'
383
# HACK - run phi3v tests separately to sidestep this transformers bug
384
# https://github.com/huggingface/transformers/issues/34307
docs/source/models/supported_models.rst
+5
-1
@@ -555,7 +555,7 @@ Text Generation
555
* - :code:`LlavaForConditionalGeneration`
556
- LLaVA-1.5
557
- T + I\ :sup:`E+`
558
- - :code:`llava-hf/llava-1.5-7b-hf`, :code:`llava-hf/llava-1.5-13b-hf`, etc.
558
+ - :code:`llava-hf/llava-1.5-7b-hf`, :code:`TIGER-Lab/Mantis-8B-siglip-llama3` (see note), etc.
559
-
560
- ✅︎
561
* - :code:`LlavaNextForConditionalGeneration`
@@ -664,6 +664,10 @@ Text Generation
664
.. note::
665
vLLM currently only supports adding LoRA to the language backbone of multimodal models.
666
667
+.. note::
668
+ To use :code:`TIGER-Lab/Mantis-8B-siglip-llama3`, you have to install their GitHub repo (:code:`pip install git+https://github.com/TIGER-AI-Lab/Mantis.git`)
669
+ and pass :code:`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
670
+
671
.. note::
672
The official :code:`openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
673
For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
examples/offline_inference_vision_language.py
+17
@@ -419,6 +419,22 @@ def run_aria(question: str, modality: str):
419
return llm, prompt, stop_token_ids
420
421
422
+# Mantis
423
+def run_mantis(question: str, modality: str):
424
+ assert modality == "image"
425
+
426
+ llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
427
+ prompt = llama3_template.format(f"{question}\n<image>")
428
+
429
+ llm = LLM(
430
+ model="TIGER-Lab/Mantis-8B-siglip-llama3",
431
+ max_model_len=4096,
432
+ hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
433
+ )
434
+ stop_token_ids = [128009]
435
+ return llm, prompt, stop_token_ids
436
+
437
+
438
model_example_map = {
439
"llava": run_llava,
440
"llava-next": run_llava_next,
@@ -441,6 +457,7 @@ model_example_map = {
457
"glm4v": run_glm4v,
458
"idefics3": run_idefics3,
459
"aria": run_aria,
460
+ "mantis": run_mantis,
461
}
462
463
requirements-test.in
-3
@@ -24,9 +24,6 @@ mistral_common[opencv] >= 1.5.0 # required for pixtral test
24
datamodel_code_generator # required for minicpm3 test
25
lm-eval[api]==0.4.4 # required for model evaluation test
26
27
-# TODO: Add this after fully implementing llava(mantis)
28
-# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
29
-
27
# quantization
28
bitsandbytes>=0.44.0
29
buildkite-test-collector==0.1.9
tests/models/decoder_only/vision_language/test_models.py
+22
-8
@@ -34,7 +34,7 @@ COMMON_BROADCAST_SETTINGS = {
34
"dtype": "half",
35
"max_tokens": 5,
36
"tensor_parallel_size": 2,
37
- "model_kwargs": {"device_map": "auto"},
37
+ "hf_model_kwargs": {"device_map": "auto"},
38
"image_size_factors": [(.25, 0.5, 1.0)],
39
"distributed_executor_backend": (
40
"ray",
@@ -108,7 +108,7 @@ VLM_TEST_SETTINGS = {
108
"cherry_blossom": "What is in the picture?",
109
}),
110
auto_cls=AutoModelForVision2Seq,
111
- postprocess_inputs=model_utils.get_key_type_post_processor(
111
+ postprocess_inputs=model_utils.cast_dtype_post_processor(
112
"pixel_values"
113
),
114
vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
@@ -151,7 +151,7 @@ VLM_TEST_SETTINGS = {
151
"cherry_blossom": "<vlm_image>Please infer the season with reason.",
152
}),
153
multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
154
- postprocess_inputs=model_utils.get_key_type_post_processor("pixel_values"),
154
+ postprocess_inputs=model_utils.cast_dtype_post_processor("pixel_values"),
155
stop_str=["<|im_end|>"],
156
image_size_factors=[(0.10, 0.15)],
157
max_tokens=64,
@@ -177,7 +177,7 @@ VLM_TEST_SETTINGS = {
177
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
178
max_model_len=4096,
179
auto_cls=AutoModelForVision2Seq,
180
- postprocess_inputs=model_utils.get_key_type_post_processor(
180
+ postprocess_inputs=model_utils.cast_dtype_post_processor(
181
"pixel_values"
182
),
183
# For chameleon, we only compare the sequences
@@ -281,7 +281,7 @@ VLM_TEST_SETTINGS = {
281
prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
282
num_video_frames=16,
283
max_model_len=16384,
284
- postprocess_inputs=model_utils.get_key_type_post_processor(
284
+ postprocess_inputs=model_utils.cast_dtype_post_processor(
285
"pixel_values_videos"
286
),
287
auto_cls=AutoModelForVision2Seq,
@@ -306,6 +306,20 @@ VLM_TEST_SETTINGS = {
306
vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
307
image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
308
),
309
+ "mantis": VLMTestInfo(
310
+ models=["TIGER-Lab/Mantis-8B-siglip-llama3"],
311
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
312
+ prompt_formatter=lambda img_prompt: f"<|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
313
+ max_model_len=4096,
314
+ postprocess_inputs=model_utils.cast_dtype_post_processor(
315
+ "pixel_values"
316
+ ),
317
+ vllm_runner_kwargs={"hf_overrides": {"architectures": ["MantisForConditionalGeneration"]}}, # noqa: E501
318
+ get_stop_token_ids=lambda tok: [128009],
319
+ auto_cls=AutoModelForVision2Seq,
320
+ vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
321
+ patch_hf_runner=model_utils.mantis_patch_hf_runner,
322
+ ),
323
"minicpmv_25": VLMTestInfo(
324
models=["openbmb/MiniCPM-Llama3-V-2_5"],
325
test_type=VLMTestType.IMAGE,
@@ -342,7 +356,7 @@ VLM_TEST_SETTINGS = {
356
# max_num_seqs=2,
357
# task="generate",
358
# # use eager mode for hf runner since phi3v didn't work with flash_attn
345
- # model_kwargs={"_attn_implementation": "eager"},
359
+ # hf_model_kwargs={"_attn_implementation": "eager"},
360
# use_tokenizer_eos=True,
361
# vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
362
# num_logprobs=10,
@@ -373,7 +387,7 @@ VLM_TEST_SETTINGS = {
387
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
388
max_model_len=4096,
389
auto_cls=AutoModelForVision2Seq,
376
- postprocess_inputs=model_utils.get_key_type_post_processor(
390
+ postprocess_inputs=model_utils.cast_dtype_post_processor(
391
"pixel_values"
392
),
393
vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
@@ -438,7 +452,7 @@ VLM_TEST_SETTINGS = {
452
test_type=VLMTestType.CUSTOM_INPUTS,
453
max_model_len=16384,
454
max_num_seqs=2,
441
- postprocess_inputs=model_utils.get_key_type_post_processor(
455
+ postprocess_inputs=model_utils.cast_dtype_post_processor(
456
"pixel_values"
457
),
458
auto_cls=AutoModelForVision2Seq,
tests/models/decoder_only/vision_language/vlm_utils/core.py
+14
-6
@@ -3,9 +3,11 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
3
4
import torch
5
from PIL.Image import Image
6
-from transformers import AutoTokenizer, BatchEncoding
6
+from transformers import AutoTokenizer, BatchEncoding, PreTrainedTokenizerBase
7
from transformers.models.auto.auto_factory import _BaseAutoModelClass
8
9
+from vllm.config import TaskOption
10
+
11
from .....conftest import HfRunner, VllmRunner
12
from .types import RunnerOutput
13
@@ -28,13 +30,15 @@ def run_test(
30
use_tokenizer_eos: bool,
31
postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
32
comparator: Callable[..., None],
31
- get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]],
33
+ get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
34
+ List[int]]],
35
stop_str: Optional[List[str]],
36
tokenizer_mode: str,
37
limit_mm_per_prompt: Dict[str, int],
35
- model_kwargs: Optional[Dict[str, Any]],
38
+ vllm_runner_kwargs: Optional[Dict[str, Any]],
39
+ hf_model_kwargs: Optional[Dict[str, Any]],
40
patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]],
37
- task: str = "auto",
41
+ task: TaskOption = "auto",
42
runner_mm_key: str = "images",
43
distributed_executor_backend: Optional[str] = None,
44
tensor_parallel_size: int = 1,
@@ -58,6 +62,9 @@ def run_test(
62
if stop_str:
63
vllm_kwargs["stop"] = stop_str
64
65
+ if vllm_runner_kwargs is None:
66
+ vllm_runner_kwargs = {}
67
+
68
with vllm_runner(model,
69
tokenizer_mode=tokenizer_mode,
70
max_model_len=max_model_len,
@@ -67,7 +74,8 @@ def run_test(
74
tensor_parallel_size=tensor_parallel_size,
75
distributed_executor_backend=distributed_executor_backend,
76
enforce_eager=enforce_eager,
70
- task=task) as vllm_model:
77
+ task=task,
78
+ **vllm_runner_kwargs) as vllm_model:
79
for prompts, media in vllm_inputs:
80
vllm_kwargs[runner_mm_key] = media
81
vllm_output = vllm_model.generate_greedy_logprobs(
@@ -78,7 +86,7 @@ def run_test(
86
dtype=dtype,
87
auto_cls=auto_cls,
88
postprocess_inputs=postprocess_inputs,
81
- model_kwargs=model_kwargs)
89
+ model_kwargs=hf_model_kwargs)
90
91
# Some models need to patch things like the model processor, e.g., internvl
92
if patch_hf_runner is not None:
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+34
-1
@@ -126,6 +126,16 @@ def llava_onevision_vllm_to_hf_output(vllm_output: RunnerOutput,
126
return hf_output_ids, hf_output_str, out_logprobs
127
128
129
+def mantis_vllm_to_hf_output(vllm_output: RunnerOutput,
130
+ model: str) -> RunnerOutput:
131
+ """Sanitize vllm output [mantis] to compare with hf output."""
132
+ output_ids, output_str, out_logprobs = vllm_output
133
+
134
+ hf_output_str = output_str + "<|eot_id|>"
135
+
136
+ return output_ids, hf_output_str, out_logprobs
137
+
138
+
139
def phi3v_vllm_to_hf_output(vllm_output: RunnerOutput,
140
model: str) -> RunnerOutput:
141
"""Sanitize vllm output [phi3v] to be comparable with hf output."""
@@ -184,7 +194,7 @@ def get_llava_embeddings(image_assets: _ImageAssets):
194
195
196
####### postprocessors to run on HF BatchEncoding
187
-def get_key_type_post_processor(
197
+def cast_dtype_post_processor(
198
hf_inp_key: str) -> Callable[[BatchEncoding, str], BatchEncoding]:
199
"""Gets a handle to a post processor which converts a given key into a
200
target data type."""
@@ -418,3 +428,26 @@ def _internvl_generate(
428
)
429
430
return outputs
431
+
432
+
433
+def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
434
+ from mantis.models.mllava import MLlavaProcessor
435
+
436
+ hf_model.processor = MLlavaProcessor.from_pretrained(hf_model.model_name)
437
+
438
+ orig_generate = hf_model.model.generate
439
+ tokenizer = hf_model.processor.tokenizer
440
+
441
+ def _generate(self, *args, **kwargs):
442
+ return orig_generate(
443
+ *args,
444
+ **kwargs,
445
+ eos_token_id=[
446
+ tokenizer.eos_token_id,
447
+ tokenizer.convert_tokens_to_ids("<|eot_id|>"),
448
+ ],
449
+ )
450
+
451
+ hf_model.model.generate = types.MethodType(_generate, hf_model.model)
452
+
453
+ return hf_model
tests/models/decoder_only/vision_language/vlm_utils/types.py
+12
-7
@@ -7,9 +7,11 @@ from typing import (Any, Callable, Dict, Iterable, List, NamedTuple, Optional,
7
import torch
8
from PIL.Image import Image
9
from pytest import MarkDecorator
10
-from transformers import AutoModelForCausalLM, AutoTokenizer, BatchEncoding
10
+from transformers import (AutoModelForCausalLM, BatchEncoding,
11
+ PreTrainedTokenizerBase)
12
from transformers.models.auto.auto_factory import _BaseAutoModelClass
13
14
+from vllm.config import TaskOption
15
from vllm.sequence import SampleLogprobs
16
from vllm.utils import identity
17
@@ -66,7 +68,7 @@ class ImageSizeWrapper(NamedTuple):
68
class VLMTestInfo(NamedTuple):
69
"""Holds the configuration for 1+ tests for one model architecture."""
70
69
- models: Union[List[str]]
71
+ models: List[str]
72
test_type: Union[VLMTestType, Iterable[VLMTestType]]
73
74
# Should be None only if this is a CUSTOM_INPUTS test
@@ -92,18 +94,20 @@ class VLMTestInfo(NamedTuple):
94
enforce_eager: bool = True
95
max_model_len: int = 1024
96
max_num_seqs: int = 256
95
- task: str = "auto"
97
+ task: TaskOption = "auto"
98
tensor_parallel_size: int = 1
99
+ vllm_runner_kwargs: Optional[Dict[str, Any]] = None
100
101
# Optional callable which gets a list of token IDs from the model tokenizer
99
- get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]] = None
102
+ get_stop_token_ids: Optional[Callable[[PreTrainedTokenizerBase],
103
+ List[int]]] = None
104
# Optional list of strings to stop generation, useful when stop tokens are
105
# not special tokens in the tokenizer
106
stop_str: Optional[List[str]] = None
107
108
# Exposed options for HF runner
105
- model_kwargs: Optional[Dict[str, Any]] = None
106
- # Indicates we should explicitly pass the EOS from the tokeniezr
109
+ hf_model_kwargs: Optional[Dict[str, Any]] = None
110
+ # Indicates we should explicitly pass the EOS from the tokenizer
111
use_tokenizer_eos: bool = False
112
auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM
113
# Callable to pass to the HF runner to run on inputs; for now, we also pass
@@ -164,6 +168,7 @@ class VLMTestInfo(NamedTuple):
168
"max_num_seqs": self.max_num_seqs,
169
"task": self.task,
170
"tensor_parallel_size": self.tensor_parallel_size,
171
+ "vllm_runner_kwargs": self.vllm_runner_kwargs,
172
"hf_output_post_proc": self.hf_output_post_proc,
173
"vllm_output_post_proc": self.vllm_output_post_proc,
174
"auto_cls": self.auto_cls,
@@ -171,8 +176,8 @@ class VLMTestInfo(NamedTuple):
176
"postprocess_inputs": self.postprocess_inputs,
177
"comparator": self.comparator,
178
"get_stop_token_ids": self.get_stop_token_ids,
179
+ "hf_model_kwargs": self.hf_model_kwargs,
180
"stop_str": self.stop_str,
175
- "model_kwargs": self.model_kwargs,
181
"patch_hf_runner": self.patch_hf_runner,
182
"tokenizer_mode": self.tokenizer_mode
183
}
tests/models/registry.py
+1
@@ -176,6 +176,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
176
"LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
177
"LlavaNextVideoForConditionalGeneration": _HfExamplesInfo("llava-hf/LLaVA-NeXT-Video-7B-hf"), # noqa: E501
178
"LlavaOnevisionForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
179
+ "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3"), # noqa: E501
180
"MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
181
trust_remote_code=True),
182
"MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+2
-4
@@ -3,16 +3,14 @@ from typing import Optional
3
import torch
4
5
from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6
- create_metadata_for_llava,
7
- dummy_mm_kwargs_for_llava,
6
+ LlavaProcessor,
7
get_max_llava_image_tokens)
8
from vllm.model_executor.sampling_metadata import SamplingMetadata
9
from vllm.multimodal import MULTIMODAL_REGISTRY
10
11
12
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
14
-@MULTIMODAL_REGISTRY.register_processor_by_metadata(create_metadata_for_llava,
15
- dummy_mm_kwargs_for_llava)
13
+@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
14
class MyLlava(LlavaForConditionalGeneration):
15
16
def compute_logits(
vllm/model_executor/models/llava.py
+60
-8
@@ -22,10 +22,11 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
25
-from vllm.multimodal.processing import (InputProcessingContext,
25
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
26
+ InputProcessingContext,
27
ModalityProcessingMetadata,
28
MultiModalProcessingMetadata,
28
- MultiModalProcessor, PromptReplacement)
29
+ PromptReplacement)
30
from vllm.sequence import IntermediateTensors
31
32
from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -163,7 +164,13 @@ def create_metadata_for_llava(
164
}
165
166
166
-class LlavaProcessor(MultiModalProcessor):
167
+class LlavaProcessor(BaseMultiModalProcessor):
168
+
169
+ def __init__(self, ctx: InputProcessingContext) -> None:
170
+ super().__init__(
171
+ ctx=ctx,
172
+ metadata=create_metadata_for_llava(ctx),
173
+ )
174
175
def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
176
if getattr(hf_processor, "__is_patched__", False):
@@ -193,7 +200,30 @@ class LlavaProcessor(MultiModalProcessor):
200
self,
201
mm_counts: Mapping[str, int],
202
) -> MultiModalKwargs:
196
- return dummy_mm_kwargs_for_llava(self.ctx, mm_counts)
203
+ hf_config = self.ctx.get_hf_config(LlavaConfig)
204
+ vision_config = hf_config.vision_config
205
+ num_images = mm_counts["image"]
206
+
207
+ if isinstance(vision_config, CLIPVisionConfig):
208
+ data = dummy_image_for_clip(vision_config, num_images)
209
+ elif isinstance(vision_config, SiglipVisionConfig):
210
+ data = dummy_image_for_siglip(vision_config, num_images)
211
+ elif isinstance(vision_config, PixtralVisionConfig):
212
+ data = dummy_image_for_pixtral_hf(vision_config, num_images)
213
+ else:
214
+ msg = f"Unsupported vision config: {type(vision_config)}"
215
+ raise NotImplementedError(msg)
216
+
217
+ hf_processor = self._get_hf_processor()
218
+ image_processor = hf_processor.image_processor # type: ignore
219
+ hf_inputs = image_processor.preprocess(data['image'],
220
+ return_tensors="pt")
221
+ is_pixtral = isinstance(hf_processor, PixtralProcessor)
222
+
223
+ return MultiModalKwargs(
224
+ **hf_inputs,
225
+ is_pixtral=torch.tensor(is_pixtral),
226
+ )
227
228
229
class LlavaLikeConfig(Protocol):
@@ -277,10 +307,7 @@ def init_vision_tower_for_llava(
307
308
309
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
280
-@MULTIMODAL_REGISTRY.register_processor(lambda ctx: LlavaProcessor(
281
- ctx=ctx,
282
- metadata=create_metadata_for_llava(ctx),
283
-))
310
+@MULTIMODAL_REGISTRY.register_processor(LlavaProcessor)
311
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
312
# BitandBytes specific attributes
313
bitsandbytes_stacked_params_mapping = {
@@ -559,3 +586,28 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
586
torch.Tensor]]) -> Set[str]:
587
loader = AutoWeightsLoader(self)
588
return loader.load_weights(weights)
589
+
590
+
591
+class MantisProcessor(LlavaProcessor):
592
+
593
+ def _get_hf_processor(self) -> ProcessorMixin:
594
+ try:
595
+ from mantis.models.mllava import MLlavaProcessor
596
+ except ModuleNotFoundError as exc:
597
+ raise ModuleNotFoundError(
598
+ "You need to `pip install "
599
+ "git+https://github.com/TIGER-AI-Lab/Mantis.git` "
600
+ "to use this model") from exc
601
+
602
+ processor = MLlavaProcessor.from_pretrained(
603
+ self.ctx.model_config.tokenizer)
604
+ assert isinstance(processor, ProcessorMixin)
605
+ return processor
606
+
607
+
608
+# To use this model, please use
609
+# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
610
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
611
+@MULTIMODAL_REGISTRY.register_processor(MantisProcessor)
612
+class MantisForConditionalGeneration(LlavaForConditionalGeneration):
613
+ pass
vllm/model_executor/models/registry.py
+1
@@ -152,6 +152,7 @@ _MULTIMODAL_MODELS = {
152
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
153
"LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
154
"LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
155
+ "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"), # noqa: E501
156
"MiniCPMV": ("minicpmv", "MiniCPMV"),
157
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
158
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
vllm/multimodal/processing.py
+2
-2
@@ -529,9 +529,9 @@ def iter_placeholders(
529
yield placeholder
530
531
532
-class MultiModalProcessor(ABC):
532
+class BaseMultiModalProcessor(ABC):
533
"""
534
- Helper class to process multi-modal inputs to be used in vLLM.
534
+ Abstract base class to process multi-modal inputs to be used in vLLM.
535
"""
536
537
def __init__(
vllm/multimodal/registry.py
+3
-38
@@ -15,7 +15,7 @@ from .audio import AudioPlugin
15
from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
16
from .image import ImagePlugin
17
from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18
-from .processing import MultiModalProcessingMetadata, MultiModalProcessor
18
+from .processing import BaseMultiModalProcessor
19
from .video import VideoPlugin
20
21
if TYPE_CHECKING:
@@ -26,7 +26,7 @@ logger = init_logger(__name__)
26
N = TypeVar("N", bound=Type[nn.Module])
27
28
MultiModalProcessorFactory: TypeAlias = Callable[[InputProcessingContext],
29
- MultiModalProcessor]
29
+ BaseMultiModalProcessor]
30
"""
31
Constructs a :class:`MultiModalProcessor` instance from the context.
32
@@ -311,41 +311,6 @@ class MultiModalRegistry:
311
312
return wrapper
313
314
- def register_processor_by_metadata(
315
- self,
316
- metadata_factory: Callable[[InputProcessingContext],
317
- MultiModalProcessingMetadata],
318
- get_dummy_mm_kwargs: Callable[
319
- [InputProcessingContext, Mapping[str, int]], MultiModalKwargs],
320
- ):
321
- """
322
- Convenience method to register a multi-modal processor to a model class
323
- according to a function that constructs its metadata.
324
-
325
- When the model receives multi-modal data, the provided function is
326
- invoked to transform the data into a dictionary of model inputs.
327
-
328
- See also:
329
- - :ref:`input_processing_pipeline`
330
- - :ref:`enabling_multimodal_inputs`
331
- """
332
-
333
- class ConcreteMultiModalProcessor(MultiModalProcessor):
334
-
335
- def _get_dummy_mm_kwargs(
336
- self,
337
- mm_counts: Mapping[str, int],
338
- ) -> MultiModalKwargs:
339
- return get_dummy_mm_kwargs(self.ctx, mm_counts)
340
-
341
- def factory(ctx: InputProcessingContext):
342
- return ConcreteMultiModalProcessor(
343
- ctx=ctx,
344
- metadata=metadata_factory(ctx),
345
- )
346
-
347
- return self.register_processor(factory)
348
-
314
def has_processor(self, model_config: "ModelConfig") -> bool:
315
"""
316
Test whether a multi-modal processor is defined for a specific model.
@@ -360,7 +325,7 @@ class MultiModalRegistry:
325
self,
326
model_config: "ModelConfig",
327
tokenizer: AnyTokenizer,
363
- ) -> MultiModalProcessor:
328
+ ) -> BaseMultiModalProcessor:
329
"""
330
Create a multi-modal processor for a specific model and tokenizer.
331
"""