Remove mantis (#46806)
Signed-off-by: Xianbao QIAN <xianbao.qian@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
Tiezhen WANG committed
Jul 1, 2026 at 15:13 UTC
4470ae84de525640a6ed8701a28882ff392e06b6
10 files changed
+8
-203
.buildkite/intel_jobs/models_multimodal_intel.yaml
+5
-7
@@ -22,7 +22,7 @@ steps:
22
commands:
23
- >-
24
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
25
- 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
25
+ 'pip install av &&
26
cd tests &&
27
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
28
pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
@@ -47,8 +47,7 @@ steps:
47
commands:
48
- >-
49
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
50
- 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
51
- cd tests &&
50
+ 'cd tests &&
51
pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
52
53
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
@@ -71,8 +70,7 @@ steps:
70
commands:
71
- >-
72
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
74
- 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
75
- cd tests &&
73
+ 'cd tests &&
74
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
75
pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
76
@@ -96,7 +94,7 @@ steps:
94
commands:
95
- >-
96
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
99
- 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
97
+ 'pip install av &&
98
cd tests &&
99
pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
100
@@ -121,7 +119,7 @@ steps:
119
commands:
120
- >-
121
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
124
- 'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
122
+ 'pip install av matplotlib ftfy &&
123
pip install open-clip-torch --no-deps &&
124
cd tests &&
125
pytest -v -s models/multimodal/processing/test_tensor_schema.py
.buildkite/test-amd.yaml
-14
@@ -239,7 +239,6 @@ steps:
239
- vllm/
240
- tests/models/multimodal
241
commands:
242
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
242
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
243
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
244
@@ -1514,7 +1513,6 @@ steps:
1513
- tests/models/multimodal/generation
1514
- tests/models/multimodal/test_mapping.py
1515
commands:
1517
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1516
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
1517
- pytest -v -s models/multimodal/test_mapping.py
1518
@@ -1528,7 +1526,6 @@ steps:
1526
- vllm/
1527
- tests/models/multimodal/generation
1528
commands:
1531
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1529
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
1530
1531
@@ -1542,7 +1539,6 @@ steps:
1539
- vllm/
1540
- tests/models/multimodal/generation
1541
commands:
1545
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1542
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
1543
1544
- label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
@@ -1556,7 +1552,6 @@ steps:
1552
- vllm/
1553
- tests/models/multimodal
1554
commands:
1559
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1555
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
1556
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
1557
@@ -1572,7 +1567,6 @@ steps:
1567
- tests/models/multimodal/generation
1568
- tests/models/multimodal/test_mapping.py
1569
commands:
1575
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1570
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
1571
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
1572
@@ -1587,7 +1581,6 @@ steps:
1581
- tests/models/multimodal/generation
1582
- tests/models/multimodal/test_mapping.py
1583
commands:
1590
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1584
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
1585
- pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
1586
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
@@ -1603,7 +1596,6 @@ steps:
1596
- tests/models/multimodal
1597
- tests/models/registry.py
1598
commands:
1606
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1599
- pytest -v -s models/multimodal/processing/test_tensor_schema.py
1600
1601
- label: Multi-Modal Processor (CPU) %N # TBD
@@ -1619,7 +1611,6 @@ steps:
1611
- tests/models/multimodal
1612
- tests/models/registry.py
1613
commands:
1622
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
1614
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
1615
1616
#----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------#
@@ -2467,7 +2458,6 @@ steps:
2458
- vllm/
2459
- tests/models/multimodal
2460
commands:
2470
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
2461
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
2462
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
2463
@@ -2978,7 +2968,6 @@ steps:
2968
- tests/models/multimodal/generation
2969
- tests/models/multimodal/test_mapping.py
2970
commands:
2981
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
2971
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
2972
- pytest -v -s models/multimodal/test_mapping.py
2973
@@ -2992,7 +2981,6 @@ steps:
2981
- vllm/
2982
- tests/models/multimodal/generation
2983
commands:
2995
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
2984
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
2985
2986
- label: Multi-Modal Models (Extended Pooling) # TBD
@@ -3018,7 +3006,6 @@ steps:
3006
- vllm/
3007
- tests/models/multimodal
3008
commands:
3021
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
3009
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
3010
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
3011
@@ -3033,7 +3020,6 @@ steps:
3020
- vllm/
3021
- tests/models/multimodal/generation
3022
commands:
3036
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
3023
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
3024
- pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
3025
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
.buildkite/test_areas/models_multimodal.yaml
-9
@@ -10,7 +10,6 @@ steps:
10
- vllm/
11
- tests/models/multimodal
12
commands:
13
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
13
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
14
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
15
mirror:
@@ -27,7 +26,6 @@ steps:
26
- vllm/
27
- tests/models/multimodal
28
commands:
30
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
29
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
30
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
31
mirror:
@@ -44,7 +42,6 @@ steps:
42
- vllm/
43
- tests/models/multimodal
44
commands:
47
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
45
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
46
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
47
mirror:
@@ -61,7 +58,6 @@ steps:
58
- vllm/
59
- tests/models/multimodal
60
commands:
64
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
61
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
62
- pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
63
- pytest models/multimodal/generation/test_memory_leak.py -m core_model
@@ -83,7 +79,6 @@ steps:
79
- tests/models/registry.py
80
device: cpu-medium
81
commands:
86
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
82
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
83
84
- label: Multi-Modal Processor # 44min
@@ -95,7 +90,6 @@ steps:
90
- tests/models/multimodal
91
- tests/models/registry.py
92
commands:
98
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
93
- pytest -v -s models/multimodal/processing/test_tensor_schema.py
94
95
- label: Multi-Modal Accuracy Eval (Small Models) # 50min
@@ -129,7 +123,6 @@ steps:
123
- tests/models/multimodal/generation
124
- tests/models/multimodal/test_mapping.py
125
commands:
132
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
126
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
127
- pytest -v -s models/multimodal/test_mapping.py
128
mirror:
@@ -146,7 +139,6 @@ steps:
139
- vllm/
140
- tests/models/multimodal/generation
141
commands:
149
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
142
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
143
144
- label: Multi-Modal Models (Extended Generation 3)
@@ -157,7 +149,6 @@ steps:
149
- vllm/
150
- tests/models/multimodal/generation
151
commands:
160
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
152
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
153
154
- label: Multi-Modal Models (Extended Pooling)
docs/models/supported_models.md
+1
-4
@@ -579,7 +579,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
579
| `Lfm2VlForConditionalGeneration` | LFM2-VL | T + I<sup>+</sup> | `LiquidAI/LFM2-VL-450M`, `LiquidAI/LFM2-VL-3B`, `LiquidAI/LFM2-VL-8B-A1B`, etc. | ✅︎ | ✅︎ |
580
| `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
581
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
582
-| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
582
+| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
583
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
584
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
585
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I<sup>+</sup> + V<sup>+</sup> | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
@@ -678,9 +678,6 @@ Some models are supported only via the [Transformers modeling backend](#transfor
678
coordinate decoding and are not exposed by this vLLM implementation.
679
See [Moondream3 prompt recipes](../features/multimodal_inputs.md#moondream3-prompt-recipes).
680
681
-!!! note
682
- To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
683
-
681
!!! note
682
The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
683
For more details, please see: <https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630>
examples/generate/multimodal/vision_language_offline.py
-23
@@ -1377,28 +1377,6 @@ def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData
1377
)
1378
1379
1380
-# Mantis
1381
-def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
1382
- assert modality == "image"
1383
-
1384
- llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" # noqa: E501
1385
- prompts = [llama3_template.format(f"{question}\n<image>") for question in questions]
1386
-
1387
- engine_args = EngineArgs(
1388
- model="TIGER-Lab/Mantis-8B-siglip-llama3",
1389
- max_model_len=4096,
1390
- hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
1391
- limit_mm_per_prompt={modality: 1},
1392
- )
1393
- stop_token_ids = [128009]
1394
-
1395
- return ModelRequestData(
1396
- engine_args=engine_args,
1397
- prompts=prompts,
1398
- stop_token_ids=stop_token_ids,
1399
- )
1400
-
1401
-
1380
# MiniCPM-V
1381
def run_minicpmv_base(questions: list[str], modality: str, model_name):
1382
assert modality in ["image", "video", "image+video"]
@@ -2390,7 +2368,6 @@ model_example_map = {
2368
"llava-next": run_llava_next,
2369
"llava-next-video": run_llava_next_video,
2370
"llava-onevision": run_llava_onevision,
2393
- "mantis": run_mantis,
2371
"minicpmo": run_minicpmo,
2372
"minicpmv": run_minicpmv,
2373
"mistral3": run_mistral3,
tests/models/multimodal/generation/test_common.py
-10
@@ -763,16 +763,6 @@ VLM_TEST_SETTINGS = {
763
auto_cls=AutoModelForImageTextToText,
764
vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
765
),
766
- "mantis": VLMTestInfo(
767
- models=["TIGER-Lab/Mantis-8B-siglip-llama3"],
768
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
769
- prompt_formatter=lambda img_prompt: f"<|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
770
- max_model_len=4096,
771
- get_stop_token_ids=lambda tok: [128009],
772
- auto_cls=AutoModelForImageTextToText,
773
- vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
774
- patch_hf_runner=model_utils.mantis_patch_hf_runner,
775
- ),
766
"minicpmv_25": VLMTestInfo(
767
models=["openbmb/MiniCPM-Llama3-V-2_5"],
768
test_type=VLMTestType.IMAGE,
tests/models/multimodal/generation/vlm_utils/model_utils.py
-32
@@ -167,15 +167,6 @@ def llava_onevision_vllm_to_hf_output(
167
return hf_output_ids, hf_output_str, out_logprobs
168
169
170
-def mantis_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput:
171
- """Sanitize vllm output [mantis] to compare with hf output."""
172
- output_ids, output_str, out_logprobs = vllm_output
173
-
174
- hf_output_str = output_str + "<|eot_id|>"
175
-
176
- return output_ids, hf_output_str, out_logprobs
177
-
178
-
170
def phi3v_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput:
171
"""Sanitize vllm output [phi3v] to be comparable with hf output."""
172
_, output_str, out_logprobs = vllm_output
@@ -940,29 +931,6 @@ def _internvl_generate(
931
return outputs
932
933
943
-def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
944
- from mantis.models.mllava import MLlavaProcessor
945
-
946
- hf_model.processor = MLlavaProcessor.from_pretrained(hf_model.model_name)
947
-
948
- orig_generate = hf_model.model.generate
949
- tokenizer = hf_model.processor.tokenizer
950
-
951
- def _generate(self, *args, **kwargs):
952
- return orig_generate(
953
- *args,
954
- **kwargs,
955
- eos_token_id=[
956
- tokenizer.eos_token_id,
957
- tokenizer.convert_tokens_to_ids("<|eot_id|>"),
958
- ],
959
- )
960
-
961
- hf_model.model.generate = types.MethodType(_generate, hf_model.model)
962
-
963
- return hf_model
964
-
965
-
934
def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
935
orig_generate = hf_model.model.generate
936
tests/models/registry.py
-6
@@ -1059,12 +1059,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1059
"LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
1060
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
1061
),
1062
- "MantisForConditionalGeneration": _HfExamplesInfo(
1063
- "TIGER-Lab/Mantis-8B-siglip-llama3",
1064
- max_transformers_version="4.48",
1065
- transformers_version_reason={"hf": "HF model is not compatible."},
1066
- hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
1067
- ),
1062
"MiDashengLMModel": _HfExamplesInfo(
1063
"mispeech/midashenglm-7b", trust_remote_code=True
1064
),
vllm/model_executor/models/llava.py
+1
-97
@@ -20,7 +20,7 @@ from transformers.models.pixtral import PixtralProcessor
20
21
from vllm.config import VllmConfig
22
from vllm.config.multimodal import BaseDummyOptions
23
-from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
23
+from vllm.inputs import MultiModalDataDict
24
from vllm.model_executor.layers.activation import get_act_fn
25
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
26
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -41,11 +41,9 @@ from vllm.multimodal.processing import (
41
BaseMultiModalProcessor,
42
BaseProcessingInfo,
43
InputProcessingContext,
44
- ProcessorInputs,
44
PromptReplacement,
45
PromptUpdate,
46
PromptUpdateDetails,
48
- TimingContext,
47
)
48
from vllm.sequence import IntermediateTensors
49
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -754,97 +752,3 @@ class LlavaForConditionalGeneration(
752
# LLaVA's MLP projector outputs the same number of tokens
753
# as it receives from the vision encoder (1:1 mapping)
754
return num_vision_tokens
757
-
758
-
759
-class MantisProcessingInfo(LlavaProcessingInfo):
760
- def get_hf_processor(self, **kwargs: object):
761
- hf_config = self.get_hf_config()
762
- vision_info = self.get_vision_encoder_info()
763
-
764
- kwargs.setdefault("patch_size", vision_info.get_patch_size())
765
- kwargs.setdefault(
766
- "vision_feature_select_strategy",
767
- hf_config.vision_feature_select_strategy,
768
- )
769
-
770
- return self.ctx.get_hf_processor(LlavaProcessor, **kwargs)
771
-
772
-
773
-class MantisMultiModalProcessor(LlavaMultiModalProcessor):
774
- def apply(
775
- self,
776
- inputs: ProcessorInputs,
777
- timing_ctx: TimingContext,
778
- ) -> MultiModalInput:
779
- hf_config = self.info.get_hf_config()
780
- image_token_id = hf_config.image_token_index
781
-
782
- # Assume that it doesn't depend on the image size
783
- num_image_tokens = self.info.get_num_image_tokens(
784
- image_width=-1,
785
- image_height=-1,
786
- )
787
-
788
- result = super().apply(inputs, timing_ctx)
789
-
790
- mm_item_counts = inputs.mm_data_items.get_all_counts()
791
- mm_kwargs = result["mm_kwargs"]
792
- mm_hashes = result["mm_hashes"]
793
-
794
- # We reimplement the functionality of MLlavaProcessor from
795
- # https://github.com/TIGER-AI-Lab/Mantis.git
796
- def get_replacement_mantis(item_idx: int):
797
- return "".join(
798
- [
799
- f"(image {item_idx + 1}: <Image>", # 7 tokens
800
- "<image>" * num_image_tokens,
801
- "</Image>)", # 3 tokens
802
- ]
803
- )
804
-
805
- mantis_mm_repls = self._bind_and_group_updates(
806
- [
807
- PromptReplacement(
808
- modality="image",
809
- target=[image_token_id] * num_image_tokens,
810
- replacement=get_replacement_mantis,
811
- )
812
- ],
813
- mm_item_counts,
814
- )
815
-
816
- prompt_ids, _ = self._apply_prompt_updates(
817
- result["prompt_token_ids"],
818
- mantis_mm_repls,
819
- )
820
-
821
- orig_repls = self._get_mm_prompt_updates(
822
- inputs.mm_data_items,
823
- inputs.hf_processor_mm_kwargs,
824
- mm_kwargs,
825
- )
826
- mm_placeholders = self._find_mm_placeholders(prompt_ids, orig_repls)
827
- self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
828
-
829
- mm_placeholder_ranges = {
830
- modality: [item.to_range() for item in placeholders]
831
- for modality, placeholders in mm_placeholders.items()
832
- }
833
-
834
- return mm_input(
835
- prompt_token_ids=prompt_ids,
836
- mm_kwargs=mm_kwargs,
837
- mm_hashes=mm_hashes,
838
- mm_placeholders=mm_placeholder_ranges,
839
- )
840
-
841
-
842
-# To use this model, please use
843
-# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
844
-@MULTIMODAL_REGISTRY.register_processor(
845
- MantisMultiModalProcessor,
846
- info=MantisProcessingInfo,
847
- dummy_inputs=LlavaDummyInputsBuilder,
848
-)
849
-class MantisForConditionalGeneration(LlavaForConditionalGeneration):
850
- pass
vllm/model_executor/models/registry.py
+1
-1
@@ -477,7 +477,6 @@ _MULTIMODAL_MODELS = {
477
"llava_onevision",
478
"LlavaOnevisionForConditionalGeneration",
479
),
480
- "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
480
"MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
481
"MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"),
482
"MiniMaxM3SparseForConditionalGeneration": (
@@ -733,6 +732,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
732
"Grok1ForCausalLM": "0.24.0",
733
"TarsierForConditionalGeneration": "0.24.0",
734
"Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4
735
+ "MantisForConditionalGeneration": "0.24.0",
736
}
737
738
_OOT_SUPPORTED_MODELS = {