A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] Rename MiniCPMVQwen2 to MiniCPMV2.6 (#7273)
Jee Jee Li committed
Aug 8, 2024 at 22:02 UTC
757ac70a64b5a643b68281c0b65f72f847cedbd6
3 files changed
+51
-31
docs/source/models/supported_models.rst
+1
-1
@@ -222,7 +222,7 @@ Vision Language Models
222
-
223
* - :code:`MiniCPMV`
224
- MiniCPM-V
225
- - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
225
+ - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
226
-
227
228
.. note::
examples/offline_inference_vision_language.py
+35
-16
@@ -22,8 +22,8 @@ def run_llava(question):
22
prompt = f"USER: <image>\n{question}\nASSISTANT:"
23
24
llm = LLM(model="llava-hf/llava-1.5-7b-hf")
25
-
26
- return llm, prompt
25
+ stop_token_ids = None
26
+ return llm, prompt, stop_token_ids
27
28
29
# LLaVA-1.6/LLaVA-NeXT
@@ -31,8 +31,8 @@ def run_llava_next(question):
31
32
prompt = f"[INST] <image>\n{question} [/INST]"
33
llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf")
34
-
35
- return llm, prompt
34
+ stop_token_ids = None
35
+ return llm, prompt, stop_token_ids
36
37
38
# Fuyu
@@ -40,8 +40,8 @@ def run_fuyu(question):
40
41
prompt = f"{question}\n"
42
llm = LLM(model="adept/fuyu-8b")
43
-
44
- return llm, prompt
43
+ stop_token_ids = None
44
+ return llm, prompt, stop_token_ids
45
46
47
# Phi-3-Vision
@@ -59,7 +59,8 @@ def run_phi3v(question):
59
trust_remote_code=True,
60
max_num_seqs=5,
61
)
62
- return llm, prompt
62
+ stop_token_ids = None
63
+ return llm, prompt, stop_token_ids
64
65
66
# PaliGemma
@@ -68,8 +69,8 @@ def run_paligemma(question):
69
# PaliGemma has special prompt format for VQA
70
prompt = "caption en"
71
llm = LLM(model="google/paligemma-3b-mix-224")
71
-
72
- return llm, prompt
72
+ stop_token_ids = None
73
+ return llm, prompt, stop_token_ids
74
75
76
# Chameleon
@@ -77,7 +78,8 @@ def run_chameleon(question):
78
79
prompt = f"{question}<image>"
80
llm = LLM(model="facebook/chameleon-7b")
80
- return llm, prompt
81
+ stop_token_ids = None
82
+ return llm, prompt, stop_token_ids
83
84
85
# MiniCPM-V
@@ -89,13 +91,26 @@ def run_minicpmv(question):
91
# model_name = "HwwwH/MiniCPM-V-2"
92
93
# 2.5
92
- model_name = "openbmb/MiniCPM-Llama3-V-2_5"
94
+ # model_name = "openbmb/MiniCPM-Llama3-V-2_5"
95
+
96
+ #2.6
97
+ model_name = "openbmb/MiniCPM-V-2_6"
98
tokenizer = AutoTokenizer.from_pretrained(model_name,
99
trust_remote_code=True)
100
llm = LLM(
101
model=model_name,
102
trust_remote_code=True,
103
)
104
+ # NOTE The stop_token_ids are different for various versions of MiniCPM-V
105
+ # 2.0
106
+ # stop_token_ids = [tokenizer.eos_id]
107
+
108
+ # 2.5
109
+ # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
110
+
111
+ # 2.6
112
+ stop_tokens = ['<|im_end|>', '<|endoftext|>']
113
+ stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
114
115
messages = [{
116
'role': 'user',
@@ -104,7 +119,7 @@ def run_minicpmv(question):
119
prompt = tokenizer.apply_chat_template(messages,
120
tokenize=False,
121
add_generation_prompt=True)
107
- return llm, prompt
122
+ return llm, prompt, stop_token_ids
123
124
125
# InternVL
@@ -118,7 +133,8 @@ def run_internvl(question):
133
trust_remote_code=True,
134
max_num_seqs=5,
135
)
121
- return llm, prompt
136
+ stop_token_ids = None
137
+ return llm, prompt, stop_token_ids
138
139
140
# BLIP-2
@@ -128,7 +144,8 @@ def run_blip2(question):
144
# See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
145
prompt = f"Question: {question} Answer:"
146
llm = LLM(model="Salesforce/blip2-opt-2.7b")
131
- return llm, prompt
147
+ stop_token_ids = None
148
+ return llm, prompt, stop_token_ids
149
150
151
model_example_map = {
@@ -149,11 +166,13 @@ def main(args):
166
if model not in model_example_map:
167
raise ValueError(f"Model type {model} is not supported.")
168
152
- llm, prompt = model_example_map[model](question)
169
+ llm, prompt, stop_token_ids = model_example_map[model](question)
170
171
# We set temperature to 0.2 so that outputs can be different
172
# even when all prompts are identical when running batch inference.
156
- sampling_params = SamplingParams(temperature=0.2, max_tokens=64)
173
+ sampling_params = SamplingParams(temperature=0.2,
174
+ max_tokens=64,
175
+ stop_token_ids=stop_token_ids)
176
177
assert args.num_prompts > 0
178
if args.num_prompts == 1:
vllm/model_executor/models/minicpmv.py
+15
-14
@@ -216,7 +216,6 @@ class BaseResampler(nn.Module):
216
217
self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
218
trunc_normal_(self.query, std=0.02)
219
-
219
if kv_dim is not None and kv_dim != embed_dim:
220
self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
221
else:
@@ -225,7 +224,6 @@ class BaseResampler(nn.Module):
224
nn.Identity()(*args, **kwargs),
225
None,
226
)
228
-
227
self.attn = nn.MultiheadAttention(embed_dim, num_heads)
228
self.ln_q = norm_layer(embed_dim)
229
self.ln_kv = norm_layer(embed_dim)
@@ -261,7 +259,6 @@ class Resampler2(BaseResampler):
259
norm_layer)
260
261
self.adaptive = adaptive
264
-
262
pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
263
grid_size,
264
version=(2, 0))
@@ -717,7 +714,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsVision):
714
raise NotImplementedError
715
716
720
-class MiniCPMV2(MiniCPMVBaseModel):
717
+class MiniCPMV2_0(MiniCPMVBaseModel):
718
719
def __init__(
720
self,
@@ -890,10 +887,7 @@ class MiniCPMV2_5(MiniCPMVBaseModel):
887
return "resampler" in name
888
889
893
-# NOTE: Currently, information about this model is unavailable. We are
894
-# temporarily using `MiniCPMVQwen2` as it's name. The name may need
895
-# to be modified in the future.
896
-class MiniCPMVQwen2(MiniCPMVBaseModel):
890
+class MiniCPMV2_6(MiniCPMVBaseModel):
891
892
def __init__(
893
self,
@@ -903,6 +897,7 @@ class MiniCPMVQwen2(MiniCPMVBaseModel):
897
quant_config: Optional[QuantizationConfig] = None,
898
):
899
super().__init__(config, multimodal_config, cache_config, quant_config)
900
+ assert self.version == (2, 6)
901
902
def init_llm(
903
self,
@@ -930,6 +925,7 @@ class MiniCPMVQwen2(MiniCPMVBaseModel):
925
926
def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
927
with set_default_torch_dtype(torch.float16):
928
+ # The resampler in 2.6 remains consistent with the one in 2.5.
929
resampler = Resampler2_5(
930
num_queries=self.config.query_num,
931
embed_dim=embed_dim,
@@ -989,6 +985,13 @@ class MiniCPMVQwen2(MiniCPMVBaseModel):
985
return "resampler" in name or "vpm" in name
986
987
988
+_SUPPORT_VERSION = {
989
+ (2, 0): MiniCPMV2_0,
990
+ (2, 5): MiniCPMV2_5,
991
+ (2, 6): MiniCPMV2_6
992
+}
993
+
994
+
995
@MULTIMODAL_REGISTRY.register_image_input_mapper()
996
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
997
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
@@ -1016,11 +1019,9 @@ class MiniCPMV(MiniCPMVBaseModel):
1019
version = str(config.version).split(".")
1020
version = tuple([int(x) for x in version])
1021
# Dispatch class based on version
1019
- if version == (2, 0):
1020
- instance_class = MiniCPMV2
1021
- elif version == (2, 5):
1022
- instance_class = MiniCPMV2_5
1023
- else:
1024
- instance_class = MiniCPMVQwen2
1022
+ instance_class = _SUPPORT_VERSION.get(version, None)
1023
+ if instance_class is None:
1024
+ raise ValueError(
1025
+ "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1026
return instance_class(config, multimodal_config, cache_config,
1027
quant_config)