@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 757ac70a64
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] Rename MiniCPMVQwen2 to MiniCPMV2.6 (#7273)

Jee Jee Li committed Aug 8, 2024 at 22:02 UTC 757ac70a64b5a643b68281c0b65f72f847cedbd6
3 files changed +51 -31
docs/source/models/supported_models.rst
+1 -1
@@ -222,7 +222,7 @@ Vision Language Models
222 -
223 * - :code:`MiniCPMV`
224 - MiniCPM-V
225 - - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
225 + - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
226 -
227
228 .. note::
examples/offline_inference_vision_language.py
+35 -16
@@ -22,8 +22,8 @@ def run_llava(question):
22 prompt = f"USER: <image>\n{question}\nASSISTANT:"
23
24 llm = LLM(model="llava-hf/llava-1.5-7b-hf")
25 -
26 - return llm, prompt
25 + stop_token_ids = None
26 + return llm, prompt, stop_token_ids
27
28
29 # LLaVA-1.6/LLaVA-NeXT
@@ -31,8 +31,8 @@ def run_llava_next(question):
31
32 prompt = f"[INST] <image>\n{question} [/INST]"
33 llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf")
34 -
35 - return llm, prompt
34 + stop_token_ids = None
35 + return llm, prompt, stop_token_ids
36
37
38 # Fuyu
@@ -40,8 +40,8 @@ def run_fuyu(question):
40
41 prompt = f"{question}\n"
42 llm = LLM(model="adept/fuyu-8b")
43 -
44 - return llm, prompt
43 + stop_token_ids = None
44 + return llm, prompt, stop_token_ids
45
46
47 # Phi-3-Vision
@@ -59,7 +59,8 @@ def run_phi3v(question):
59 trust_remote_code=True,
60 max_num_seqs=5,
61 )
62 - return llm, prompt
62 + stop_token_ids = None
63 + return llm, prompt, stop_token_ids
64
65
66 # PaliGemma
@@ -68,8 +69,8 @@ def run_paligemma(question):
69 # PaliGemma has special prompt format for VQA
70 prompt = "caption en"
71 llm = LLM(model="google/paligemma-3b-mix-224")
71 -
72 - return llm, prompt
72 + stop_token_ids = None
73 + return llm, prompt, stop_token_ids
74
75
76 # Chameleon
@@ -77,7 +78,8 @@ def run_chameleon(question):
78
79 prompt = f"{question}<image>"
80 llm = LLM(model="facebook/chameleon-7b")
80 - return llm, prompt
81 + stop_token_ids = None
82 + return llm, prompt, stop_token_ids
83
84
85 # MiniCPM-V
@@ -89,13 +91,26 @@ def run_minicpmv(question):
91 # model_name = "HwwwH/MiniCPM-V-2"
92
93 # 2.5
92 - model_name = "openbmb/MiniCPM-Llama3-V-2_5"
94 + # model_name = "openbmb/MiniCPM-Llama3-V-2_5"
95 +
96 + #2.6
97 + model_name = "openbmb/MiniCPM-V-2_6"
98 tokenizer = AutoTokenizer.from_pretrained(model_name,
99 trust_remote_code=True)
100 llm = LLM(
101 model=model_name,
102 trust_remote_code=True,
103 )
104 + # NOTE The stop_token_ids are different for various versions of MiniCPM-V
105 + # 2.0
106 + # stop_token_ids = [tokenizer.eos_id]
107 +
108 + # 2.5
109 + # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
110 +
111 + # 2.6
112 + stop_tokens = ['<|im_end|>', '<|endoftext|>']
113 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
114
115 messages = [{
116 'role': 'user',
@@ -104,7 +119,7 @@ def run_minicpmv(question):
119 prompt = tokenizer.apply_chat_template(messages,
120 tokenize=False,
121 add_generation_prompt=True)
107 - return llm, prompt
122 + return llm, prompt, stop_token_ids
123
124
125 # InternVL
@@ -118,7 +133,8 @@ def run_internvl(question):
133 trust_remote_code=True,
134 max_num_seqs=5,
135 )
121 - return llm, prompt
136 + stop_token_ids = None
137 + return llm, prompt, stop_token_ids
138
139
140 # BLIP-2
@@ -128,7 +144,8 @@ def run_blip2(question):
144 # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
145 prompt = f"Question: {question} Answer:"
146 llm = LLM(model="Salesforce/blip2-opt-2.7b")
131 - return llm, prompt
147 + stop_token_ids = None
148 + return llm, prompt, stop_token_ids
149
150
151 model_example_map = {
@@ -149,11 +166,13 @@ def main(args):
166 if model not in model_example_map:
167 raise ValueError(f"Model type {model} is not supported.")
168
152 - llm, prompt = model_example_map[model](question)
169 + llm, prompt, stop_token_ids = model_example_map[model](question)
170
171 # We set temperature to 0.2 so that outputs can be different
172 # even when all prompts are identical when running batch inference.
156 - sampling_params = SamplingParams(temperature=0.2, max_tokens=64)
173 + sampling_params = SamplingParams(temperature=0.2,
174 + max_tokens=64,
175 + stop_token_ids=stop_token_ids)
176
177 assert args.num_prompts > 0
178 if args.num_prompts == 1:
vllm/model_executor/models/minicpmv.py
+15 -14
@@ -216,7 +216,6 @@ class BaseResampler(nn.Module):
216
217 self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
218 trunc_normal_(self.query, std=0.02)
219 -
219 if kv_dim is not None and kv_dim != embed_dim:
220 self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
221 else:
@@ -225,7 +224,6 @@ class BaseResampler(nn.Module):
224 nn.Identity()(*args, **kwargs),
225 None,
226 )
228 -
227 self.attn = nn.MultiheadAttention(embed_dim, num_heads)
228 self.ln_q = norm_layer(embed_dim)
229 self.ln_kv = norm_layer(embed_dim)
@@ -261,7 +259,6 @@ class Resampler2(BaseResampler):
259 norm_layer)
260
261 self.adaptive = adaptive
264 -
262 pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
263 grid_size,
264 version=(2, 0))
@@ -717,7 +714,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsVision):
714 raise NotImplementedError
715
716
720 -class MiniCPMV2(MiniCPMVBaseModel):
717 +class MiniCPMV2_0(MiniCPMVBaseModel):
718
719 def __init__(
720 self,
@@ -890,10 +887,7 @@ class MiniCPMV2_5(MiniCPMVBaseModel):
887 return "resampler" in name
888
889
893 -# NOTE: Currently, information about this model is unavailable. We are
894 -# temporarily using `MiniCPMVQwen2` as it's name. The name may need
895 -# to be modified in the future.
896 -class MiniCPMVQwen2(MiniCPMVBaseModel):
890 +class MiniCPMV2_6(MiniCPMVBaseModel):
891
892 def __init__(
893 self,
@@ -903,6 +897,7 @@ class MiniCPMVQwen2(MiniCPMVBaseModel):
897 quant_config: Optional[QuantizationConfig] = None,
898 ):
899 super().__init__(config, multimodal_config, cache_config, quant_config)
900 + assert self.version == (2, 6)
901
902 def init_llm(
903 self,
@@ -930,6 +925,7 @@ class MiniCPMVQwen2(MiniCPMVBaseModel):
925
926 def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
927 with set_default_torch_dtype(torch.float16):
928 + # The resampler in 2.6 remains consistent with the one in 2.5.
929 resampler = Resampler2_5(
930 num_queries=self.config.query_num,
931 embed_dim=embed_dim,
@@ -989,6 +985,13 @@ class MiniCPMVQwen2(MiniCPMVBaseModel):
985 return "resampler" in name or "vpm" in name
986
987
988 +_SUPPORT_VERSION = {
989 + (2, 0): MiniCPMV2_0,
990 + (2, 5): MiniCPMV2_5,
991 + (2, 6): MiniCPMV2_6
992 +}
993 +
994 +
995 @MULTIMODAL_REGISTRY.register_image_input_mapper()
996 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
997 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
@@ -1016,11 +1019,9 @@ class MiniCPMV(MiniCPMVBaseModel):
1019 version = str(config.version).split(".")
1020 version = tuple([int(x) for x in version])
1021 # Dispatch class based on version
1019 - if version == (2, 0):
1020 - instance_class = MiniCPMV2
1021 - elif version == (2, 5):
1022 - instance_class = MiniCPMV2_5
1023 - else:
1024 - instance_class = MiniCPMVQwen2
1022 + instance_class = _SUPPORT_VERSION.get(version, None)
1023 + if instance_class is None:
1024 + raise ValueError(
1025 + "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
1026 return instance_class(config, multimodal_config, cache_config,
1027 quant_config)