A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Add image placeholder for OpenAI Compatible Server of MiniCPM-V (#6787)
Co-authored-by: hezhihui <hzh7269@modelbest.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Alphi committed
Jul 26, 2024 at 00:42 UTC
b75e314fff29bdc94b2fb1dd78519e92f9520e65
2 files changed
+5
-1
examples/minicpmv_example.py
+2
@@ -4,6 +4,8 @@ from vllm import LLM, SamplingParams
4
from vllm.assets.image import ImageAsset
5
6
# 2.0
7
+# The official repo doesn't work yet, so we need to use a fork for now
8
+# For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
9
# MODEL_NAME = "HwwwH/MiniCPM-V-2"
10
# 2.5
11
MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5"
vllm/entrypoints/chat_utils.py
+3
-1
@@ -100,7 +100,9 @@ def _image_token_str(model_config: ModelConfig,
100
if model_type == "phi3_v":
101
# Workaround since this token is not defined in the tokenizer
102
return "<|image_1|>"
103
- if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv", "paligemma"):
103
+ if model_type == "minicpmv":
104
+ return "(<image>./</image>)"
105
+ if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
106
# These models do not use image tokens in the prompt
107
return None
108
if model_type.startswith("llava"):