@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b75e314fff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Add image placeholder for OpenAI Compatible Server of MiniCPM-V (#6787)

Co-authored-by: hezhihui <hzh7269@modelbest.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Alphi committed Jul 26, 2024 at 00:42 UTC b75e314fff29bdc94b2fb1dd78519e92f9520e65
2 files changed +5 -1
examples/minicpmv_example.py
+2
@@ -4,6 +4,8 @@ from vllm import LLM, SamplingParams
4 from vllm.assets.image import ImageAsset
5
6 # 2.0
7 +# The official repo doesn't work yet, so we need to use a fork for now
8 +# For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
9 # MODEL_NAME = "HwwwH/MiniCPM-V-2"
10 # 2.5
11 MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5"
vllm/entrypoints/chat_utils.py
+3 -1
@@ -100,7 +100,9 @@ def _image_token_str(model_config: ModelConfig,
100 if model_type == "phi3_v":
101 # Workaround since this token is not defined in the tokenizer
102 return "<|image_1|>"
103 - if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv", "paligemma"):
103 + if model_type == "minicpmv":
104 + return "(<image>./</image>)"
105 + if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
106 # These models do not use image tokens in the prompt
107 return None
108 if model_type.startswith("llava"):