A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[VLM][Frontend] Proper Image Prompt Formatting from OpenAI API (#6091)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Roger Wang committed
Jul 2, 2024 at 23:41 UTC
3a86b54fb00bde01da2680bf3cfd989b6b21511c
1 file changed
+27
-10
vllm/entrypoints/openai/serving_chat.py
+27
-10
@@ -127,6 +127,16 @@ class OpenAIServingChat(OpenAIServing):
127
128
return self.tokenizer.decode(image_token_id)
129
130
+ # TODO: Let user specify how to insert image tokens into prompt
131
+ # (similar to chat template)
132
+ def _get_full_image_text_prompt(self, image_token_str: str,
133
+ text_prompt: str) -> str:
134
+ """Combine image and text prompts for vision language model"""
135
+
136
+ # NOTE: For now we assume all model architectures use the same
137
+ # image + text prompt format. This may change in the future.
138
+ return f"{image_token_str}\n{text_prompt}"
139
+
140
def _parse_chat_message_content_parts(
141
self,
142
role: str,
@@ -146,15 +156,6 @@ class OpenAIServingChat(OpenAIServing):
156
"Multiple 'image_url' input is currently not supported."
157
)
158
149
- image_token_str = self.image_token_str
150
- if image_token_str is not None:
151
- if any(image_token_str in text for text in texts):
152
- logger.warning(
153
- "Detected image token string in the text prompt. "
154
- "Skipping prompt formatting.")
155
- else:
156
- texts.append(image_token_str)
157
-
159
image_url = cast(ChatCompletionContentPartImageParam,
160
part)["image_url"]
161
@@ -169,6 +170,20 @@ class OpenAIServingChat(OpenAIServing):
170
raise NotImplementedError(f"Unknown part type: {part_type}")
171
172
text_prompt = "\n".join(texts)
173
+
174
+ if mm_futures:
175
+ image_token_str = self.image_token_str
176
+ if image_token_str is not None:
177
+ if image_token_str in text_prompt:
178
+ logger.warning(
179
+ "Detected image token string in the text prompt. "
180
+ "Skipping prompt formatting.")
181
+ else:
182
+ text_prompt = self._get_full_image_text_prompt(
183
+ image_token_str=image_token_str,
184
+ text_prompt=text_prompt,
185
+ )
186
+
187
messages = [ConversationMessage(role=role, content=text_prompt)]
188
189
return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
@@ -238,7 +253,9 @@ class OpenAIServingChat(OpenAIServing):
253
try:
254
if len(mm_futures):
255
# since we support only single mm data currently
241
- assert len(mm_futures) == 1
256
+ assert len(
257
+ mm_futures
258
+ ) == 1, "Multiple 'image_url' input is currently not supported."
259
mm_data = await mm_futures[0]
260
except Exception as e:
261
logger.error("Error in loading multi-modal data: %s", e)