@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a86b54fb0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[VLM][Frontend] Proper Image Prompt Formatting from OpenAI API (#6091)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Roger Wang committed Jul 2, 2024 at 23:41 UTC 3a86b54fb00bde01da2680bf3cfd989b6b21511c
1 file changed +27 -10
vllm/entrypoints/openai/serving_chat.py
+27 -10
@@ -127,6 +127,16 @@ class OpenAIServingChat(OpenAIServing):
127
128 return self.tokenizer.decode(image_token_id)
129
130 + # TODO: Let user specify how to insert image tokens into prompt
131 + # (similar to chat template)
132 + def _get_full_image_text_prompt(self, image_token_str: str,
133 + text_prompt: str) -> str:
134 + """Combine image and text prompts for vision language model"""
135 +
136 + # NOTE: For now we assume all model architectures use the same
137 + # image + text prompt format. This may change in the future.
138 + return f"{image_token_str}\n{text_prompt}"
139 +
140 def _parse_chat_message_content_parts(
141 self,
142 role: str,
@@ -146,15 +156,6 @@ class OpenAIServingChat(OpenAIServing):
156 "Multiple 'image_url' input is currently not supported."
157 )
158
149 - image_token_str = self.image_token_str
150 - if image_token_str is not None:
151 - if any(image_token_str in text for text in texts):
152 - logger.warning(
153 - "Detected image token string in the text prompt. "
154 - "Skipping prompt formatting.")
155 - else:
156 - texts.append(image_token_str)
157 -
159 image_url = cast(ChatCompletionContentPartImageParam,
160 part)["image_url"]
161
@@ -169,6 +170,20 @@ class OpenAIServingChat(OpenAIServing):
170 raise NotImplementedError(f"Unknown part type: {part_type}")
171
172 text_prompt = "\n".join(texts)
173 +
174 + if mm_futures:
175 + image_token_str = self.image_token_str
176 + if image_token_str is not None:
177 + if image_token_str in text_prompt:
178 + logger.warning(
179 + "Detected image token string in the text prompt. "
180 + "Skipping prompt formatting.")
181 + else:
182 + text_prompt = self._get_full_image_text_prompt(
183 + image_token_str=image_token_str,
184 + text_prompt=text_prompt,
185 + )
186 +
187 messages = [ConversationMessage(role=role, content=text_prompt)]
188
189 return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
@@ -238,7 +253,9 @@ class OpenAIServingChat(OpenAIServing):
253 try:
254 if len(mm_futures):
255 # since we support only single mm data currently
241 - assert len(mm_futures) == 1
256 + assert len(
257 + mm_futures
258 + ) == 1, "Multiple 'image_url' input is currently not supported."
259 mm_data = await mm_futures[0]
260 except Exception as e:
261 logger.error("Error in loading multi-modal data: %s", e)