A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 19:00 UTC
Add multimodal input method in the documentation (#31601)
Signed-off-by: xiaoming <1259730330@qq.com>
labAxiaoming committed
Jan 2, 2026 at 20:43 UTC
a01f2faedfab1fa7f73ed0ca396370791c20daa3
2 files changed
+96
docs/features/multimodal_inputs.md
+33
@@ -506,6 +506,7 @@ Then, you can use the OpenAI client as follows:
506
??? code
507
508
```python
509
+ import os
510
from openai import OpenAI
511
512
openai_api_key = "EMPTY"
@@ -517,8 +518,11 @@ Then, you can use the OpenAI client as follows:
518
)
519
520
# Single-image input inference
521
+
522
+ # Public image URL for testing remote image processing
523
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
524
525
+ # Create chat completion with remote image
526
chat_response = client.chat.completions.create(
527
model="microsoft/Phi-3.5-vision-instruct",
528
messages=[
@@ -542,6 +546,35 @@ Then, you can use the OpenAI client as follows:
546
)
547
print("Chat completion output:", chat_response.choices[0].message.content)
548
549
+ # Local image file path (update this to point to your actual image file)
550
+ image_file = "/path/to/image.jpg"
551
+
552
+ # Create chat completion with local image file
553
+ # Launch the API server/engine with the --allowed-local-media-path argument.
554
+ if os.path.exists(image_file):
555
+ chat_completion_from_local_image_url = client.chat.completions.create(
556
+ model="microsoft/Phi-3.5-vision-instruct",
557
+ messages=[
558
+ {
559
+ "role": "user",
560
+ "content": [
561
+ {
562
+ "type": "text",
563
+ "text": "What’s in this image?",
564
+ },
565
+ {
566
+ "type": "image_url",
567
+ "image_url": {"url": f"file://{image_file}"},
568
+ },
569
+ ],
570
+ }
571
+ ],
572
+ )
573
+ result = chat_completion_from_local_image_url.choices[0].message.content
574
+ print("Chat completion output from local image file:\n", result)
575
+ else:
576
+ print(f"Local image file not found at {image_file}, skipping local file test.")
577
+
578
# Multi-image input inference
579
image_url_duck = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/duck.jpg"
580
image_url_lion = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/lion.jpg"
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+63
@@ -21,6 +21,7 @@ python openai_chat_completion_client_for_multimodal.py --chat-type audio
21
"""
22
23
import base64
24
+import os
25
26
import requests
27
from openai import OpenAI
@@ -51,6 +52,16 @@ def encode_base64_content_from_url(content_url: str) -> str:
52
return result
53
54
55
+def encode_base64_content_from_file(file_path: str) -> str:
56
+ """Encode a local file content to base64 format."""
57
+
58
+ with open(file_path, "rb") as file:
59
+ file_content = file.read()
60
+ result = base64.b64encode(file_content).decode("utf-8")
61
+
62
+ return result
63
+
64
+
65
# Text-only inference
66
def run_text_only(model: str, max_completion_tokens: int) -> None:
67
chat_completion = client.chat.completions.create(
@@ -67,6 +78,7 @@ def run_text_only(model: str, max_completion_tokens: int) -> None:
78
def run_single_image(model: str, max_completion_tokens: int) -> None:
79
## Use image url in the payload
80
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
81
+ image_file = "/path/to/image.jpg" # local file
82
chat_completion_from_url = client.chat.completions.create(
83
messages=[
84
{
@@ -87,6 +99,30 @@ def run_single_image(model: str, max_completion_tokens: int) -> None:
99
result = chat_completion_from_url.choices[0].message.content
100
print("Chat completion output from image url:\n", result)
101
102
+ ## Use local image url in the payload
103
+ # Launch the API server/engine with the --allowed-local-media-path argument.
104
+ if os.path.exists(image_file):
105
+ chat_completion_from_local_image_url = client.chat.completions.create(
106
+ messages=[
107
+ {
108
+ "role": "user",
109
+ "content": [
110
+ {"type": "text", "text": "What's in this image?"},
111
+ {
112
+ "type": "image_url",
113
+ "image_url": {"url": f"file://{image_file}"},
114
+ },
115
+ ],
116
+ }
117
+ ],
118
+ model=model,
119
+ max_completion_tokens=max_completion_tokens,
120
+ )
121
+ result = chat_completion_from_local_image_url.choices[0].message.content
122
+ print("Chat completion output from local image file:\n", result)
123
+ else:
124
+ print(f"Local image file not found at {image_file}, skipping local file test.")
125
+
126
## Use base64 encoded image in the payload
127
image_base64 = encode_base64_content_from_url(image_url)
128
chat_completion_from_base64 = client.chat.completions.create(
@@ -109,6 +145,33 @@ def run_single_image(model: str, max_completion_tokens: int) -> None:
145
result = chat_completion_from_base64.choices[0].message.content
146
print("Chat completion output from base64 encoded image:", result)
147
148
+ ## Use base64 encoded local image in the payload
149
+ if os.path.exists(image_file):
150
+ local_image_base64 = encode_base64_content_from_file(image_file)
151
+ chat_completion_from_local_image_base64 = client.chat.completions.create(
152
+ messages=[
153
+ {
154
+ "role": "user",
155
+ "content": [
156
+ {"type": "text", "text": "What's in this image?"},
157
+ {
158
+ "type": "image_url",
159
+ "image_url": {
160
+ "url": f"data:image/jpeg;base64,{local_image_base64}"
161
+ },
162
+ },
163
+ ],
164
+ }
165
+ ],
166
+ model=model,
167
+ max_completion_tokens=max_completion_tokens,
168
+ )
169
+
170
+ result = chat_completion_from_local_image_base64.choices[0].message.content
171
+ print("Chat completion output from base64 encoded local image:", result)
172
+ else:
173
+ print(f"Local image file not found at {image_file}, skipping local file test.")
174
+
175
176
# Multi-image input inference
177
def run_multi_image(model: str, max_completion_tokens: int) -> None: