A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Misc] Consolidate example usage of OpenAI client for multimodal models (#9412)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Roger Wang committed
Oct 16, 2024 at 04:20 UTC
59230ef32b0b9132ea9a6ea39d8e823574657a87
4 files changed
+237
-217
docs/source/models/vlm.rst
+1
-1
@@ -241,7 +241,7 @@ To consume the server, you can use the OpenAI client like in the example below:
241
print("Chat completion output:", chat_response.choices[0].message.content)
242
243
244
-A full code example can be found in `examples/openai_vision_api_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_vision_api_client.py>`_.
244
+A full code example can be found in `examples/openai_api_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_api_client_for_multimodal.py>`_.
245
246
.. note::
247
examples/openai_api_client_for_multimodal.py
new
+236
@@ -0,0 +1,236 @@
1
+"""An example showing how to use vLLM to serve multimodal models
2
+and run online inference with OpenAI client.
3
+
4
+Launch the vLLM server with the following command:
5
+
6
+(single image inference with Llava)
7
+vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
8
+
9
+(multi-image inference with Phi-3.5-vision-instruct)
10
+vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
11
+ --trust-remote-code --limit-mm-per-prompt image=2
12
+
13
+(audio inference with Ultravox)
14
+vllm serve fixie-ai/ultravox-v0_3 --max-model-len 4096
15
+"""
16
+import base64
17
+
18
+import requests
19
+from openai import OpenAI
20
+
21
+from vllm.assets.audio import AudioAsset
22
+from vllm.utils import FlexibleArgumentParser
23
+
24
+# Modify OpenAI's API key and API base to use vLLM's API server.
25
+openai_api_key = "EMPTY"
26
+openai_api_base = "http://localhost:8000/v1"
27
+
28
+client = OpenAI(
29
+ # defaults to os.environ.get("OPENAI_API_KEY")
30
+ api_key=openai_api_key,
31
+ base_url=openai_api_base,
32
+)
33
+
34
+models = client.models.list()
35
+model = models.data[0].id
36
+
37
+
38
+def encode_base64_content_from_url(content_url: str) -> str:
39
+ """Encode a content retrieved from a remote url to base64 format."""
40
+
41
+ with requests.get(content_url) as response:
42
+ response.raise_for_status()
43
+ result = base64.b64encode(response.content).decode('utf-8')
44
+
45
+ return result
46
+
47
+
48
+# Text-only inference
49
+def run_text_only() -> None:
50
+ chat_completion = client.chat.completions.create(
51
+ messages=[{
52
+ "role": "user",
53
+ "content": "What's the capital of France?"
54
+ }],
55
+ model=model,
56
+ max_tokens=64,
57
+ )
58
+
59
+ result = chat_completion.choices[0].message.content
60
+ print("Chat completion output:", result)
61
+
62
+
63
+# Single-image input inference
64
+def run_single_image() -> None:
65
+
66
+ ## Use image url in the payload
67
+ image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
68
+ chat_completion_from_url = client.chat.completions.create(
69
+ messages=[{
70
+ "role":
71
+ "user",
72
+ "content": [
73
+ {
74
+ "type": "text",
75
+ "text": "What's in this image?"
76
+ },
77
+ {
78
+ "type": "image_url",
79
+ "image_url": {
80
+ "url": image_url
81
+ },
82
+ },
83
+ ],
84
+ }],
85
+ model=model,
86
+ max_tokens=64,
87
+ )
88
+
89
+ result = chat_completion_from_url.choices[0].message.content
90
+ print("Chat completion output from image url:", result)
91
+
92
+ ## Use base64 encoded image in the payload
93
+ image_base64 = encode_base64_content_from_url(image_url)
94
+ chat_completion_from_base64 = client.chat.completions.create(
95
+ messages=[{
96
+ "role":
97
+ "user",
98
+ "content": [
99
+ {
100
+ "type": "text",
101
+ "text": "What's in this image?"
102
+ },
103
+ {
104
+ "type": "image_url",
105
+ "image_url": {
106
+ "url": f"data:image/jpeg;base64,{image_base64}"
107
+ },
108
+ },
109
+ ],
110
+ }],
111
+ model=model,
112
+ max_tokens=64,
113
+ )
114
+
115
+ result = chat_completion_from_base64.choices[0].message.content
116
+ print("Chat completion output from base64 encoded image:", result)
117
+
118
+
119
+# Multi-image input inference
120
+def run_multi_image() -> None:
121
+ image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
122
+ image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
123
+ chat_completion_from_url = client.chat.completions.create(
124
+ messages=[{
125
+ "role":
126
+ "user",
127
+ "content": [
128
+ {
129
+ "type": "text",
130
+ "text": "What are the animals in these images?"
131
+ },
132
+ {
133
+ "type": "image_url",
134
+ "image_url": {
135
+ "url": image_url_duck
136
+ },
137
+ },
138
+ {
139
+ "type": "image_url",
140
+ "image_url": {
141
+ "url": image_url_lion
142
+ },
143
+ },
144
+ ],
145
+ }],
146
+ model=model,
147
+ max_tokens=64,
148
+ )
149
+
150
+ result = chat_completion_from_url.choices[0].message.content
151
+ print("Chat completion output:", result)
152
+
153
+
154
+# Audio input inference
155
+def run_audio() -> None:
156
+ # Any format supported by librosa is supported
157
+ audio_url = AudioAsset("winning_call").url
158
+
159
+ # Use audio url in the payload
160
+ chat_completion_from_url = client.chat.completions.create(
161
+ messages=[{
162
+ "role":
163
+ "user",
164
+ "content": [
165
+ {
166
+ "type": "text",
167
+ "text": "What's in this audio?"
168
+ },
169
+ {
170
+ "type": "audio_url",
171
+ "audio_url": {
172
+ "url": audio_url
173
+ },
174
+ },
175
+ ],
176
+ }],
177
+ model=model,
178
+ max_tokens=64,
179
+ )
180
+
181
+ result = chat_completion_from_url.choices[0].message.content
182
+ print("Chat completion output from audio url:", result)
183
+
184
+ audio_base64 = encode_base64_content_from_url(audio_url)
185
+ chat_completion_from_base64 = client.chat.completions.create(
186
+ messages=[{
187
+ "role":
188
+ "user",
189
+ "content": [
190
+ {
191
+ "type": "text",
192
+ "text": "What's in this audio?"
193
+ },
194
+ {
195
+ "type": "audio_url",
196
+ "audio_url": {
197
+ # Any format supported by librosa is supported
198
+ "url": f"data:audio/ogg;base64,{audio_base64}"
199
+ },
200
+ },
201
+ ],
202
+ }],
203
+ model=model,
204
+ max_tokens=64,
205
+ )
206
+
207
+ result = chat_completion_from_base64.choices[0].message.content
208
+ print("Chat completion output from base64 encoded audio:", result)
209
+
210
+
211
+example_function_map = {
212
+ "text-only": run_text_only,
213
+ "single-image": run_single_image,
214
+ "multi-image": run_multi_image,
215
+ "audio": run_audio,
216
+}
217
+
218
+
219
+def main(args) -> None:
220
+ chat_type = args.chat_type
221
+ example_function_map[chat_type]()
222
+
223
+
224
+if __name__ == "__main__":
225
+ parser = FlexibleArgumentParser(
226
+ description='Demo on using OpenAI client for online inference with '
227
+ 'multimodal language models served with vLLM.')
228
+ parser.add_argument(
229
+ '--chat-type',
230
+ '-c',
231
+ type=str,
232
+ default="single-image",
233
+ choices=["text-only", "single-image", "multi-image", "audio"],
234
+ help='Conversation type with multimodal data.')
235
+ args = parser.parse_args()
236
+ main(args)
examples/openai_audio_api_client.py
deleted
-90
@@ -1,90 +0,0 @@
1
-"""An example showing how to use vLLM to serve VLMs.
2
-
3
-Launch the vLLM server with the following command:
4
-vllm serve fixie-ai/ultravox-v0_3
5
-"""
6
-import base64
7
-
8
-import requests
9
-from openai import OpenAI
10
-
11
-from vllm.assets.audio import AudioAsset
12
-
13
-# Modify OpenAI's API key and API base to use vLLM's API server.
14
-openai_api_key = "EMPTY"
15
-openai_api_base = "http://localhost:8000/v1"
16
-
17
-client = OpenAI(
18
- # defaults to os.environ.get("OPENAI_API_KEY")
19
- api_key=openai_api_key,
20
- base_url=openai_api_base,
21
-)
22
-
23
-models = client.models.list()
24
-model = models.data[0].id
25
-
26
-# Any format supported by librosa is supported
27
-audio_url = AudioAsset("winning_call").url
28
-
29
-# Use audio url in the payload
30
-chat_completion_from_url = client.chat.completions.create(
31
- messages=[{
32
- "role":
33
- "user",
34
- "content": [
35
- {
36
- "type": "text",
37
- "text": "What's in this audio?"
38
- },
39
- {
40
- "type": "audio_url",
41
- "audio_url": {
42
- "url": audio_url
43
- },
44
- },
45
- ],
46
- }],
47
- model=model,
48
- max_tokens=64,
49
-)
50
-
51
-result = chat_completion_from_url.choices[0].message.content
52
-print(f"Chat completion output:{result}")
53
-
54
-
55
-# Use base64 encoded audio in the payload
56
-def encode_audio_base64_from_url(audio_url: str) -> str:
57
- """Encode an audio retrieved from a remote url to base64 format."""
58
-
59
- with requests.get(audio_url) as response:
60
- response.raise_for_status()
61
- result = base64.b64encode(response.content).decode('utf-8')
62
-
63
- return result
64
-
65
-
66
-audio_base64 = encode_audio_base64_from_url(audio_url=audio_url)
67
-chat_completion_from_base64 = client.chat.completions.create(
68
- messages=[{
69
- "role":
70
- "user",
71
- "content": [
72
- {
73
- "type": "text",
74
- "text": "What's in this audio?"
75
- },
76
- {
77
- "type": "audio_url",
78
- "audio_url": {
79
- # Any format supported by librosa is supported
80
- "url": f"data:audio/ogg;base64,{audio_base64}"
81
- },
82
- },
83
- ],
84
- }],
85
- model=model,
86
- max_tokens=64,
87
-)
88
-
89
-result = chat_completion_from_base64.choices[0].message.content
90
-print(f"Chat completion output:{result}")
examples/openai_vision_api_client.py
deleted
-126
@@ -1,126 +0,0 @@
1
-"""An example showing how to use vLLM to serve VLMs.
2
-
3
-Launch the vLLM server with the following command:
4
-
5
-(single image inference with Llava)
6
-vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
7
-
8
-(multi-image inference with Phi-3.5-vision-instruct)
9
-vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
10
- --trust-remote-code --limit-mm-per-prompt image=2
11
-"""
12
-import base64
13
-
14
-import requests
15
-from openai import OpenAI
16
-
17
-# Modify OpenAI's API key and API base to use vLLM's API server.
18
-openai_api_key = "EMPTY"
19
-openai_api_base = "http://localhost:8000/v1"
20
-
21
-client = OpenAI(
22
- # defaults to os.environ.get("OPENAI_API_KEY")
23
- api_key=openai_api_key,
24
- base_url=openai_api_base,
25
-)
26
-
27
-models = client.models.list()
28
-model = models.data[0].id
29
-
30
-# Single-image input inference
31
-image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
32
-
33
-## Use image url in the payload
34
-chat_completion_from_url = client.chat.completions.create(
35
- messages=[{
36
- "role":
37
- "user",
38
- "content": [
39
- {
40
- "type": "text",
41
- "text": "What's in this image?"
42
- },
43
- {
44
- "type": "image_url",
45
- "image_url": {
46
- "url": image_url
47
- },
48
- },
49
- ],
50
- }],
51
- model=model,
52
- max_tokens=64,
53
-)
54
-
55
-result = chat_completion_from_url.choices[0].message.content
56
-print("Chat completion output:", result)
57
-
58
-
59
-## Use base64 encoded image in the payload
60
-def encode_image_base64_from_url(image_url: str) -> str:
61
- """Encode an image retrieved from a remote url to base64 format."""
62
-
63
- with requests.get(image_url) as response:
64
- response.raise_for_status()
65
- result = base64.b64encode(response.content).decode('utf-8')
66
-
67
- return result
68
-
69
-
70
-image_base64 = encode_image_base64_from_url(image_url=image_url)
71
-chat_completion_from_base64 = client.chat.completions.create(
72
- messages=[{
73
- "role":
74
- "user",
75
- "content": [
76
- {
77
- "type": "text",
78
- "text": "What's in this image?"
79
- },
80
- {
81
- "type": "image_url",
82
- "image_url": {
83
- "url": f"data:image/jpeg;base64,{image_base64}"
84
- },
85
- },
86
- ],
87
- }],
88
- model=model,
89
- max_tokens=64,
90
-)
91
-
92
-result = chat_completion_from_base64.choices[0].message.content
93
-print(f"Chat completion output:{result}")
94
-
95
-# Multi-image input inference
96
-image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
97
-image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
98
-chat_completion_from_url = client.chat.completions.create(
99
- messages=[{
100
- "role":
101
- "user",
102
- "content": [
103
- {
104
- "type": "text",
105
- "text": "What are the animals in these images?"
106
- },
107
- {
108
- "type": "image_url",
109
- "image_url": {
110
- "url": image_url_duck
111
- },
112
- },
113
- {
114
- "type": "image_url",
115
- "image_url": {
116
- "url": image_url_lion
117
- },
118
- },
119
- ],
120
- }],
121
- model=model,
122
- max_tokens=64,
123
-)
124
-
125
-result = chat_completion_from_url.choices[0].message.content
126
-print("Chat completion output:", result)