A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc][Doc] Add Example of using OpenAI Server with VLM (#5832)
Roger Wang committed
Jun 25, 2024 at 20:34 UTC
3aa7b6cf66890c042ebecf9e8094f4f5e3dbf96e
3 files changed
+101
-3
docs/source/models/vlm.rst
+2
@@ -130,6 +130,8 @@ To consume the server, you can use the OpenAI client like in the example below:
130
)
131
print("Chat response:", chat_response)
132
133
+A full code example can be found in `examples/openai_vision_api_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_vision_api_client.py>`_.
134
+
135
.. note::
136
137
By default, the timeout for fetching images through http url is ``5`` seconds. You can override this by setting the environment variable:
examples/openai_vision_api_client.py
new
+90
@@ -0,0 +1,90 @@
1
+"""An example showing how to use vLLM to serve VLMs.
2
+
3
+Launch the vLLM server with the following command:
4
+python -m vllm.entrypoints.openai.api_server \
5
+ --model llava-hf/llava-1.5-7b-hf \
6
+ --image-input-type pixel_values \
7
+ --image-token-id 32000 \
8
+ --image-input-shape 1,3,336,336 \
9
+ --image-feature-size 576 \
10
+ --chat-template template_llava.jinja
11
+"""
12
+import base64
13
+
14
+import requests
15
+from openai import OpenAI
16
+
17
+# Modify OpenAI's API key and API base to use vLLM's API server.
18
+openai_api_key = "EMPTY"
19
+openai_api_base = "http://localhost:8000/v1"
20
+
21
+client = OpenAI(
22
+ # defaults to os.environ.get("OPENAI_API_KEY")
23
+ api_key=openai_api_key,
24
+ base_url=openai_api_base,
25
+)
26
+
27
+models = client.models.list()
28
+model = models.data[0].id
29
+
30
+image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
31
+
32
+# Use image url in the payload
33
+chat_completion_from_url = client.chat.completions.create(
34
+ messages=[{
35
+ "role":
36
+ "user",
37
+ "content": [
38
+ {
39
+ "type": "text",
40
+ "text": "What’s in this image?"
41
+ },
42
+ {
43
+ "type": "image_url",
44
+ "image_url": {
45
+ "url": image_url
46
+ },
47
+ },
48
+ ],
49
+ }],
50
+ model=model,
51
+)
52
+
53
+result = chat_completion_from_url.choices[0].message.content
54
+print(f"Chat completion output:{result}")
55
+
56
+
57
+# Use base64 encoded image in the payload
58
+def encode_image_base64_from_url(image_url: str) -> str:
59
+ """Encode an image retrieved from a remote url to base64 format."""
60
+
61
+ with requests.get(image_url) as response:
62
+ response.raise_for_status()
63
+ result = base64.b64encode(response.content).decode('utf-8')
64
+
65
+ return result
66
+
67
+
68
+image_base64 = encode_image_base64_from_url(image_url=image_url)
69
+chat_completion_from_base64 = client.chat.completions.create(
70
+ messages=[{
71
+ "role":
72
+ "user",
73
+ "content": [
74
+ {
75
+ "type": "text",
76
+ "text": "What’s in this image?"
77
+ },
78
+ {
79
+ "type": "image_url",
80
+ "image_url": {
81
+ "url": f"data:image/jpeg;base64,{image_base64}"
82
+ },
83
+ },
84
+ ],
85
+ }],
86
+ model=model,
87
+)
88
+
89
+result = chat_completion_from_base64.choices[0].message.content
90
+print(f"Chat completion output:{result}")
vllm/multimodal/utils.py
+9
-3
@@ -1,6 +1,7 @@
1
import base64
2
from io import BytesIO
3
from typing import Optional, Union
4
+from urllib.parse import urlparse
5
6
import aiohttp
7
from PIL import Image
@@ -28,6 +29,10 @@ class ImageFetchAiohttp:
29
"""Load PIL image from a url or base64 encoded openai GPT4V format"""
30
31
if image_url.startswith('http'):
32
+ parsed_url = urlparse(image_url)
33
+ if parsed_url.scheme not in ["http", "https"]:
34
+ raise ValueError("Invalid 'image_url': A valid 'image_url' "
35
+ "must have scheme 'http' or 'https'.")
36
# Avoid circular import
37
from vllm import __version__ as VLLM_VERSION
38
@@ -44,8 +49,9 @@ class ImageFetchAiohttp:
49
image = load_image_from_base64(image_url.split(',', 1)[1])
50
51
else:
47
- raise ValueError("Invalid image url: A valid image url must start "
48
- "with either 'data:image' or 'http'.")
52
+ raise ValueError(
53
+ "Invalid 'image_url': A valid 'image_url' must start "
54
+ "with either 'data:image' or 'http'.")
55
56
return image
57
@@ -56,7 +62,7 @@ async def async_get_and_parse_image(image_url: str) -> ImagePixelData:
62
63
64
def encode_image_base64(image: Image.Image, format: str = 'JPEG') -> str:
59
- """encode image to base64 format."""
65
+ """Encode a pillow image to base64 format."""
66
67
buffered = BytesIO()
68
if format == 'JPEG':