@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3aa7b6cf66
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc][Doc] Add Example of using OpenAI Server with VLM (#5832)

Roger Wang committed Jun 25, 2024 at 20:34 UTC 3aa7b6cf66890c042ebecf9e8094f4f5e3dbf96e
3 files changed +101 -3
docs/source/models/vlm.rst
+2
@@ -130,6 +130,8 @@ To consume the server, you can use the OpenAI client like in the example below:
130 )
131 print("Chat response:", chat_response)
132
133 +A full code example can be found in `examples/openai_vision_api_client.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_vision_api_client.py>`_.
134 +
135 .. note::
136
137 By default, the timeout for fetching images through http url is ``5`` seconds. You can override this by setting the environment variable:
examples/openai_vision_api_client.py new
+90
@@ -0,0 +1,90 @@
1 +"""An example showing how to use vLLM to serve VLMs.
2 +
3 +Launch the vLLM server with the following command:
4 +python -m vllm.entrypoints.openai.api_server \
5 + --model llava-hf/llava-1.5-7b-hf \
6 + --image-input-type pixel_values \
7 + --image-token-id 32000 \
8 + --image-input-shape 1,3,336,336 \
9 + --image-feature-size 576 \
10 + --chat-template template_llava.jinja
11 +"""
12 +import base64
13 +
14 +import requests
15 +from openai import OpenAI
16 +
17 +# Modify OpenAI's API key and API base to use vLLM's API server.
18 +openai_api_key = "EMPTY"
19 +openai_api_base = "http://localhost:8000/v1"
20 +
21 +client = OpenAI(
22 + # defaults to os.environ.get("OPENAI_API_KEY")
23 + api_key=openai_api_key,
24 + base_url=openai_api_base,
25 +)
26 +
27 +models = client.models.list()
28 +model = models.data[0].id
29 +
30 +image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
31 +
32 +# Use image url in the payload
33 +chat_completion_from_url = client.chat.completions.create(
34 + messages=[{
35 + "role":
36 + "user",
37 + "content": [
38 + {
39 + "type": "text",
40 + "text": "What’s in this image?"
41 + },
42 + {
43 + "type": "image_url",
44 + "image_url": {
45 + "url": image_url
46 + },
47 + },
48 + ],
49 + }],
50 + model=model,
51 +)
52 +
53 +result = chat_completion_from_url.choices[0].message.content
54 +print(f"Chat completion output:{result}")
55 +
56 +
57 +# Use base64 encoded image in the payload
58 +def encode_image_base64_from_url(image_url: str) -> str:
59 + """Encode an image retrieved from a remote url to base64 format."""
60 +
61 + with requests.get(image_url) as response:
62 + response.raise_for_status()
63 + result = base64.b64encode(response.content).decode('utf-8')
64 +
65 + return result
66 +
67 +
68 +image_base64 = encode_image_base64_from_url(image_url=image_url)
69 +chat_completion_from_base64 = client.chat.completions.create(
70 + messages=[{
71 + "role":
72 + "user",
73 + "content": [
74 + {
75 + "type": "text",
76 + "text": "What’s in this image?"
77 + },
78 + {
79 + "type": "image_url",
80 + "image_url": {
81 + "url": f"data:image/jpeg;base64,{image_base64}"
82 + },
83 + },
84 + ],
85 + }],
86 + model=model,
87 +)
88 +
89 +result = chat_completion_from_base64.choices[0].message.content
90 +print(f"Chat completion output:{result}")
vllm/multimodal/utils.py
+9 -3
@@ -1,6 +1,7 @@
1 import base64
2 from io import BytesIO
3 from typing import Optional, Union
4 +from urllib.parse import urlparse
5
6 import aiohttp
7 from PIL import Image
@@ -28,6 +29,10 @@ class ImageFetchAiohttp:
29 """Load PIL image from a url or base64 encoded openai GPT4V format"""
30
31 if image_url.startswith('http'):
32 + parsed_url = urlparse(image_url)
33 + if parsed_url.scheme not in ["http", "https"]:
34 + raise ValueError("Invalid 'image_url': A valid 'image_url' "
35 + "must have scheme 'http' or 'https'.")
36 # Avoid circular import
37 from vllm import __version__ as VLLM_VERSION
38
@@ -44,8 +49,9 @@ class ImageFetchAiohttp:
49 image = load_image_from_base64(image_url.split(',', 1)[1])
50
51 else:
47 - raise ValueError("Invalid image url: A valid image url must start "
48 - "with either 'data:image' or 'http'.")
52 + raise ValueError(
53 + "Invalid 'image_url': A valid 'image_url' must start "
54 + "with either 'data:image' or 'http'.")
55
56 return image
57
@@ -56,7 +62,7 @@ async def async_get_and_parse_image(image_url: str) -> ImagePixelData:
62
63
64 def encode_image_base64(image: Image.Image, format: str = 'JPEG') -> str:
59 - """encode image to base64 format."""
65 + """Encode a pillow image to base64 format."""
66
67 buffered = BytesIO()
68 if format == 'JPEG':