A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
chore(examples-docs): upgrade to OpenAI V1 (#1785)
Massimiliano Pronesti committed
Dec 3, 2023 at 10:11 UTC
c07a4428545c7708626ab04bb0033284e71d4ed7
3 files changed
+47
-28
docs/source/getting_started/quickstart.rst
+19
-8
@@ -157,11 +157,16 @@ Since this server is compatible with OpenAI API, you can use it as a drop-in rep
157
158
.. code-block:: python
159
160
- import openai
160
+ from openai import OpenAI
161
+
162
# Modify OpenAI's API key and API base to use vLLM's API server.
162
- openai.api_key = "EMPTY"
163
- openai.api_base = "http://localhost:8000/v1"
164
- completion = openai.Completion.create(model="facebook/opt-125m",
163
+ openai_api_key = "EMPTY"
164
+ openai_api_base = "http://localhost:8000/v1"
165
+ client = OpenAI(
166
+ api_key=openai_api_key,
167
+ base_url=openai_api_base,
168
+ )
169
+ completion = client.completions.create(model="facebook/opt-125m",
170
prompt="San Francisco is a")
171
print("Completion result:", completion)
172
@@ -194,11 +199,17 @@ Using the `openai` python package, you can also communicate with the model in a
199
200
.. code-block:: python
201
197
- import openai
202
+ from openai import OpenAI
203
# Set OpenAI's API key and API base to use vLLM's API server.
199
- openai.api_key = "EMPTY"
200
- openai.api_base = "http://localhost:8000/v1"
201
- chat_response = openai.ChatCompletion.create(
204
+ openai_api_key = "EMPTY"
205
+ openai_api_base = "http://localhost:8000/v1"
206
+
207
+ client = OpenAI(
208
+ api_key=openai_api_key,
209
+ base_url=openai_api_base,
210
+ )
211
+
212
+ chat_response = client.chat.completions.create(
213
model="facebook/opt-125m",
214
messages=[
215
{"role": "system", "content": "You are a helpful assistant."},
examples/openai_chatcompletion_client.py
+15
-11
@@ -1,18 +1,19 @@
1
-import openai
1
+from openai import OpenAI
2
3
# Modify OpenAI's API key and API base to use vLLM's API server.
4
-openai.api_key = "EMPTY"
5
-openai.api_base = "http://localhost:8000/v1"
4
+openai_api_key = "EMPTY"
5
+openai_api_base = "http://localhost:8000/v1"
6
7
-# List models API
8
-models = openai.Model.list()
9
-print("Models:", models)
7
+client = OpenAI(
8
+ # defaults to os.environ.get("OPENAI_API_KEY")
9
+ api_key=openai_api_key,
10
+ base_url=openai_api_base,
11
+)
12
11
-model = models["data"][0]["id"]
13
+models = client.models.list()
14
+model = models.data[0].id
15
13
-# Chat completion API
14
-chat_completion = openai.ChatCompletion.create(
15
- model=model,
16
+chat_completion = client.chat.completions.create(
17
messages=[{
18
"role": "system",
19
"content": "You are a helpful assistant."
@@ -27,7 +28,10 @@ chat_completion = openai.ChatCompletion.create(
28
}, {
29
"role": "user",
30
"content": "Where was it played?"
30
- }])
31
+ }],
32
+ model=model,
33
+)
34
+
35
36
print("Chat completion results:")
37
print(chat_completion)
examples/openai_completion_client.py
+13
-9
@@ -1,24 +1,28 @@
1
-import openai
1
+from openai import OpenAI
2
3
# Modify OpenAI's API key and API base to use vLLM's API server.
4
-openai.api_key = "EMPTY"
5
-openai.api_base = "http://localhost:8000/v1"
4
+openai_api_key = "EMPTY"
5
+openai_api_base = "http://localhost:8000/v1"
6
7
-# List models API
8
-models = openai.Model.list()
9
-print("Models:", models)
7
+client = OpenAI(
8
+ # defaults to os.environ.get("OPENAI_API_KEY")
9
+ api_key=openai_api_key,
10
+ base_url=openai_api_base,
11
+)
12
11
-model = models["data"][0]["id"]
13
+models = client.models.list()
14
+model = models.data[0].id
15
16
# Completion API
17
stream = False
15
-completion = openai.Completion.create(
18
+completion = client.completions.create(
19
model=model,
20
prompt="A robot may not injure a human being",
21
echo=False,
22
n=2,
23
stream=stream,
21
- logprobs=3)
24
+ logprobs=3
25
+)
26
27
print("Completion results:")
28
if stream: