@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c07a442854
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

chore(examples-docs): upgrade to OpenAI V1 (#1785)

Massimiliano Pronesti committed Dec 3, 2023 at 10:11 UTC c07a4428545c7708626ab04bb0033284e71d4ed7
3 files changed +47 -28
docs/source/getting_started/quickstart.rst
+19 -8
@@ -157,11 +157,16 @@ Since this server is compatible with OpenAI API, you can use it as a drop-in rep
157
158 .. code-block:: python
159
160 - import openai
160 + from openai import OpenAI
161 +
162 # Modify OpenAI's API key and API base to use vLLM's API server.
162 - openai.api_key = "EMPTY"
163 - openai.api_base = "http://localhost:8000/v1"
164 - completion = openai.Completion.create(model="facebook/opt-125m",
163 + openai_api_key = "EMPTY"
164 + openai_api_base = "http://localhost:8000/v1"
165 + client = OpenAI(
166 + api_key=openai_api_key,
167 + base_url=openai_api_base,
168 + )
169 + completion = client.completions.create(model="facebook/opt-125m",
170 prompt="San Francisco is a")
171 print("Completion result:", completion)
172
@@ -194,11 +199,17 @@ Using the `openai` python package, you can also communicate with the model in a
199
200 .. code-block:: python
201
197 - import openai
202 + from openai import OpenAI
203 # Set OpenAI's API key and API base to use vLLM's API server.
199 - openai.api_key = "EMPTY"
200 - openai.api_base = "http://localhost:8000/v1"
201 - chat_response = openai.ChatCompletion.create(
204 + openai_api_key = "EMPTY"
205 + openai_api_base = "http://localhost:8000/v1"
206 +
207 + client = OpenAI(
208 + api_key=openai_api_key,
209 + base_url=openai_api_base,
210 + )
211 +
212 + chat_response = client.chat.completions.create(
213 model="facebook/opt-125m",
214 messages=[
215 {"role": "system", "content": "You are a helpful assistant."},
examples/openai_chatcompletion_client.py
+15 -11
@@ -1,18 +1,19 @@
1 -import openai
1 +from openai import OpenAI
2
3 # Modify OpenAI's API key and API base to use vLLM's API server.
4 -openai.api_key = "EMPTY"
5 -openai.api_base = "http://localhost:8000/v1"
4 +openai_api_key = "EMPTY"
5 +openai_api_base = "http://localhost:8000/v1"
6
7 -# List models API
8 -models = openai.Model.list()
9 -print("Models:", models)
7 +client = OpenAI(
8 + # defaults to os.environ.get("OPENAI_API_KEY")
9 + api_key=openai_api_key,
10 + base_url=openai_api_base,
11 +)
12
11 -model = models["data"][0]["id"]
13 +models = client.models.list()
14 +model = models.data[0].id
15
13 -# Chat completion API
14 -chat_completion = openai.ChatCompletion.create(
15 - model=model,
16 +chat_completion = client.chat.completions.create(
17 messages=[{
18 "role": "system",
19 "content": "You are a helpful assistant."
@@ -27,7 +28,10 @@ chat_completion = openai.ChatCompletion.create(
28 }, {
29 "role": "user",
30 "content": "Where was it played?"
30 - }])
31 + }],
32 + model=model,
33 +)
34 +
35
36 print("Chat completion results:")
37 print(chat_completion)
examples/openai_completion_client.py
+13 -9
@@ -1,24 +1,28 @@
1 -import openai
1 +from openai import OpenAI
2
3 # Modify OpenAI's API key and API base to use vLLM's API server.
4 -openai.api_key = "EMPTY"
5 -openai.api_base = "http://localhost:8000/v1"
4 +openai_api_key = "EMPTY"
5 +openai_api_base = "http://localhost:8000/v1"
6
7 -# List models API
8 -models = openai.Model.list()
9 -print("Models:", models)
7 +client = OpenAI(
8 + # defaults to os.environ.get("OPENAI_API_KEY")
9 + api_key=openai_api_key,
10 + base_url=openai_api_base,
11 +)
12
11 -model = models["data"][0]["id"]
13 +models = client.models.list()
14 +model = models.data[0].id
15
16 # Completion API
17 stream = False
15 -completion = openai.Completion.create(
18 +completion = client.completions.create(
19 model=model,
20 prompt="A robot may not injure a human being",
21 echo=False,
22 n=2,
23 stream=stream,
21 - logprobs=3)
24 + logprobs=3
25 +)
26
27 print("Completion results:")
28 if stream: