A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Doc] Support --stream arg in openai_completion_client.py script (#18388)
Signed-off-by: googs1025 <googs1025@gmail.com>
CYJiang committed
May 22, 2025 at 21:20 UTC
71075029f214bd4db409ba553cf083a883fdd61f
3 files changed
+25
-9
examples/online_serving/openai_chat_completion_structured_outputs.py
+5
-2
@@ -12,6 +12,9 @@ from enum import Enum
12
from openai import BadRequestError, OpenAI
13
from pydantic import BaseModel
14
15
+openai_api_key = "EMPTY"
16
+openai_api_base = "http://localhost:8000/v1"
17
+
18
19
# Guided decoding by Choice (list of possible options)
20
def guided_choice_completion(client: OpenAI, model: str):
@@ -134,8 +137,8 @@ def extra_backend_options_completion(client: OpenAI, model: str):
137
138
def main():
139
client: OpenAI = OpenAI(
137
- base_url="http://localhost:8000/v1",
138
- api_key="-",
140
+ base_url=openai_api_base,
141
+ api_key=openai_api_key,
142
)
143
144
model = client.models.list().data[0].id
examples/online_serving/openai_chat_completion_structured_outputs_structural_tag.py
+5
-2
@@ -7,11 +7,14 @@ from openai import OpenAI
7
# to enforce the format of a tool call response, but it could be used for
8
# any structured output within a subset of the response.
9
10
+openai_api_key = "EMPTY"
11
+openai_api_base = "http://localhost:8000/v1"
12
+
13
14
def main():
15
client = OpenAI(
13
- base_url="http://localhost:8000/v1",
14
- api_key="-",
16
+ base_url=openai_api_base,
17
+ api_key=openai_api_key,
18
)
19
20
messages = [{
examples/online_serving/openai_completion_client.py
+15
-5
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+import argparse
4
+
5
from openai import OpenAI
6
7
# Modify OpenAI's API key and API base to use vLLM's API server.
@@ -7,7 +9,15 @@ openai_api_key = "EMPTY"
9
openai_api_base = "http://localhost:8000/v1"
10
11
10
-def main():
12
+def parse_args():
13
+ parser = argparse.ArgumentParser(description="Client for vLLM API server")
14
+ parser.add_argument("--stream",
15
+ action="store_true",
16
+ help="Enable streaming response")
17
+ return parser.parse_args()
18
+
19
+
20
+def main(args):
21
client = OpenAI(
22
# defaults to os.environ.get("OPENAI_API_KEY")
23
api_key=openai_api_key,
@@ -18,18 +28,17 @@ def main():
28
model = models.data[0].id
29
30
# Completion API
21
- stream = False
31
completion = client.completions.create(
32
model=model,
33
prompt="A robot may not injure a human being",
34
echo=False,
35
n=2,
27
- stream=stream,
36
+ stream=args.stream,
37
logprobs=3)
38
39
print("-" * 50)
40
print("Completion results:")
32
- if stream:
41
+ if args.stream:
42
for c in completion:
43
print(c)
44
else:
@@ -38,4 +47,5 @@ def main():
47
48
49
if __name__ == "__main__":
41
- main()
50
+ args = parse_args()
51
+ main(args)