A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Misc] update api_client example (#16459)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Apr 11, 2025 at 18:05 UTC
35e076b3a8207e7d39ebab82152fd4b3db648233
1 file changed
+16
-9
examples/online_serving/api_client.py
+16
-9
@@ -1,5 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""Example Python client for `vllm.entrypoints.api_server`
3
+Start the demo server:
4
+ python -m vllm.entrypoints.api_server --model <model_name>
5
+
6
NOTE: The API server is used only for demonstration and simple performance
7
benchmarks. It is not intended for production use.
8
For production use, we recommend `vllm serve` and the OpenAI client API.
@@ -7,6 +10,7 @@ For production use, we recommend `vllm serve` and the OpenAI client API.
10
11
import argparse
12
import json
13
+from argparse import Namespace
14
from collections.abc import Iterable
15
16
import requests
@@ -27,7 +31,6 @@ def post_http_request(prompt: str,
31
pload = {
32
"prompt": prompt,
33
"n": n,
30
- "use_beam_search": True,
34
"temperature": 0.0,
35
"max_tokens": 16,
36
"stream": stream,
@@ -55,14 +58,7 @@ def get_response(response: requests.Response) -> list[str]:
58
return output
59
60
58
-if __name__ == "__main__":
59
- parser = argparse.ArgumentParser()
60
- parser.add_argument("--host", type=str, default="localhost")
61
- parser.add_argument("--port", type=int, default=8000)
62
- parser.add_argument("--n", type=int, default=4)
63
- parser.add_argument("--prompt", type=str, default="San Francisco is a")
64
- parser.add_argument("--stream", action="store_true")
65
- args = parser.parse_args()
61
+def main(args: Namespace):
62
prompt = args.prompt
63
api_url = f"http://{args.host}:{args.port}/generate"
64
n = args.n
@@ -83,3 +79,14 @@ if __name__ == "__main__":
79
output = get_response(response)
80
for i, line in enumerate(output):
81
print(f"Beam candidate {i}: {line!r}", flush=True)
82
+
83
+
84
+if __name__ == "__main__":
85
+ parser = argparse.ArgumentParser()
86
+ parser.add_argument("--host", type=str, default="localhost")
87
+ parser.add_argument("--port", type=int, default=8000)
88
+ parser.add_argument("--n", type=int, default=1)
89
+ parser.add_argument("--prompt", type=str, default="San Francisco is a")
90
+ parser.add_argument("--stream", action="store_true")
91
+ args = parser.parse_args()
92
+ main(args)