A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Feature] vLLM CLI (#5090)
Co-authored-by: simon-mo <simon.mo@hey.com>
Ethan Xu committed
Jul 14, 2024 at 15:36 UTC
dbfe254eda918bc3cc52cf448d518824ad6593b9
7 files changed
+223
-36
benchmarks/benchmark_serving.py
+2
-2
@@ -2,8 +2,8 @@
2
3
On the server side, run one of the following commands:
4
vLLM OpenAI API server
5
- python -m vllm.entrypoints.openai.api_server \
6
- --model <your_model> --swap-space 16 \
5
+ vllm serve <your_model> \
6
+ --swap-space 16 \
7
--disable-log-requests
8
9
(TGI backend)
docs/source/serving/openai_compatible_server.md
+1
-1
@@ -109,7 +109,7 @@ directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
109
110
```{argparse}
111
:module: vllm.entrypoints.openai.cli_args
112
-:func: make_arg_parser
112
+:func: create_parser_for_docs
113
:prog: -m vllm.entrypoints.openai.api_server
114
```
115
setup.py
+5
@@ -488,4 +488,9 @@ setup(
488
},
489
cmdclass={"build_ext": cmake_build_ext} if _build_custom_ops() else {},
490
package_data=package_data,
491
+ entry_points={
492
+ "console_scripts": [
493
+ "vllm=vllm.scripts:main",
494
+ ],
495
+ },
496
)
tests/utils.py
+4
-2
@@ -14,7 +14,7 @@ import requests
14
from vllm.distributed import (ensure_model_parallel_initialized,
15
init_distributed_environment)
16
from vllm.entrypoints.openai.cli_args import make_arg_parser
17
-from vllm.utils import get_open_port, is_hip
17
+from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
18
19
if is_hip():
20
from amdsmi import (amdsmi_get_gpu_vram_usage,
@@ -57,7 +57,9 @@ class RemoteOpenAIServer:
57
58
cli_args = cli_args + ["--port", str(get_open_port())]
59
60
- parser = make_arg_parser()
60
+ parser = FlexibleArgumentParser(
61
+ description="vLLM's remote OpenAI server.")
62
+ parser = make_arg_parser(parser)
63
args = parser.parse_args(cli_args)
64
self.host = str(args.host or 'localhost')
65
self.port = int(args.port)
vllm/entrypoints/openai/api_server.py
+50
-28
@@ -8,7 +8,7 @@ from typing import Optional, Set
8
9
import fastapi
10
import uvicorn
11
-from fastapi import Request
11
+from fastapi import APIRouter, Request
12
from fastapi.exceptions import RequestValidationError
13
from fastapi.middleware.cors import CORSMiddleware
14
from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -35,10 +35,14 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
35
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
36
from vllm.logger import init_logger
37
from vllm.usage.usage_lib import UsageContext
38
+from vllm.utils import FlexibleArgumentParser
39
from vllm.version import __version__ as VLLM_VERSION
40
41
TIMEOUT_KEEP_ALIVE = 5 # seconds
42
43
+logger = init_logger(__name__)
44
+engine: AsyncLLMEngine
45
+engine_args: AsyncEngineArgs
46
openai_serving_chat: OpenAIServingChat
47
openai_serving_completion: OpenAIServingCompletion
48
openai_serving_embedding: OpenAIServingEmbedding
@@ -64,35 +68,23 @@ async def lifespan(app: fastapi.FastAPI):
68
yield
69
70
67
-app = fastapi.FastAPI(lifespan=lifespan)
68
-
69
-
70
-def parse_args():
71
- parser = make_arg_parser()
72
- return parser.parse_args()
73
-
71
+router = APIRouter()
72
73
# Add prometheus asgi middleware to route /metrics requests
74
route = Mount("/metrics", make_asgi_app())
75
# Workaround for 307 Redirect for /metrics
76
route.path_regex = re.compile('^/metrics(?P<path>.*)$')
79
-app.routes.append(route)
80
-
81
-
82
-@app.exception_handler(RequestValidationError)
83
-async def validation_exception_handler(_, exc):
84
- err = openai_serving_chat.create_error_response(message=str(exc))
85
- return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
77
+router.routes.append(route)
78
79
88
-@app.get("/health")
80
+@router.get("/health")
81
async def health() -> Response:
82
"""Health check."""
83
await openai_serving_chat.engine.check_health()
84
return Response(status_code=200)
85
86
95
-@app.post("/tokenize")
87
+@router.post("/tokenize")
88
async def tokenize(request: TokenizeRequest):
89
generator = await openai_serving_completion.create_tokenize(request)
90
if isinstance(generator, ErrorResponse):
@@ -103,7 +95,7 @@ async def tokenize(request: TokenizeRequest):
95
return JSONResponse(content=generator.model_dump())
96
97
106
-@app.post("/detokenize")
98
+@router.post("/detokenize")
99
async def detokenize(request: DetokenizeRequest):
100
generator = await openai_serving_completion.create_detokenize(request)
101
if isinstance(generator, ErrorResponse):
@@ -114,19 +106,19 @@ async def detokenize(request: DetokenizeRequest):
106
return JSONResponse(content=generator.model_dump())
107
108
117
-@app.get("/v1/models")
109
+@router.get("/v1/models")
110
async def show_available_models():
111
models = await openai_serving_completion.show_available_models()
112
return JSONResponse(content=models.model_dump())
113
114
123
-@app.get("/version")
115
+@router.get("/version")
116
async def show_version():
117
ver = {"version": VLLM_VERSION}
118
return JSONResponse(content=ver)
119
120
129
-@app.post("/v1/chat/completions")
121
+@router.post("/v1/chat/completions")
122
async def create_chat_completion(request: ChatCompletionRequest,
123
raw_request: Request):
124
generator = await openai_serving_chat.create_chat_completion(
@@ -142,7 +134,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
134
return JSONResponse(content=generator.model_dump())
135
136
145
-@app.post("/v1/completions")
137
+@router.post("/v1/completions")
138
async def create_completion(request: CompletionRequest, raw_request: Request):
139
generator = await openai_serving_completion.create_completion(
140
request, raw_request)
@@ -156,7 +148,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
148
return JSONResponse(content=generator.model_dump())
149
150
159
-@app.post("/v1/embeddings")
151
+@router.post("/v1/embeddings")
152
async def create_embedding(request: EmbeddingRequest, raw_request: Request):
153
generator = await openai_serving_embedding.create_embedding(
154
request, raw_request)
@@ -167,8 +159,10 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
159
return JSONResponse(content=generator.model_dump())
160
161
170
-if __name__ == "__main__":
171
- args = parse_args()
162
+def build_app(args):
163
+ app = fastapi.FastAPI(lifespan=lifespan)
164
+ app.include_router(router)
165
+ app.root_path = args.root_path
166
167
app.add_middleware(
168
CORSMiddleware,
@@ -178,6 +172,12 @@ if __name__ == "__main__":
172
allow_headers=args.allowed_headers,
173
)
174
175
+ @app.exception_handler(RequestValidationError)
176
+ async def validation_exception_handler(_, exc):
177
+ err = openai_serving_chat.create_error_response(message=str(exc))
178
+ return JSONResponse(err.model_dump(),
179
+ status_code=HTTPStatus.BAD_REQUEST)
180
+
181
if token := envs.VLLM_API_KEY or args.api_key:
182
183
@app.middleware("http")
@@ -203,6 +203,12 @@ if __name__ == "__main__":
203
raise ValueError(f"Invalid middleware {middleware}. "
204
f"Must be a function or a class.")
205
206
+ return app
207
+
208
+
209
+def run_server(args, llm_engine=None):
210
+ app = build_app(args)
211
+
212
logger.info("vLLM API server version %s", VLLM_VERSION)
213
logger.info("args: %s", args)
214
@@ -211,10 +217,12 @@ if __name__ == "__main__":
217
else:
218
served_model_names = [args.model]
219
214
- engine_args = AsyncEngineArgs.from_cli_args(args)
220
+ global engine, engine_args
221
216
- engine = AsyncLLMEngine.from_engine_args(
217
- engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
222
+ engine_args = AsyncEngineArgs.from_cli_args(args)
223
+ engine = (llm_engine
224
+ if llm_engine is not None else AsyncLLMEngine.from_engine_args(
225
+ engine_args, usage_context=UsageContext.OPENAI_API_SERVER))
226
227
event_loop: Optional[asyncio.AbstractEventLoop]
228
try:
@@ -230,6 +238,10 @@ if __name__ == "__main__":
238
# When using single vLLM without engine_use_ray
239
model_config = asyncio.run(engine.get_model_config())
240
241
+ global openai_serving_chat
242
+ global openai_serving_completion
243
+ global openai_serving_embedding
244
+
245
openai_serving_chat = OpenAIServingChat(engine, model_config,
246
served_model_names,
247
args.response_role,
@@ -258,3 +270,13 @@ if __name__ == "__main__":
270
ssl_certfile=args.ssl_certfile,
271
ssl_ca_certs=args.ssl_ca_certs,
272
ssl_cert_reqs=args.ssl_cert_reqs)
273
+
274
+
275
+if __name__ == "__main__":
276
+ # NOTE(simon):
277
+ # This section should be in sync with vllm/scripts.py for CLI entrypoints.
278
+ parser = FlexibleArgumentParser(
279
+ description="vLLM OpenAI-Compatible RESTful API server.")
280
+ parser = make_arg_parser(parser)
281
+ args = parser.parse_args()
282
+ run_server(args)
vllm/entrypoints/openai/cli_args.py
+7
-3
@@ -34,9 +34,7 @@ class PromptAdapterParserAction(argparse.Action):
34
setattr(namespace, self.dest, adapter_list)
35
36
37
-def make_arg_parser():
38
- parser = FlexibleArgumentParser(
39
- description="vLLM OpenAI-Compatible RESTful API server.")
37
+def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
38
parser.add_argument("--host",
39
type=nullable_str,
40
default=None,
@@ -133,3 +131,9 @@ def make_arg_parser():
131
132
parser = AsyncEngineArgs.add_cli_args(parser)
133
return parser
134
+
135
+
136
+def create_parser_for_docs() -> FlexibleArgumentParser:
137
+ parser_for_docs = FlexibleArgumentParser(
138
+ prog="-m vllm.entrypoints.openai.api_server")
139
+ return make_arg_parser(parser_for_docs)
vllm/scripts.py
new
+154
@@ -0,0 +1,154 @@
1
+# The CLI entrypoint to vLLM.
2
+import argparse
3
+import os
4
+import signal
5
+import sys
6
+from typing import Optional
7
+
8
+from openai import OpenAI
9
+
10
+from vllm.entrypoints.openai.api_server import run_server
11
+from vllm.entrypoints.openai.cli_args import make_arg_parser
12
+from vllm.utils import FlexibleArgumentParser
13
+
14
+
15
+def registrer_signal_handlers():
16
+
17
+ def signal_handler(sig, frame):
18
+ sys.exit(0)
19
+
20
+ signal.signal(signal.SIGINT, signal_handler)
21
+ signal.signal(signal.SIGTSTP, signal_handler)
22
+
23
+
24
+def serve(args: argparse.Namespace) -> None:
25
+ # EngineArgs expects the model name to be passed as --model.
26
+ args.model = args.model_tag
27
+
28
+ run_server(args)
29
+
30
+
31
+def interactive_cli(args: argparse.Namespace) -> None:
32
+ registrer_signal_handlers()
33
+
34
+ base_url = args.url
35
+ api_key = args.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY")
36
+ openai_client = OpenAI(api_key=api_key, base_url=base_url)
37
+
38
+ if args.model_name:
39
+ model_name = args.model_name
40
+ else:
41
+ available_models = openai_client.models.list()
42
+ model_name = available_models.data[0].id
43
+
44
+ print(f"Using model: {model_name}")
45
+
46
+ if args.command == "complete":
47
+ complete(model_name, openai_client)
48
+ elif args.command == "chat":
49
+ chat(args.system_prompt, model_name, openai_client)
50
+
51
+
52
+def complete(model_name: str, client: OpenAI) -> None:
53
+ print("Please enter prompt to complete:")
54
+ while True:
55
+ input_prompt = input("> ")
56
+
57
+ completion = client.completions.create(model=model_name,
58
+ prompt=input_prompt)
59
+ output = completion.choices[0].text
60
+ print(output)
61
+
62
+
63
+def chat(system_prompt: Optional[str], model_name: str,
64
+ client: OpenAI) -> None:
65
+ conversation = []
66
+ if system_prompt is not None:
67
+ conversation.append({"role": "system", "content": system_prompt})
68
+
69
+ print("Please enter a message for the chat model:")
70
+ while True:
71
+ input_message = input("> ")
72
+ message = {"role": "user", "content": input_message}
73
+ conversation.append(message)
74
+
75
+ chat_completion = client.chat.completions.create(model=model_name,
76
+ messages=conversation)
77
+
78
+ response_message = chat_completion.choices[0].message
79
+ output = response_message.content
80
+
81
+ conversation.append(response_message)
82
+ print(output)
83
+
84
+
85
+def _add_query_options(
86
+ parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
87
+ parser.add_argument(
88
+ "--url",
89
+ type=str,
90
+ default="http://localhost:8000/v1",
91
+ help="url of the running OpenAI-Compatible RESTful API server")
92
+ parser.add_argument(
93
+ "--model-name",
94
+ type=str,
95
+ default=None,
96
+ help=("The model name used in prompt completion, default to "
97
+ "the first model in list models API call."))
98
+ parser.add_argument(
99
+ "--api-key",
100
+ type=str,
101
+ default=None,
102
+ help=(
103
+ "API key for OpenAI services. If provided, this api key "
104
+ "will overwrite the api key obtained through environment variables."
105
+ ))
106
+ return parser
107
+
108
+
109
+def main():
110
+ parser = FlexibleArgumentParser(description="vLLM CLI")
111
+ subparsers = parser.add_subparsers(required=True)
112
+
113
+ serve_parser = subparsers.add_parser(
114
+ "serve",
115
+ help="Start the vLLM OpenAI Compatible API server",
116
+ usage="vllm serve <model_tag> [options]")
117
+ serve_parser.add_argument("model_tag",
118
+ type=str,
119
+ help="The model tag to serve")
120
+ serve_parser = make_arg_parser(serve_parser)
121
+ serve_parser.set_defaults(dispatch_function=serve)
122
+
123
+ complete_parser = subparsers.add_parser(
124
+ "complete",
125
+ help=("Generate text completions based on the given prompt "
126
+ "via the running API server"),
127
+ usage="vllm complete [options]")
128
+ _add_query_options(complete_parser)
129
+ complete_parser.set_defaults(dispatch_function=interactive_cli,
130
+ command="complete")
131
+
132
+ chat_parser = subparsers.add_parser(
133
+ "chat",
134
+ help="Generate chat completions via the running API server",
135
+ usage="vllm chat [options]")
136
+ _add_query_options(chat_parser)
137
+ chat_parser.add_argument(
138
+ "--system-prompt",
139
+ type=str,
140
+ default=None,
141
+ help=("The system prompt to be added to the chat template, "
142
+ "used for models that support system prompts."))
143
+ chat_parser.set_defaults(dispatch_function=interactive_cli, command="chat")
144
+
145
+ args = parser.parse_args()
146
+ # One of the sub commands should be executed.
147
+ if hasattr(args, "dispatch_function"):
148
+ args.dispatch_function(args)
149
+ else:
150
+ parser.print_help()
151
+
152
+
153
+if __name__ == "__main__":
154
+ main()