@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dbfe254eda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Feature] vLLM CLI (#5090)

Co-authored-by: simon-mo <simon.mo@hey.com>

Ethan Xu committed Jul 14, 2024 at 15:36 UTC dbfe254eda918bc3cc52cf448d518824ad6593b9
7 files changed +223 -36
benchmarks/benchmark_serving.py
+2 -2
@@ -2,8 +2,8 @@
2
3 On the server side, run one of the following commands:
4 vLLM OpenAI API server
5 - python -m vllm.entrypoints.openai.api_server \
6 - --model <your_model> --swap-space 16 \
5 + vllm serve <your_model> \
6 + --swap-space 16 \
7 --disable-log-requests
8
9 (TGI backend)
docs/source/serving/openai_compatible_server.md
+1 -1
@@ -109,7 +109,7 @@ directory [here](https://github.com/vllm-project/vllm/tree/main/examples/)
109
110 ```{argparse}
111 :module: vllm.entrypoints.openai.cli_args
112 -:func: make_arg_parser
112 +:func: create_parser_for_docs
113 :prog: -m vllm.entrypoints.openai.api_server
114 ```
115
setup.py
+5
@@ -488,4 +488,9 @@ setup(
488 },
489 cmdclass={"build_ext": cmake_build_ext} if _build_custom_ops() else {},
490 package_data=package_data,
491 + entry_points={
492 + "console_scripts": [
493 + "vllm=vllm.scripts:main",
494 + ],
495 + },
496 )
tests/utils.py
+4 -2
@@ -14,7 +14,7 @@ import requests
14 from vllm.distributed import (ensure_model_parallel_initialized,
15 init_distributed_environment)
16 from vllm.entrypoints.openai.cli_args import make_arg_parser
17 -from vllm.utils import get_open_port, is_hip
17 +from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
18
19 if is_hip():
20 from amdsmi import (amdsmi_get_gpu_vram_usage,
@@ -57,7 +57,9 @@ class RemoteOpenAIServer:
57
58 cli_args = cli_args + ["--port", str(get_open_port())]
59
60 - parser = make_arg_parser()
60 + parser = FlexibleArgumentParser(
61 + description="vLLM's remote OpenAI server.")
62 + parser = make_arg_parser(parser)
63 args = parser.parse_args(cli_args)
64 self.host = str(args.host or 'localhost')
65 self.port = int(args.port)
vllm/entrypoints/openai/api_server.py
+50 -28
@@ -8,7 +8,7 @@ from typing import Optional, Set
8
9 import fastapi
10 import uvicorn
11 -from fastapi import Request
11 +from fastapi import APIRouter, Request
12 from fastapi.exceptions import RequestValidationError
13 from fastapi.middleware.cors import CORSMiddleware
14 from fastapi.responses import JSONResponse, Response, StreamingResponse
@@ -35,10 +35,14 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
35 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
36 from vllm.logger import init_logger
37 from vllm.usage.usage_lib import UsageContext
38 +from vllm.utils import FlexibleArgumentParser
39 from vllm.version import __version__ as VLLM_VERSION
40
41 TIMEOUT_KEEP_ALIVE = 5 # seconds
42
43 +logger = init_logger(__name__)
44 +engine: AsyncLLMEngine
45 +engine_args: AsyncEngineArgs
46 openai_serving_chat: OpenAIServingChat
47 openai_serving_completion: OpenAIServingCompletion
48 openai_serving_embedding: OpenAIServingEmbedding
@@ -64,35 +68,23 @@ async def lifespan(app: fastapi.FastAPI):
68 yield
69
70
67 -app = fastapi.FastAPI(lifespan=lifespan)
68 -
69 -
70 -def parse_args():
71 - parser = make_arg_parser()
72 - return parser.parse_args()
73 -
71 +router = APIRouter()
72
73 # Add prometheus asgi middleware to route /metrics requests
74 route = Mount("/metrics", make_asgi_app())
75 # Workaround for 307 Redirect for /metrics
76 route.path_regex = re.compile('^/metrics(?P<path>.*)$')
79 -app.routes.append(route)
80 -
81 -
82 -@app.exception_handler(RequestValidationError)
83 -async def validation_exception_handler(_, exc):
84 - err = openai_serving_chat.create_error_response(message=str(exc))
85 - return JSONResponse(err.model_dump(), status_code=HTTPStatus.BAD_REQUEST)
77 +router.routes.append(route)
78
79
88 -@app.get("/health")
80 +@router.get("/health")
81 async def health() -> Response:
82 """Health check."""
83 await openai_serving_chat.engine.check_health()
84 return Response(status_code=200)
85
86
95 -@app.post("/tokenize")
87 +@router.post("/tokenize")
88 async def tokenize(request: TokenizeRequest):
89 generator = await openai_serving_completion.create_tokenize(request)
90 if isinstance(generator, ErrorResponse):
@@ -103,7 +95,7 @@ async def tokenize(request: TokenizeRequest):
95 return JSONResponse(content=generator.model_dump())
96
97
106 -@app.post("/detokenize")
98 +@router.post("/detokenize")
99 async def detokenize(request: DetokenizeRequest):
100 generator = await openai_serving_completion.create_detokenize(request)
101 if isinstance(generator, ErrorResponse):
@@ -114,19 +106,19 @@ async def detokenize(request: DetokenizeRequest):
106 return JSONResponse(content=generator.model_dump())
107
108
117 -@app.get("/v1/models")
109 +@router.get("/v1/models")
110 async def show_available_models():
111 models = await openai_serving_completion.show_available_models()
112 return JSONResponse(content=models.model_dump())
113
114
123 -@app.get("/version")
115 +@router.get("/version")
116 async def show_version():
117 ver = {"version": VLLM_VERSION}
118 return JSONResponse(content=ver)
119
120
129 -@app.post("/v1/chat/completions")
121 +@router.post("/v1/chat/completions")
122 async def create_chat_completion(request: ChatCompletionRequest,
123 raw_request: Request):
124 generator = await openai_serving_chat.create_chat_completion(
@@ -142,7 +134,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
134 return JSONResponse(content=generator.model_dump())
135
136
145 -@app.post("/v1/completions")
137 +@router.post("/v1/completions")
138 async def create_completion(request: CompletionRequest, raw_request: Request):
139 generator = await openai_serving_completion.create_completion(
140 request, raw_request)
@@ -156,7 +148,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
148 return JSONResponse(content=generator.model_dump())
149
150
159 -@app.post("/v1/embeddings")
151 +@router.post("/v1/embeddings")
152 async def create_embedding(request: EmbeddingRequest, raw_request: Request):
153 generator = await openai_serving_embedding.create_embedding(
154 request, raw_request)
@@ -167,8 +159,10 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
159 return JSONResponse(content=generator.model_dump())
160
161
170 -if __name__ == "__main__":
171 - args = parse_args()
162 +def build_app(args):
163 + app = fastapi.FastAPI(lifespan=lifespan)
164 + app.include_router(router)
165 + app.root_path = args.root_path
166
167 app.add_middleware(
168 CORSMiddleware,
@@ -178,6 +172,12 @@ if __name__ == "__main__":
172 allow_headers=args.allowed_headers,
173 )
174
175 + @app.exception_handler(RequestValidationError)
176 + async def validation_exception_handler(_, exc):
177 + err = openai_serving_chat.create_error_response(message=str(exc))
178 + return JSONResponse(err.model_dump(),
179 + status_code=HTTPStatus.BAD_REQUEST)
180 +
181 if token := envs.VLLM_API_KEY or args.api_key:
182
183 @app.middleware("http")
@@ -203,6 +203,12 @@ if __name__ == "__main__":
203 raise ValueError(f"Invalid middleware {middleware}. "
204 f"Must be a function or a class.")
205
206 + return app
207 +
208 +
209 +def run_server(args, llm_engine=None):
210 + app = build_app(args)
211 +
212 logger.info("vLLM API server version %s", VLLM_VERSION)
213 logger.info("args: %s", args)
214
@@ -211,10 +217,12 @@ if __name__ == "__main__":
217 else:
218 served_model_names = [args.model]
219
214 - engine_args = AsyncEngineArgs.from_cli_args(args)
220 + global engine, engine_args
221
216 - engine = AsyncLLMEngine.from_engine_args(
217 - engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
222 + engine_args = AsyncEngineArgs.from_cli_args(args)
223 + engine = (llm_engine
224 + if llm_engine is not None else AsyncLLMEngine.from_engine_args(
225 + engine_args, usage_context=UsageContext.OPENAI_API_SERVER))
226
227 event_loop: Optional[asyncio.AbstractEventLoop]
228 try:
@@ -230,6 +238,10 @@ if __name__ == "__main__":
238 # When using single vLLM without engine_use_ray
239 model_config = asyncio.run(engine.get_model_config())
240
241 + global openai_serving_chat
242 + global openai_serving_completion
243 + global openai_serving_embedding
244 +
245 openai_serving_chat = OpenAIServingChat(engine, model_config,
246 served_model_names,
247 args.response_role,
@@ -258,3 +270,13 @@ if __name__ == "__main__":
270 ssl_certfile=args.ssl_certfile,
271 ssl_ca_certs=args.ssl_ca_certs,
272 ssl_cert_reqs=args.ssl_cert_reqs)
273 +
274 +
275 +if __name__ == "__main__":
276 + # NOTE(simon):
277 + # This section should be in sync with vllm/scripts.py for CLI entrypoints.
278 + parser = FlexibleArgumentParser(
279 + description="vLLM OpenAI-Compatible RESTful API server.")
280 + parser = make_arg_parser(parser)
281 + args = parser.parse_args()
282 + run_server(args)
vllm/entrypoints/openai/cli_args.py
+7 -3
@@ -34,9 +34,7 @@ class PromptAdapterParserAction(argparse.Action):
34 setattr(namespace, self.dest, adapter_list)
35
36
37 -def make_arg_parser():
38 - parser = FlexibleArgumentParser(
39 - description="vLLM OpenAI-Compatible RESTful API server.")
37 +def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
38 parser.add_argument("--host",
39 type=nullable_str,
40 default=None,
@@ -133,3 +131,9 @@ def make_arg_parser():
131
132 parser = AsyncEngineArgs.add_cli_args(parser)
133 return parser
134 +
135 +
136 +def create_parser_for_docs() -> FlexibleArgumentParser:
137 + parser_for_docs = FlexibleArgumentParser(
138 + prog="-m vllm.entrypoints.openai.api_server")
139 + return make_arg_parser(parser_for_docs)
vllm/scripts.py new
+154
@@ -0,0 +1,154 @@
1 +# The CLI entrypoint to vLLM.
2 +import argparse
3 +import os
4 +import signal
5 +import sys
6 +from typing import Optional
7 +
8 +from openai import OpenAI
9 +
10 +from vllm.entrypoints.openai.api_server import run_server
11 +from vllm.entrypoints.openai.cli_args import make_arg_parser
12 +from vllm.utils import FlexibleArgumentParser
13 +
14 +
15 +def registrer_signal_handlers():
16 +
17 + def signal_handler(sig, frame):
18 + sys.exit(0)
19 +
20 + signal.signal(signal.SIGINT, signal_handler)
21 + signal.signal(signal.SIGTSTP, signal_handler)
22 +
23 +
24 +def serve(args: argparse.Namespace) -> None:
25 + # EngineArgs expects the model name to be passed as --model.
26 + args.model = args.model_tag
27 +
28 + run_server(args)
29 +
30 +
31 +def interactive_cli(args: argparse.Namespace) -> None:
32 + registrer_signal_handlers()
33 +
34 + base_url = args.url
35 + api_key = args.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY")
36 + openai_client = OpenAI(api_key=api_key, base_url=base_url)
37 +
38 + if args.model_name:
39 + model_name = args.model_name
40 + else:
41 + available_models = openai_client.models.list()
42 + model_name = available_models.data[0].id
43 +
44 + print(f"Using model: {model_name}")
45 +
46 + if args.command == "complete":
47 + complete(model_name, openai_client)
48 + elif args.command == "chat":
49 + chat(args.system_prompt, model_name, openai_client)
50 +
51 +
52 +def complete(model_name: str, client: OpenAI) -> None:
53 + print("Please enter prompt to complete:")
54 + while True:
55 + input_prompt = input("> ")
56 +
57 + completion = client.completions.create(model=model_name,
58 + prompt=input_prompt)
59 + output = completion.choices[0].text
60 + print(output)
61 +
62 +
63 +def chat(system_prompt: Optional[str], model_name: str,
64 + client: OpenAI) -> None:
65 + conversation = []
66 + if system_prompt is not None:
67 + conversation.append({"role": "system", "content": system_prompt})
68 +
69 + print("Please enter a message for the chat model:")
70 + while True:
71 + input_message = input("> ")
72 + message = {"role": "user", "content": input_message}
73 + conversation.append(message)
74 +
75 + chat_completion = client.chat.completions.create(model=model_name,
76 + messages=conversation)
77 +
78 + response_message = chat_completion.choices[0].message
79 + output = response_message.content
80 +
81 + conversation.append(response_message)
82 + print(output)
83 +
84 +
85 +def _add_query_options(
86 + parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
87 + parser.add_argument(
88 + "--url",
89 + type=str,
90 + default="http://localhost:8000/v1",
91 + help="url of the running OpenAI-Compatible RESTful API server")
92 + parser.add_argument(
93 + "--model-name",
94 + type=str,
95 + default=None,
96 + help=("The model name used in prompt completion, default to "
97 + "the first model in list models API call."))
98 + parser.add_argument(
99 + "--api-key",
100 + type=str,
101 + default=None,
102 + help=(
103 + "API key for OpenAI services. If provided, this api key "
104 + "will overwrite the api key obtained through environment variables."
105 + ))
106 + return parser
107 +
108 +
109 +def main():
110 + parser = FlexibleArgumentParser(description="vLLM CLI")
111 + subparsers = parser.add_subparsers(required=True)
112 +
113 + serve_parser = subparsers.add_parser(
114 + "serve",
115 + help="Start the vLLM OpenAI Compatible API server",
116 + usage="vllm serve <model_tag> [options]")
117 + serve_parser.add_argument("model_tag",
118 + type=str,
119 + help="The model tag to serve")
120 + serve_parser = make_arg_parser(serve_parser)
121 + serve_parser.set_defaults(dispatch_function=serve)
122 +
123 + complete_parser = subparsers.add_parser(
124 + "complete",
125 + help=("Generate text completions based on the given prompt "
126 + "via the running API server"),
127 + usage="vllm complete [options]")
128 + _add_query_options(complete_parser)
129 + complete_parser.set_defaults(dispatch_function=interactive_cli,
130 + command="complete")
131 +
132 + chat_parser = subparsers.add_parser(
133 + "chat",
134 + help="Generate chat completions via the running API server",
135 + usage="vllm chat [options]")
136 + _add_query_options(chat_parser)
137 + chat_parser.add_argument(
138 + "--system-prompt",
139 + type=str,
140 + default=None,
141 + help=("The system prompt to be added to the chat template, "
142 + "used for models that support system prompts."))
143 + chat_parser.set_defaults(dispatch_function=interactive_cli, command="chat")
144 +
145 + args = parser.parse_args()
146 + # One of the sub commands should be executed.
147 + if hasattr(args, "dispatch_function"):
148 + args.dispatch_function(args)
149 + else:
150 + parser.print_help()
151 +
152 +
153 +if __name__ == "__main__":
154 + main()