A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Frontend] Move CLI code into vllm.cmd package (#12971)
Russell Bryant committed
Feb 13, 2025 at 02:12 UTC
d46d490c275091b4900ce10aa21032c222e85180
9 files changed
+348
-205
docs/source/design/arch_overview.md
+1
-1
@@ -66,7 +66,7 @@ This server can be started using the `vllm serve` command.
66
vllm serve <model>
67
```
68
69
-The code for the `vllm` CLI can be found in <gh-file:vllm/scripts.py>.
69
+The code for the `vllm` CLI can be found in <gh-file:vllm/entrypoints/cli/main.py>.
70
71
Sometimes you may see the API server entrypoint used directly instead of via the
72
`vllm` CLI command. For example:
setup.py
+1
-1
@@ -689,7 +689,7 @@ setup(
689
package_data=package_data,
690
entry_points={
691
"console_scripts": [
692
- "vllm=vllm.scripts:main",
692
+ "vllm=vllm.entrypoints.cli.main:main",
693
],
694
},
695
)
vllm/entrypoints/cli/__init__.py
vllm/entrypoints/cli/main.py
new
+79
@@ -0,0 +1,79 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# The CLI entrypoint to vLLM.
4
+import os
5
+import signal
6
+import sys
7
+
8
+import vllm.entrypoints.cli.openai
9
+import vllm.entrypoints.cli.serve
10
+import vllm.version
11
+from vllm.logger import init_logger
12
+from vllm.utils import FlexibleArgumentParser
13
+
14
+logger = init_logger(__name__)
15
+
16
+CMD_MODULES = [
17
+ vllm.entrypoints.cli.openai,
18
+ vllm.entrypoints.cli.serve,
19
+]
20
+
21
+
22
+def register_signal_handlers():
23
+
24
+ def signal_handler(sig, frame):
25
+ sys.exit(0)
26
+
27
+ signal.signal(signal.SIGINT, signal_handler)
28
+ signal.signal(signal.SIGTSTP, signal_handler)
29
+
30
+
31
+def env_setup():
32
+ # The safest multiprocessing method is `spawn`, as the default `fork` method
33
+ # is not compatible with some accelerators. The default method will be
34
+ # changing in future versions of Python, so we should use it explicitly when
35
+ # possible.
36
+ #
37
+ # We only set it here in the CLI entrypoint, because changing to `spawn`
38
+ # could break some existing code using vLLM as a library. `spawn` will cause
39
+ # unexpected behavior if the code is not protected by
40
+ # `if __name__ == "__main__":`.
41
+ #
42
+ # References:
43
+ # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
44
+ # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing
45
+ # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors
46
+ # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders
47
+ if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
48
+ logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'")
49
+ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
50
+
51
+
52
+def main():
53
+ env_setup()
54
+
55
+ parser = FlexibleArgumentParser(description="vLLM CLI")
56
+ parser.add_argument('-v',
57
+ '--version',
58
+ action='version',
59
+ version=vllm.version.__version__)
60
+ subparsers = parser.add_subparsers(required=False, dest="subparser")
61
+ cmds = {}
62
+ for cmd_module in CMD_MODULES:
63
+ new_cmds = cmd_module.cmd_init()
64
+ for cmd in new_cmds:
65
+ cmd.subparser_init(subparsers).set_defaults(
66
+ dispatch_function=cmd.cmd)
67
+ cmds[cmd.name] = cmd
68
+ args = parser.parse_args()
69
+ if args.subparser in cmds:
70
+ cmds[args.subparser].validate(args)
71
+
72
+ if hasattr(args, "dispatch_function"):
73
+ args.dispatch_function(args)
74
+ else:
75
+ parser.print_help()
76
+
77
+
78
+if __name__ == "__main__":
79
+ main()
vllm/entrypoints/cli/openai.py
new
+172
@@ -0,0 +1,172 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# Commands that act as an interactive OpenAI API client
3
+
4
+import argparse
5
+import os
6
+import signal
7
+import sys
8
+from typing import List, Optional, Tuple
9
+
10
+from openai import OpenAI
11
+from openai.types.chat import ChatCompletionMessageParam
12
+
13
+from vllm.entrypoints.cli.types import CLISubcommand
14
+from vllm.utils import FlexibleArgumentParser
15
+
16
+
17
+def _register_signal_handlers():
18
+
19
+ def signal_handler(sig, frame):
20
+ sys.exit(0)
21
+
22
+ signal.signal(signal.SIGINT, signal_handler)
23
+ signal.signal(signal.SIGTSTP, signal_handler)
24
+
25
+
26
+def _interactive_cli(args: argparse.Namespace) -> Tuple[str, OpenAI]:
27
+ _register_signal_handlers()
28
+
29
+ base_url = args.url
30
+ api_key = args.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY")
31
+ openai_client = OpenAI(api_key=api_key, base_url=base_url)
32
+
33
+ if args.model_name:
34
+ model_name = args.model_name
35
+ else:
36
+ available_models = openai_client.models.list()
37
+ model_name = available_models.data[0].id
38
+
39
+ print(f"Using model: {model_name}")
40
+
41
+ return model_name, openai_client
42
+
43
+
44
+def chat(system_prompt: Optional[str], model_name: str,
45
+ client: OpenAI) -> None:
46
+ conversation: List[ChatCompletionMessageParam] = []
47
+ if system_prompt is not None:
48
+ conversation.append({"role": "system", "content": system_prompt})
49
+
50
+ print("Please enter a message for the chat model:")
51
+ while True:
52
+ try:
53
+ input_message = input("> ")
54
+ except EOFError:
55
+ return
56
+ conversation.append({"role": "user", "content": input_message})
57
+
58
+ chat_completion = client.chat.completions.create(model=model_name,
59
+ messages=conversation)
60
+
61
+ response_message = chat_completion.choices[0].message
62
+ output = response_message.content
63
+
64
+ conversation.append(response_message) # type: ignore
65
+ print(output)
66
+
67
+
68
+def _add_query_options(
69
+ parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
70
+ parser.add_argument(
71
+ "--url",
72
+ type=str,
73
+ default="http://localhost:8000/v1",
74
+ help="url of the running OpenAI-Compatible RESTful API server")
75
+ parser.add_argument(
76
+ "--model-name",
77
+ type=str,
78
+ default=None,
79
+ help=("The model name used in prompt completion, default to "
80
+ "the first model in list models API call."))
81
+ parser.add_argument(
82
+ "--api-key",
83
+ type=str,
84
+ default=None,
85
+ help=(
86
+ "API key for OpenAI services. If provided, this api key "
87
+ "will overwrite the api key obtained through environment variables."
88
+ ))
89
+ return parser
90
+
91
+
92
+class ChatCommand(CLISubcommand):
93
+ """The `chat` subcommand for the vLLM CLI. """
94
+
95
+ def __init__(self):
96
+ self.name = "chat"
97
+ super().__init__()
98
+
99
+ @staticmethod
100
+ def cmd(args: argparse.Namespace) -> None:
101
+ model_name, client = _interactive_cli(args)
102
+ system_prompt = args.system_prompt
103
+ conversation: List[ChatCompletionMessageParam] = []
104
+ if system_prompt is not None:
105
+ conversation.append({"role": "system", "content": system_prompt})
106
+
107
+ print("Please enter a message for the chat model:")
108
+ while True:
109
+ try:
110
+ input_message = input("> ")
111
+ except EOFError:
112
+ return
113
+ conversation.append({"role": "user", "content": input_message})
114
+
115
+ chat_completion = client.chat.completions.create(
116
+ model=model_name, messages=conversation)
117
+
118
+ response_message = chat_completion.choices[0].message
119
+ output = response_message.content
120
+
121
+ conversation.append(response_message) # type: ignore
122
+ print(output)
123
+
124
+ def subparser_init(
125
+ self,
126
+ subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
127
+ chat_parser = subparsers.add_parser(
128
+ "chat",
129
+ help="Generate chat completions via the running API server",
130
+ usage="vllm chat [options]")
131
+ _add_query_options(chat_parser)
132
+ chat_parser.add_argument(
133
+ "--system-prompt",
134
+ type=str,
135
+ default=None,
136
+ help=("The system prompt to be added to the chat template, "
137
+ "used for models that support system prompts."))
138
+ return chat_parser
139
+
140
+
141
+class CompleteCommand(CLISubcommand):
142
+ """The `complete` subcommand for the vLLM CLI. """
143
+
144
+ def __init__(self):
145
+ self.name = "complete"
146
+ super().__init__()
147
+
148
+ @staticmethod
149
+ def cmd(args: argparse.Namespace) -> None:
150
+ model_name, client = _interactive_cli(args)
151
+ print("Please enter prompt to complete:")
152
+ while True:
153
+ input_prompt = input("> ")
154
+ completion = client.completions.create(model=model_name,
155
+ prompt=input_prompt)
156
+ output = completion.choices[0].text
157
+ print(output)
158
+
159
+ def subparser_init(
160
+ self,
161
+ subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
162
+ complete_parser = subparsers.add_parser(
163
+ "complete",
164
+ help=("Generate text completions based on the given prompt "
165
+ "via the running API server"),
166
+ usage="vllm complete [options]")
167
+ _add_query_options(complete_parser)
168
+ return complete_parser
169
+
170
+
171
+def cmd_init() -> List[CLISubcommand]:
172
+ return [ChatCommand(), CompleteCommand()]
vllm/entrypoints/cli/serve.py
new
+63
@@ -0,0 +1,63 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import argparse
4
+from typing import List
5
+
6
+import uvloop
7
+
8
+from vllm.engine.arg_utils import EngineArgs
9
+from vllm.entrypoints.cli.types import CLISubcommand
10
+from vllm.entrypoints.openai.api_server import run_server
11
+from vllm.entrypoints.openai.cli_args import (make_arg_parser,
12
+ validate_parsed_serve_args)
13
+from vllm.utils import FlexibleArgumentParser
14
+
15
+
16
+class ServeSubcommand(CLISubcommand):
17
+ """The `serve` subcommand for the vLLM CLI. """
18
+
19
+ def __init__(self):
20
+ self.name = "serve"
21
+ super().__init__()
22
+
23
+ @staticmethod
24
+ def cmd(args: argparse.Namespace) -> None:
25
+ # The default value of `--model`
26
+ if args.model != EngineArgs.model:
27
+ raise ValueError(
28
+ "With `vllm serve`, you should provide the model as a "
29
+ "positional argument instead of via the `--model` option.")
30
+
31
+ # EngineArgs expects the model name to be passed as --model.
32
+ args.model = args.model_tag
33
+
34
+ uvloop.run(run_server(args))
35
+
36
+ def validate(self, args: argparse.Namespace) -> None:
37
+ validate_parsed_serve_args(args)
38
+
39
+ def subparser_init(
40
+ self,
41
+ subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
42
+ serve_parser = subparsers.add_parser(
43
+ "serve",
44
+ help="Start the vLLM OpenAI Compatible API server",
45
+ usage="vllm serve <model_tag> [options]")
46
+ serve_parser.add_argument("model_tag",
47
+ type=str,
48
+ help="The model tag to serve")
49
+ serve_parser.add_argument(
50
+ "--config",
51
+ type=str,
52
+ default='',
53
+ required=False,
54
+ help="Read CLI options from a config file."
55
+ "Must be a YAML with the following options:"
56
+ "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#cli-reference"
57
+ )
58
+
59
+ return make_arg_parser(serve_parser)
60
+
61
+
62
+def cmd_init() -> List[CLISubcommand]:
63
+ return [ServeSubcommand()]
vllm/entrypoints/cli/types.py
new
+24
@@ -0,0 +1,24 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import argparse
4
+
5
+from vllm.utils import FlexibleArgumentParser
6
+
7
+
8
+class CLISubcommand:
9
+ """Base class for CLI argument handlers."""
10
+
11
+ name: str
12
+
13
+ @staticmethod
14
+ def cmd(args: argparse.Namespace) -> None:
15
+ raise NotImplementedError("Subclasses should implement this method")
16
+
17
+ def validate(self, args: argparse.Namespace) -> None:
18
+ # No validation by default
19
+ pass
20
+
21
+ def subparser_init(
22
+ self,
23
+ subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
24
+ raise NotImplementedError("Subclasses should implement this method")
vllm/entrypoints/openai/api_server.py
+2
-1
@@ -901,7 +901,8 @@ async def run_server(args, **uvicorn_kwargs) -> None:
901
902
if __name__ == "__main__":
903
# NOTE(simon):
904
- # This section should be in sync with vllm/scripts.py for CLI entrypoints.
904
+ # This section should be in sync with vllm/entrypoints/cli/main.py for CLI
905
+ # entrypoints.
906
parser = FlexibleArgumentParser(
907
description="vLLM OpenAI-Compatible RESTful API server.")
908
parser = make_arg_parser(parser)
vllm/scripts.py
+6
-202
@@ -1,210 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-# The CLI entrypoint to vLLM.
4
-import argparse
5
-import os
6
-import signal
7
-import sys
8
-from typing import List, Optional
9
-
10
-import uvloop
11
-from openai import OpenAI
12
-from openai.types.chat import ChatCompletionMessageParam
13
-
14
-import vllm.version
15
-from vllm.engine.arg_utils import EngineArgs
16
-from vllm.entrypoints.openai.api_server import run_server
17
-from vllm.entrypoints.openai.cli_args import (make_arg_parser,
18
- validate_parsed_serve_args)
3
+from vllm.entrypoints.cli.main import main as vllm_main
4
from vllm.logger import init_logger
20
-from vllm.utils import FlexibleArgumentParser
5
6
logger = init_logger(__name__)
7
8
25
-def register_signal_handlers():
26
-
27
- def signal_handler(sig, frame):
28
- sys.exit(0)
29
-
30
- signal.signal(signal.SIGINT, signal_handler)
31
- signal.signal(signal.SIGTSTP, signal_handler)
32
-
33
-
34
-def serve(args: argparse.Namespace) -> None:
35
- # The default value of `--model`
36
- if args.model != EngineArgs.model:
37
- raise ValueError(
38
- "With `vllm serve`, you should provide the model as a "
39
- "positional argument instead of via the `--model` option.")
40
-
41
- # EngineArgs expects the model name to be passed as --model.
42
- args.model = args.model_tag
43
-
44
- uvloop.run(run_server(args))
45
-
46
-
47
-def interactive_cli(args: argparse.Namespace) -> None:
48
- register_signal_handlers()
49
-
50
- base_url = args.url
51
- api_key = args.api_key or os.environ.get("OPENAI_API_KEY", "EMPTY")
52
- openai_client = OpenAI(api_key=api_key, base_url=base_url)
53
-
54
- if args.model_name:
55
- model_name = args.model_name
56
- else:
57
- available_models = openai_client.models.list()
58
- model_name = available_models.data[0].id
59
-
60
- print(f"Using model: {model_name}")
61
-
62
- if args.command == "complete":
63
- complete(model_name, openai_client)
64
- elif args.command == "chat":
65
- chat(args.system_prompt, model_name, openai_client)
66
-
67
-
68
-def complete(model_name: str, client: OpenAI) -> None:
69
- print("Please enter prompt to complete:")
70
- while True:
71
- input_prompt = input("> ")
72
-
73
- completion = client.completions.create(model=model_name,
74
- prompt=input_prompt)
75
- output = completion.choices[0].text
76
- print(output)
77
-
78
-
79
-def chat(system_prompt: Optional[str], model_name: str,
80
- client: OpenAI) -> None:
81
- conversation: List[ChatCompletionMessageParam] = []
82
- if system_prompt is not None:
83
- conversation.append({"role": "system", "content": system_prompt})
84
-
85
- print("Please enter a message for the chat model:")
86
- while True:
87
- input_message = input("> ")
88
- conversation.append({"role": "user", "content": input_message})
89
-
90
- chat_completion = client.chat.completions.create(model=model_name,
91
- messages=conversation)
92
-
93
- response_message = chat_completion.choices[0].message
94
- output = response_message.content
95
-
96
- conversation.append(response_message) # type: ignore
97
- print(output)
98
-
99
-
100
-def _add_query_options(
101
- parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
102
- parser.add_argument(
103
- "--url",
104
- type=str,
105
- default="http://localhost:8000/v1",
106
- help="url of the running OpenAI-Compatible RESTful API server")
107
- parser.add_argument(
108
- "--model-name",
109
- type=str,
110
- default=None,
111
- help=("The model name used in prompt completion, default to "
112
- "the first model in list models API call."))
113
- parser.add_argument(
114
- "--api-key",
115
- type=str,
116
- default=None,
117
- help=(
118
- "API key for OpenAI services. If provided, this api key "
119
- "will overwrite the api key obtained through environment variables."
120
- ))
121
- return parser
122
-
123
-
124
-def env_setup():
125
- # The safest multiprocessing method is `spawn`, as the default `fork` method
126
- # is not compatible with some accelerators. The default method will be
127
- # changing in future versions of Python, so we should use it explicitly when
128
- # possible.
129
- #
130
- # We only set it here in the CLI entrypoint, because changing to `spawn`
131
- # could break some existing code using vLLM as a library. `spawn` will cause
132
- # unexpected behavior if the code is not protected by
133
- # `if __name__ == "__main__":`.
134
- #
135
- # References:
136
- # - https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
137
- # - https://pytorch.org/docs/stable/notes/multiprocessing.html#cuda-in-multiprocessing
138
- # - https://pytorch.org/docs/stable/multiprocessing.html#sharing-cuda-tensors
139
- # - https://docs.habana.ai/en/latest/PyTorch/Getting_Started_with_PyTorch_and_Gaudi/Getting_Started_with_PyTorch.html?highlight=multiprocessing#torch-multiprocessing-for-dataloaders
140
- if "VLLM_WORKER_MULTIPROC_METHOD" not in os.environ:
141
- logger.debug("Setting VLLM_WORKER_MULTIPROC_METHOD to 'spawn'")
142
- os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
143
-
144
-
9
+# Backwards compatibility for the move from vllm.scripts to
10
+# vllm.entrypoints.cli.main
11
def main():
146
- env_setup()
147
-
148
- parser = FlexibleArgumentParser(description="vLLM CLI")
149
- parser.add_argument('-v',
150
- '--version',
151
- action='version',
152
- version=vllm.version.__version__)
153
-
154
- subparsers = parser.add_subparsers(required=True, dest="subparser")
155
-
156
- serve_parser = subparsers.add_parser(
157
- "serve",
158
- help="Start the vLLM OpenAI Compatible API server",
159
- usage="vllm serve <model_tag> [options]")
160
- serve_parser.add_argument("model_tag",
161
- type=str,
162
- help="The model tag to serve")
163
- serve_parser.add_argument(
164
- "--config",
165
- type=str,
166
- default='',
167
- required=False,
168
- help="Read CLI options from a config file."
169
- "Must be a YAML with the following options:"
170
- "https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html#cli-reference"
171
- )
172
-
173
- serve_parser = make_arg_parser(serve_parser)
174
- serve_parser.set_defaults(dispatch_function=serve)
175
-
176
- complete_parser = subparsers.add_parser(
177
- "complete",
178
- help=("Generate text completions based on the given prompt "
179
- "via the running API server"),
180
- usage="vllm complete [options]")
181
- _add_query_options(complete_parser)
182
- complete_parser.set_defaults(dispatch_function=interactive_cli,
183
- command="complete")
184
-
185
- chat_parser = subparsers.add_parser(
186
- "chat",
187
- help="Generate chat completions via the running API server",
188
- usage="vllm chat [options]")
189
- _add_query_options(chat_parser)
190
- chat_parser.add_argument(
191
- "--system-prompt",
192
- type=str,
193
- default=None,
194
- help=("The system prompt to be added to the chat template, "
195
- "used for models that support system prompts."))
196
- chat_parser.set_defaults(dispatch_function=interactive_cli, command="chat")
197
-
198
- args = parser.parse_args()
199
- if args.subparser == "serve":
200
- validate_parsed_serve_args(args)
201
-
202
- # One of the sub commands should be executed.
203
- if hasattr(args, "dispatch_function"):
204
- args.dispatch_function(args)
205
- else:
206
- parser.print_help()
207
-
208
-
209
-if __name__ == "__main__":
210
- main()
12
+ logger.warning("vllm.scripts.main() is deprecated. Please re-install "
13
+ "vllm or use vllm.entrypoints.cli.main.main() instead.")
14
+ vllm_main()