A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[Frontend] add --quick option for vllm chat/complete (#18297)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
May 20, 2025 at 00:36 UTC
20d8ce81eb9ab8c9a259278d588b73dfb04b7e50
1 file changed
+29
vllm/entrypoints/cli/openai.py
+29
@@ -101,9 +101,18 @@ class ChatCommand(CLISubcommand):
101
model_name, client = _interactive_cli(args)
102
system_prompt = args.system_prompt
103
conversation: list[ChatCompletionMessageParam] = []
104
+
105
if system_prompt is not None:
106
conversation.append({"role": "system", "content": system_prompt})
107
108
+ if args.quick:
109
+ conversation.append({"role": "user", "content": args.quick})
110
+
111
+ chat_completion = client.chat.completions.create(
112
+ model=model_name, messages=conversation)
113
+ print(chat_completion.choices[0].message.content)
114
+ return
115
+
116
print("Please enter a message for the chat model:")
117
while True:
118
try:
@@ -136,6 +145,12 @@ class ChatCommand(CLISubcommand):
145
default=None,
146
help=("The system prompt to be added to the chat template, "
147
"used for models that support system prompts."))
148
+ chat_parser.add_argument("-q",
149
+ "--quick",
150
+ type=str,
151
+ metavar="MESSAGE",
152
+ help=("Send a single prompt as MESSAGE "
153
+ "and print the response, then exit."))
154
return chat_parser
155
156
@@ -149,6 +164,13 @@ class CompleteCommand(CLISubcommand):
164
@staticmethod
165
def cmd(args: argparse.Namespace) -> None:
166
model_name, client = _interactive_cli(args)
167
+
168
+ if args.quick:
169
+ completion = client.completions.create(model=model_name,
170
+ prompt=args.quick)
171
+ print(completion.choices[0].text)
172
+ return
173
+
174
print("Please enter prompt to complete:")
175
while True:
176
input_prompt = input("> ")
@@ -168,6 +190,13 @@ class CompleteCommand(CLISubcommand):
190
"via the running API server."),
191
usage="vllm complete [options]")
192
_add_query_options(complete_parser)
193
+ complete_parser.add_argument(
194
+ "-q",
195
+ "--quick",
196
+ type=str,
197
+ metavar="PROMPT",
198
+ help=
199
+ "Send a single prompt and print the completion output, then exit.")
200
return complete_parser
201
202