[CLI] Add flag to print TTFT and TPS in `vllm chat` (#46775)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Benjamin Chislett committed
Jun 27, 2026 at 01:15 UTC
455f25aa13905e189b9268298c812f2048100f6f
2 files changed
+67
-9
docs/cli/README.md
+6
@@ -80,6 +80,9 @@ vllm chat --url http://{vllm-serve-host}:{vllm-serve-port}/v1
80
81
# Quick chat with a single prompt
82
vllm chat --quick "hi"
83
+
84
+# Print TTFT and throughput statistics after each response
85
+vllm chat --stats
86
```
87
88
See [vllm chat](./chat.md) for the full reference of all available arguments.
@@ -97,6 +100,9 @@ vllm complete --url http://{vllm-serve-host}:{vllm-serve-port}/v1
100
101
# Quick complete with a single prompt
102
vllm complete --quick "The future of AI is"
103
+
104
+# Print TTFT and throughput statistics after each response
105
+vllm complete --stats
106
```
107
108
See [vllm complete](./complete.md) for the full reference of all available arguments.
vllm/entrypoints/cli/openai.py
+61
-9
@@ -5,6 +5,7 @@ import argparse
5
import os
6
import signal
7
import sys
8
+import time
9
from typing import TYPE_CHECKING
10
11
from openai import OpenAI
@@ -44,25 +45,58 @@ def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]:
45
return model_name, openai_client
46
47
47
-def _print_chat_stream(stream) -> str:
48
+def _print_chat_stream(stream, stats: bool = False) -> str:
49
output = ""
50
+ start = time.perf_counter()
51
+ ttft: float | None = None
52
+ completion_tokens = 0
53
for chunk in stream:
54
+ if chunk.usage is not None:
55
+ completion_tokens = chunk.usage.completion_tokens
56
+ if not chunk.choices:
57
+ continue
58
delta = chunk.choices[0].delta
59
if delta.content:
60
+ if ttft is None:
61
+ ttft = time.perf_counter() - start
62
output += delta.content
63
print(delta.content, end="", flush=True)
64
print()
65
+ if stats:
66
+ _print_metrics(start, ttft, completion_tokens)
67
return output
68
69
58
-def _print_completion_stream(stream) -> str:
70
+def _print_metrics(start: float, ttft: float | None, completion_tokens: int) -> None:
71
+ total_time = time.perf_counter() - start
72
+ if ttft is None or total_time <= 0:
73
+ return
74
+ print(f"{'TTFT:':<5} {ttft * 1000:.2f} ms")
75
+ print(
76
+ f"{'TPS:':<5} {completion_tokens / total_time:.2f} tokens/s "
77
+ f"({completion_tokens} tokens in {total_time:.2f}s)"
78
+ )
79
+
80
+
81
+def _print_completion_stream(stream, stats: bool = False) -> str:
82
output = ""
83
+ start = time.perf_counter()
84
+ ttft: float | None = None
85
+ completion_tokens = 0
86
for chunk in stream:
87
+ if chunk.usage is not None:
88
+ completion_tokens = chunk.usage.completion_tokens
89
+ if not chunk.choices:
90
+ continue
91
text = chunk.choices[0].text
62
- if text is not None:
92
+ if text:
93
+ if ttft is None:
94
+ ttft = time.perf_counter() - start
95
output += text
96
print(text, end="", flush=True)
97
print()
98
+ if stats:
99
+ _print_metrics(start, ttft, completion_tokens)
100
return output
101
102
@@ -127,18 +161,23 @@ class ChatCommand(CLISubcommand):
161
def cmd(args: argparse.Namespace) -> None:
162
model_name, client = _interactive_cli(args)
163
system_prompt = args.system_prompt
164
+ stats = args.stats
165
conversation: list[ChatCompletionMessageParam] = []
166
167
if system_prompt is not None:
168
conversation.append({"role": "system", "content": system_prompt})
169
170
+ create_kwargs = {"model": model_name, "stream": True}
171
+ if stats:
172
+ create_kwargs["stream_options"] = {"include_usage": True}
173
+
174
if args.quick:
175
conversation.append({"role": "user", "content": args.quick})
176
177
stream = client.chat.completions.create(
139
- model=model_name, messages=conversation, stream=True
178
+ messages=conversation, **create_kwargs
179
)
141
- output = _print_chat_stream(stream)
180
+ output = _print_chat_stream(stream, stats)
181
conversation.append({"role": "assistant", "content": output})
182
return
183
@@ -151,9 +190,9 @@ class ChatCommand(CLISubcommand):
190
conversation.append({"role": "user", "content": input_message})
191
192
stream = client.chat.completions.create(
154
- model=model_name, messages=conversation, stream=True
193
+ messages=conversation, **create_kwargs
194
)
156
- output = _print_chat_stream(stream)
195
+ output = _print_chat_stream(stream, stats)
196
conversation.append({"role": "assistant", "content": output})
197
198
@staticmethod
@@ -176,6 +215,11 @@ class ChatCommand(CLISubcommand):
215
metavar="MESSAGE",
216
help=("Send a single prompt as MESSAGE and print the response, then exit."),
217
)
218
+ parser.add_argument(
219
+ "--stats",
220
+ action="store_true",
221
+ help="Print TTFT and TPS statistics after each response.",
222
+ )
223
return parser
224
225
def subparser_init(
@@ -198,6 +242,7 @@ class CompleteCommand(CLISubcommand):
242
@staticmethod
243
def cmd(args: argparse.Namespace) -> None:
244
model_name, client = _interactive_cli(args)
245
+ stats = args.stats
246
247
kwargs = {
248
"model": model_name,
@@ -205,10 +250,12 @@ class CompleteCommand(CLISubcommand):
250
}
251
if args.max_tokens:
252
kwargs["max_tokens"] = args.max_tokens
253
+ if stats:
254
+ kwargs["stream_options"] = {"include_usage": True}
255
256
if args.quick:
257
stream = client.completions.create(prompt=args.quick, **kwargs)
211
- _print_completion_stream(stream)
258
+ _print_completion_stream(stream, stats)
259
return
260
261
print("Please enter prompt to complete:")
@@ -218,7 +265,7 @@ class CompleteCommand(CLISubcommand):
265
except EOFError:
266
break
267
stream = client.completions.create(prompt=input_prompt, **kwargs)
221
- _print_completion_stream(stream)
268
+ _print_completion_stream(stream, stats)
269
270
@staticmethod
271
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
@@ -236,6 +283,11 @@ class CompleteCommand(CLISubcommand):
283
metavar="PROMPT",
284
help="Send a single prompt and print the completion output, then exit.",
285
)
286
+ parser.add_argument(
287
+ "--stats",
288
+ action="store_true",
289
+ help="Print TTFT and TPS statistics after each response.",
290
+ )
291
return parser
292
293
def subparser_init(