A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 01:00 UTC
[UX] Defer some imports on CLI paths to save ~2s (#40056)
[UX] Defer some imports on CLI paths to save ~2s (#40056) Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 16, 2026 at 22:48 UTC
1948d0c467da0b273b287665028ae82b9441c7c7
5 files changed
+68
-60
vllm/benchmarks/sweep/plot.py
+16
-14
@@ -8,7 +8,7 @@ from dataclasses import dataclass
8
from functools import partial
9
from pathlib import Path
10
from types import TracebackType
11
-from typing import ClassVar
11
+from typing import TYPE_CHECKING, ClassVar
12
13
from typing_extensions import Self, override
14
@@ -17,20 +17,8 @@ from vllm.utils.import_utils import PlaceholderModule
17
18
from .utils import sanitize_filename
19
20
-try:
21
- import matplotlib.pyplot as plt
22
-except ImportError:
23
- plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
24
-
25
-try:
20
+if TYPE_CHECKING:
21
import pandas as pd
27
-except ImportError:
28
- pd = PlaceholderModule("pandas")
29
-
30
-try:
31
- import seaborn as sns
32
-except ImportError:
33
- seaborn = PlaceholderModule("seaborn")
22
23
24
@dataclass
@@ -265,6 +253,20 @@ def _plot_fig(
253
fig_height: float,
254
fig_dpi: int,
255
):
256
+ # Lazy-import matplotlib/pandas/seaborn
257
+ try:
258
+ import matplotlib.pyplot as plt
259
+ except ImportError:
260
+ plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
261
+ try:
262
+ import pandas as pd
263
+ except ImportError:
264
+ pd = PlaceholderModule("pandas")
265
+ try:
266
+ import seaborn as sns
267
+ except ImportError:
268
+ sns = PlaceholderModule("seaborn")
269
+
270
fig_group, fig_data = fig_group_data
271
272
row_groups = full_groupby(
vllm/benchmarks/sweep/plot_pareto.py
+16
-14
@@ -6,7 +6,7 @@ from concurrent.futures import ProcessPoolExecutor
6
from dataclasses import dataclass
7
from functools import partial
8
from pathlib import Path
9
-from typing import ClassVar
9
+from typing import TYPE_CHECKING, ClassVar
10
11
from vllm.utils.collection_utils import full_groupby
12
from vllm.utils.import_utils import PlaceholderModule
@@ -14,20 +14,8 @@ from vllm.utils.import_utils import PlaceholderModule
14
from .plot import DummyExecutor, _json_load_bytes
15
from .utils import sanitize_filename
16
17
-try:
18
- import matplotlib.pyplot as plt
19
-except ImportError:
20
- plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
21
-
22
-try:
17
+if TYPE_CHECKING:
18
import pandas as pd
24
-except ImportError:
25
- pd = PlaceholderModule("pandas")
26
-
27
-try:
28
- import seaborn as sns
29
-except ImportError:
30
- seaborn = PlaceholderModule("seaborn")
19
20
21
def _first_present(run_data: dict[str, object], keys: list[str]):
@@ -195,6 +183,20 @@ def _plot_fig(
183
print("[END FIGURE]")
184
return
185
186
+ # Lazy-import matplotlib/pandas/seaborn
187
+ try:
188
+ import matplotlib.pyplot as plt
189
+ except ImportError:
190
+ plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
191
+ try:
192
+ import pandas as pd
193
+ except ImportError:
194
+ pd = PlaceholderModule("pandas")
195
+ try:
196
+ import seaborn as sns
197
+ except ImportError:
198
+ sns = PlaceholderModule("seaborn")
199
+
200
df = pd.DataFrame.from_records(fig_data)
201
df = df.dropna(subset=["tokens_per_user", "tokens_per_gpu"])
202
vllm/entrypoints/cli/__init__.py
-17
@@ -1,19 +1,2 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from vllm.entrypoints.cli.benchmark.latency import BenchmarkLatencySubcommand
4
-from vllm.entrypoints.cli.benchmark.mm_processor import (
5
- BenchmarkMMProcessorSubcommand,
6
-)
7
-from vllm.entrypoints.cli.benchmark.serve import BenchmarkServingSubcommand
8
-from vllm.entrypoints.cli.benchmark.startup import BenchmarkStartupSubcommand
9
-from vllm.entrypoints.cli.benchmark.sweep import BenchmarkSweepSubcommand
10
-from vllm.entrypoints.cli.benchmark.throughput import BenchmarkThroughputSubcommand
11
-
12
-__all__: list[str] = [
13
- "BenchmarkLatencySubcommand",
14
- "BenchmarkMMProcessorSubcommand",
15
- "BenchmarkServingSubcommand",
16
- "BenchmarkStartupSubcommand",
17
- "BenchmarkSweepSubcommand",
18
- "BenchmarkThroughputSubcommand",
19
-]
vllm/entrypoints/cli/benchmark/main.py
+34
-12
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import argparse
5
+import sys
6
import typing
7
8
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
@@ -14,6 +15,17 @@ else:
15
FlexibleArgumentParser = argparse.ArgumentParser
16
17
18
+def _import_bench_subcommand_modules() -> None:
19
+ # Imported lazily so `BenchmarkSubcommandBase` subclasses register only
20
+ # when `vllm bench` is actually invoked.
21
+ import vllm.entrypoints.cli.benchmark.latency # noqa: F401
22
+ import vllm.entrypoints.cli.benchmark.mm_processor # noqa: F401
23
+ import vllm.entrypoints.cli.benchmark.serve # noqa: F401
24
+ import vllm.entrypoints.cli.benchmark.startup # noqa: F401
25
+ import vllm.entrypoints.cli.benchmark.sweep # noqa: F401
26
+ import vllm.entrypoints.cli.benchmark.throughput # noqa: F401
27
+
28
+
29
class BenchmarkSubcommand(CLISubcommand):
30
"""The `bench` subcommand for the vLLM CLI."""
31
@@ -38,18 +50,28 @@ class BenchmarkSubcommand(CLISubcommand):
50
)
51
bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type")
52
41
- for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
42
- cmd_subparser = bench_subparsers.add_parser(
43
- cmd_cls.name,
44
- help=cmd_cls.help,
45
- description=cmd_cls.help,
46
- usage=f"vllm {self.name} {cmd_cls.name} [options]",
47
- )
48
- cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
49
- cmd_cls.add_cli_args(cmd_subparser)
50
- cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
51
- subcmd=f"{self.name} {cmd_cls.name}"
52
- )
53
+ # Only build the nested bench subparsers when the user is actually
54
+ # invoking `bench`; otherwise we'd drag in imports
55
+ # unnecessarily on every `vllm --help` and `vllm serve`.
56
+ # Scan for the first positional arg so global flags (e.g. `-v`)
57
+ # before the subcommand don't break detection.
58
+ first_positional = next(
59
+ (arg for arg in sys.argv[1:] if not arg.startswith("-")), None
60
+ )
61
+ if first_positional == self.name:
62
+ _import_bench_subcommand_modules()
63
+ for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
64
+ cmd_subparser = bench_subparsers.add_parser(
65
+ cmd_cls.name,
66
+ help=cmd_cls.help,
67
+ description=cmd_cls.help,
68
+ usage=f"vllm {self.name} {cmd_cls.name} [options]",
69
+ )
70
+ cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
71
+ cmd_cls.add_cli_args(cmd_subparser)
72
+ cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
73
+ subcmd=f"{self.name} {cmd_cls.name}"
74
+ )
75
return bench_parser
76
77
vllm/env_override.py
+2
-3
@@ -100,10 +100,9 @@ logger = init_logger(__name__)
100
# it avoids unintentional cuda initialization from torch.cuda.is_available()
101
os.environ["PYTORCH_NVML_BASED_CUDA_CHECK"] = "1"
102
103
-# see https://github.com/vllm-project/vllm/issues/10480
103
+# see https://github.com/vllm-project/vllm/issues/10480 and
104
+# https://github.com/vllm-project/vllm/issues/10619.
105
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
105
-# see https://github.com/vllm-project/vllm/issues/10619
106
-torch._inductor.config.compile_threads = 1
106
107
# Enable Triton autotuning result caching to disk by default.
108
# Without this, Triton re-runs autotuning on every process restart,