@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4cf260677
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 04:45 UTC

[Perf][CLI] Improve overall startup time (#19941)

Aaron Pham committed Jun 22, 2025 at 19:11 UTC c4cf26067755624eb94afda36ae1f679dec0d542
14 files changed +293 -103
.pre-commit-config.yaml
+5
@@ -115,6 +115,11 @@ repos:
115 entry: python tools/check_spdx_header.py
116 language: python
117 types: [python]
118 + - id: check-root-lazy-imports
119 + name: Check root lazy imports
120 + entry: python tools/check_init_lazy_imports.py
121 + language: python
122 + types: [python]
123 - id: check-filenames
124 name: Check for spaces in all filenames
125 entry: bash
tools/check_init_lazy_imports.py new
+108
@@ -0,0 +1,108 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Ensure we perform lazy loading in vllm/__init__.py.
4 +i.e: appears only within the ``if typing.TYPE_CHECKING:`` guard,
5 +**except** for a short whitelist.
6 +"""
7 +
8 +from __future__ import annotations
9 +
10 +import ast
11 +import pathlib
12 +import sys
13 +from collections.abc import Iterable
14 +from typing import Final
15 +
16 +REPO_ROOT: Final = pathlib.Path(__file__).resolve().parent.parent
17 +INIT_PATH: Final = REPO_ROOT / "vllm" / "__init__.py"
18 +
19 +# If you need to add items to whitelist, do it here.
20 +ALLOWED_IMPORTS: Final[frozenset[str]] = frozenset({
21 + "vllm.env_override",
22 +})
23 +ALLOWED_FROM_MODULES: Final[frozenset[str]] = frozenset({
24 + ".version",
25 +})
26 +
27 +
28 +def _is_internal(name: str | None, *, level: int = 0) -> bool:
29 + if level > 0:
30 + return True
31 + if name is None:
32 + return False
33 + return name.startswith("vllm.") or name == "vllm"
34 +
35 +
36 +def _fail(violations: Iterable[tuple[int, str]]) -> None:
37 + print("ERROR: Disallowed eager imports in vllm/__init__.py:\n",
38 + file=sys.stderr)
39 + for lineno, msg in violations:
40 + print(f" Line {lineno}: {msg}", file=sys.stderr)
41 + sys.exit(1)
42 +
43 +
44 +def main() -> None:
45 + source = INIT_PATH.read_text(encoding="utf-8")
46 + tree = ast.parse(source, filename=str(INIT_PATH))
47 +
48 + violations: list[tuple[int, str]] = []
49 +
50 + class Visitor(ast.NodeVisitor):
51 +
52 + def __init__(self) -> None:
53 + super().__init__()
54 + self._in_type_checking = False
55 +
56 + def visit_If(self, node: ast.If) -> None:
57 + guard_is_type_checking = False
58 + test = node.test
59 + if isinstance(test, ast.Attribute) and isinstance(
60 + test.value, ast.Name):
61 + guard_is_type_checking = (test.value.id == "typing"
62 + and test.attr == "TYPE_CHECKING")
63 + elif isinstance(test, ast.Name):
64 + guard_is_type_checking = test.id == "TYPE_CHECKING"
65 +
66 + if guard_is_type_checking:
67 + prev = self._in_type_checking
68 + self._in_type_checking = True
69 + for child in node.body:
70 + self.visit(child)
71 + self._in_type_checking = prev
72 + for child in node.orelse:
73 + self.visit(child)
74 + else:
75 + self.generic_visit(node)
76 +
77 + def visit_Import(self, node: ast.Import) -> None:
78 + if self._in_type_checking:
79 + return
80 + for alias in node.names:
81 + module_name = alias.name
82 + if _is_internal(
83 + module_name) and module_name not in ALLOWED_IMPORTS:
84 + violations.append((
85 + node.lineno,
86 + f"import '{module_name}' must be inside typing.TYPE_CHECKING", # noqa: E501
87 + ))
88 +
89 + def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
90 + if self._in_type_checking:
91 + return
92 + module_as_written = ("." * node.level) + (node.module or "")
93 + if _is_internal(
94 + node.module, level=node.level
95 + ) and module_as_written not in ALLOWED_FROM_MODULES:
96 + violations.append((
97 + node.lineno,
98 + f"from '{module_as_written}' import ... must be inside typing.TYPE_CHECKING", # noqa: E501
99 + ))
100 +
101 + Visitor().visit(tree)
102 +
103 + if violations:
104 + _fail(violations)
105 +
106 +
107 +if __name__ == "__main__":
108 + main()
vllm/__init__.py
+59 -16
@@ -1,29 +1,72 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
4 +
5 # The version.py should be independent library, and we always import the
6 # version library first. Such assumption is critical for some customization.
7 from .version import __version__, __version_tuple__ # isort:skip
8
9 +import typing
10 +
11 # The environment variables override should be imported before any other
12 # modules to ensure that the environment variables are set before any
13 # other modules are imported.
11 -import vllm.env_override # isort:skip # noqa: F401
12 -
13 -from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
14 -from vllm.engine.async_llm_engine import AsyncLLMEngine
15 -from vllm.engine.llm_engine import LLMEngine
16 -from vllm.entrypoints.llm import LLM
17 -from vllm.executor.ray_utils import initialize_ray_cluster
18 -from vllm.inputs import PromptType, TextPrompt, TokensPrompt
19 -from vllm.model_executor.models import ModelRegistry
20 -from vllm.outputs import (ClassificationOutput, ClassificationRequestOutput,
21 - CompletionOutput, EmbeddingOutput,
22 - EmbeddingRequestOutput, PoolingOutput,
23 - PoolingRequestOutput, RequestOutput, ScoringOutput,
24 - ScoringRequestOutput)
25 -from vllm.pooling_params import PoolingParams
26 -from vllm.sampling_params import SamplingParams
14 +import vllm.env_override # noqa: F401
15 +
16 +MODULE_ATTRS = {
17 + "AsyncEngineArgs": ".engine.arg_utils:AsyncEngineArgs",
18 + "EngineArgs": ".engine.arg_utils:EngineArgs",
19 + "AsyncLLMEngine": ".engine.async_llm_engine:AsyncLLMEngine",
20 + "LLMEngine": ".engine.llm_engine:LLMEngine",
21 + "LLM": ".entrypoints.llm:LLM",
22 + "initialize_ray_cluster": ".executor.ray_utils:initialize_ray_cluster",
23 + "PromptType": ".inputs:PromptType",
24 + "TextPrompt": ".inputs:TextPrompt",
25 + "TokensPrompt": ".inputs:TokensPrompt",
26 + "ModelRegistry": ".model_executor.models:ModelRegistry",
27 + "SamplingParams": ".sampling_params:SamplingParams",
28 + "PoolingParams": ".pooling_params:PoolingParams",
29 + "ClassificationOutput": ".outputs:ClassificationOutput",
30 + "ClassificationRequestOutput": ".outputs:ClassificationRequestOutput",
31 + "CompletionOutput": ".outputs:CompletionOutput",
32 + "EmbeddingOutput": ".outputs:EmbeddingOutput",
33 + "EmbeddingRequestOutput": ".outputs:EmbeddingRequestOutput",
34 + "PoolingOutput": ".outputs:PoolingOutput",
35 + "PoolingRequestOutput": ".outputs:PoolingRequestOutput",
36 + "RequestOutput": ".outputs:RequestOutput",
37 + "ScoringOutput": ".outputs:ScoringOutput",
38 + "ScoringRequestOutput": ".outputs:ScoringRequestOutput",
39 +}
40 +
41 +if typing.TYPE_CHECKING:
42 + from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
43 + from vllm.engine.async_llm_engine import AsyncLLMEngine
44 + from vllm.engine.llm_engine import LLMEngine
45 + from vllm.entrypoints.llm import LLM
46 + from vllm.executor.ray_utils import initialize_ray_cluster
47 + from vllm.inputs import PromptType, TextPrompt, TokensPrompt
48 + from vllm.model_executor.models import ModelRegistry
49 + from vllm.outputs import (ClassificationOutput,
50 + ClassificationRequestOutput, CompletionOutput,
51 + EmbeddingOutput, EmbeddingRequestOutput,
52 + PoolingOutput, PoolingRequestOutput,
53 + RequestOutput, ScoringOutput,
54 + ScoringRequestOutput)
55 + from vllm.pooling_params import PoolingParams
56 + from vllm.sampling_params import SamplingParams
57 +else:
58 +
59 + def __getattr__(name: str) -> typing.Any:
60 + from importlib import import_module
61 +
62 + if name in MODULE_ATTRS:
63 + module_name, attr_name = MODULE_ATTRS[name].split(":")
64 + module = import_module(module_name, __package__)
65 + return getattr(module, attr_name)
66 + else:
67 + raise AttributeError(
68 + f'module {__package__} has no attribute {name}')
69 +
70
71 __all__ = [
72 "__version__",
vllm/config.py
+16 -14
@@ -28,7 +28,7 @@ from pydantic.dataclasses import dataclass
28 from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
29 from torch.distributed import ProcessGroup, ReduceOp
30 from transformers import PretrainedConfig
31 -from typing_extensions import deprecated, runtime_checkable
31 +from typing_extensions import Self, deprecated, runtime_checkable
32
33 import vllm.envs as envs
34 from vllm import version
@@ -1537,7 +1537,6 @@ class CacheConfig:
1537 def __post_init__(self) -> None:
1538 self.swap_space_bytes = self.swap_space * GiB_bytes
1539
1540 - self._verify_args()
1540 self._verify_cache_dtype()
1541 self._verify_prefix_caching()
1542
@@ -1546,7 +1545,8 @@ class CacheConfig:
1545 # metrics info
1546 return {key: str(value) for key, value in self.__dict__.items()}
1547
1549 - def _verify_args(self) -> None:
1548 + @model_validator(mode='after')
1549 + def _verify_args(self) -> Self:
1550 if self.cpu_offload_gb < 0:
1551 raise ValueError("CPU offload space must be non-negative"
1552 f", but got {self.cpu_offload_gb}")
@@ -1556,6 +1556,8 @@ class CacheConfig:
1556 "GPU memory utilization must be less than 1.0. Got "
1557 f"{self.gpu_memory_utilization}.")
1558
1559 + return self
1560 +
1561 def _verify_cache_dtype(self) -> None:
1562 if self.cache_dtype == "auto":
1563 pass
@@ -1942,15 +1944,14 @@ class ParallelConfig:
1944 if self.distributed_executor_backend is None and self.world_size == 1:
1945 self.distributed_executor_backend = "uni"
1946
1945 - self._verify_args()
1946 -
1947 @property
1948 def use_ray(self) -> bool:
1949 return self.distributed_executor_backend == "ray" or (
1950 isinstance(self.distributed_executor_backend, type)
1951 and self.distributed_executor_backend.uses_ray)
1952
1953 - def _verify_args(self) -> None:
1953 + @model_validator(mode='after')
1954 + def _verify_args(self) -> Self:
1955 # Lazy import to avoid circular import
1956 from vllm.executor.executor_base import ExecutorBase
1957 from vllm.platforms import current_platform
@@ -1977,8 +1978,7 @@ class ParallelConfig:
1978 raise ValueError("Unable to use nsight profiling unless workers "
1979 "run with Ray.")
1980
1980 - assert isinstance(self.worker_extension_cls, str), (
1981 - "worker_extension_cls must be a string (qualified class name).")
1981 + return self
1982
1983
1984 PreemptionMode = Literal["swap", "recompute"]
@@ -2202,9 +2202,8 @@ class SchedulerConfig:
2202 self.max_num_partial_prefills, self.max_long_partial_prefills,
2203 self.long_prefill_token_threshold)
2204
2205 - self._verify_args()
2206 -
2207 - def _verify_args(self) -> None:
2205 + @model_validator(mode='after')
2206 + def _verify_args(self) -> Self:
2207 if (self.max_num_batched_tokens < self.max_model_len
2208 and not self.chunked_prefill_enabled):
2209 raise ValueError(
@@ -2263,6 +2262,8 @@ class SchedulerConfig:
2262 "must be greater than or equal to 1 and less than or equal to "
2263 f"max_num_partial_prefills ({self.max_num_partial_prefills}).")
2264
2265 + return self
2266 +
2267 @property
2268 def is_multi_step(self) -> bool:
2269 return self.num_scheduler_steps > 1
@@ -2669,8 +2670,6 @@ class SpeculativeConfig:
2670 if self.posterior_alpha is None:
2671 self.posterior_alpha = 0.3
2672
2672 - self._verify_args()
2673 -
2673 @staticmethod
2674 def _maybe_override_draft_max_model_len(
2675 speculative_max_model_len: Optional[int],
@@ -2761,7 +2760,8 @@ class SpeculativeConfig:
2760
2761 return draft_parallel_config
2762
2764 - def _verify_args(self) -> None:
2763 + @model_validator(mode='after')
2764 + def _verify_args(self) -> Self:
2765 if self.num_speculative_tokens is None:
2766 raise ValueError(
2767 "num_speculative_tokens must be provided with "
@@ -2812,6 +2812,8 @@ class SpeculativeConfig:
2812 "Eagle3 is only supported for Llama models. "
2813 f"Got {self.target_model_config.hf_text_config.model_type=}")
2814
2815 + return self
2816 +
2817 @property
2818 def num_lookahead_slots(self) -> int:
2819 """The number of additional slots the scheduler should allocate per
vllm/engine/arg_utils.py
+14 -1
@@ -3,7 +3,9 @@
3
4 # yapf: disable
5 import argparse
6 +import copy
7 import dataclasses
8 +import functools
9 import json
10 import sys
11 import threading
@@ -168,7 +170,8 @@ def get_type_hints(type_hint: TypeHint) -> set[TypeHint]:
170 return type_hints
171
172
171 -def get_kwargs(cls: ConfigType) -> dict[str, Any]:
173 +@functools.lru_cache(maxsize=30)
174 +def _compute_kwargs(cls: ConfigType) -> dict[str, Any]:
175 cls_docs = get_attr_docs(cls)
176 kwargs = {}
177 for field in fields(cls):
@@ -269,6 +272,16 @@ def get_kwargs(cls: ConfigType) -> dict[str, Any]:
272 return kwargs
273
274
275 +def get_kwargs(cls: ConfigType) -> dict[str, Any]:
276 + """Return argparse kwargs for the given Config dataclass.
277 +
278 + The heavy computation is cached via functools.lru_cache, and a deep copy
279 + is returned so callers can mutate the dictionary without affecting the
280 + cached version.
281 + """
282 + return copy.deepcopy(_compute_kwargs(cls))
283 +
284 +
285 @dataclass
286 class EngineArgs:
287 """Arguments for vLLM engine."""
vllm/entrypoints/cli/benchmark/main.py
+7 -2
@@ -1,10 +1,16 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from __future__ import annotations
5 +
6 import argparse
7 +import typing
8
9 from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
10 from vllm.entrypoints.cli.types import CLISubcommand
7 -from vllm.utils import FlexibleArgumentParser
11 +
12 +if typing.TYPE_CHECKING:
13 + from vllm.utils import FlexibleArgumentParser
14
15
16 class BenchmarkSubcommand(CLISubcommand):
@@ -23,7 +29,6 @@ class BenchmarkSubcommand(CLISubcommand):
29 def subparser_init(
30 self,
31 subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
26 -
32 bench_parser = subparsers.add_parser(
33 self.name,
34 help=self.help,
vllm/entrypoints/cli/collect_env.py
+8 -7
@@ -1,19 +1,21 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from __future__ import annotations
5 +
6 import argparse
7 +import typing
8
9 from vllm.collect_env import main as collect_env_main
10 from vllm.entrypoints.cli.types import CLISubcommand
8 -from vllm.utils import FlexibleArgumentParser
11 +
12 +if typing.TYPE_CHECKING:
13 + from vllm.utils import FlexibleArgumentParser
14
15
16 class CollectEnvSubcommand(CLISubcommand):
17 """The `collect-env` subcommand for the vLLM CLI. """
13 -
14 - def __init__(self):
15 - self.name = "collect-env"
16 - super().__init__()
18 + name = "collect-env"
19
20 @staticmethod
21 def cmd(args: argparse.Namespace) -> None:
@@ -23,12 +25,11 @@ class CollectEnvSubcommand(CLISubcommand):
25 def subparser_init(
26 self,
27 subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
26 - collect_env_parser = subparsers.add_parser(
28 + return subparsers.add_parser(
29 "collect-env",
30 help="Start collecting environment information.",
31 description="Start collecting environment information.",
32 usage="vllm collect-env")
31 - return collect_env_parser
33
34
35 def cmd_init() -> list[CLISubcommand]:
vllm/entrypoints/cli/main.py
+28 -22
@@ -1,27 +1,15 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +'''The CLI entrypoints of vLLM
4
4 -# The CLI entrypoint to vLLM.
5 +Note that all future modules must be lazily loaded within main
6 +to avoid certain eager import breakage.'''
7 +from __future__ import annotations
8 +
9 +import importlib.metadata
10 import signal
11 import sys
12
8 -import vllm.entrypoints.cli.benchmark.main
9 -import vllm.entrypoints.cli.collect_env
10 -import vllm.entrypoints.cli.openai
11 -import vllm.entrypoints.cli.run_batch
12 -import vllm.entrypoints.cli.serve
13 -import vllm.version
14 -from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG, cli_env_setup
15 -from vllm.utils import FlexibleArgumentParser
16 -
17 -CMD_MODULES = [
18 - vllm.entrypoints.cli.openai,
19 - vllm.entrypoints.cli.serve,
20 - vllm.entrypoints.cli.benchmark.main,
21 - vllm.entrypoints.cli.collect_env,
22 - vllm.entrypoints.cli.run_batch,
23 -]
24 -
13
14 def register_signal_handlers():
15
@@ -33,16 +21,34 @@ def register_signal_handlers():
21
22
23 def main():
24 + import vllm.entrypoints.cli.benchmark.main
25 + import vllm.entrypoints.cli.collect_env
26 + import vllm.entrypoints.cli.openai
27 + import vllm.entrypoints.cli.run_batch
28 + import vllm.entrypoints.cli.serve
29 + from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG, cli_env_setup
30 + from vllm.utils import FlexibleArgumentParser
31 +
32 + CMD_MODULES = [
33 + vllm.entrypoints.cli.openai,
34 + vllm.entrypoints.cli.serve,
35 + vllm.entrypoints.cli.benchmark.main,
36 + vllm.entrypoints.cli.collect_env,
37 + vllm.entrypoints.cli.run_batch,
38 + ]
39 +
40 cli_env_setup()
41
42 parser = FlexibleArgumentParser(
43 description="vLLM CLI",
44 epilog=VLLM_SUBCMD_PARSER_EPILOG,
45 )
42 - parser.add_argument('-v',
43 - '--version',
44 - action='version',
45 - version=vllm.version.__version__)
46 + parser.add_argument(
47 + '-v',
48 + '--version',
49 + action='version',
50 + version=importlib.metadata.version('vllm'),
51 + )
52 subparsers = parser.add_subparsers(required=False, dest="subparser")
53 cmds = {}
54 for cmd_module in CMD_MODULES:
vllm/entrypoints/cli/openai.py
+9 -13
@@ -1,18 +1,21 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -# Commands that act as an interactive OpenAI API client
3 +
4 +from __future__ import annotations
5
6 import argparse
7 import os
8 import signal
9 import sys
9 -from typing import Optional
10 +from typing import TYPE_CHECKING
11
12 from openai import OpenAI
13 from openai.types.chat import ChatCompletionMessageParam
14
15 from vllm.entrypoints.cli.types import CLISubcommand
15 -from vllm.utils import FlexibleArgumentParser
16 +
17 +if TYPE_CHECKING:
18 + from vllm.utils import FlexibleArgumentParser
19
20
21 def _register_signal_handlers():
@@ -42,8 +45,7 @@ def _interactive_cli(args: argparse.Namespace) -> tuple[str, OpenAI]:
45 return model_name, openai_client
46
47
45 -def chat(system_prompt: Optional[str], model_name: str,
46 - client: OpenAI) -> None:
48 +def chat(system_prompt: str | None, model_name: str, client: OpenAI) -> None:
49 conversation: list[ChatCompletionMessageParam] = []
50 if system_prompt is not None:
51 conversation.append({"role": "system", "content": system_prompt})
@@ -92,10 +94,7 @@ def _add_query_options(
94
95 class ChatCommand(CLISubcommand):
96 """The `chat` subcommand for the vLLM CLI. """
95 -
96 - def __init__(self):
97 - self.name = "chat"
98 - super().__init__()
97 + name = "chat"
98
99 @staticmethod
100 def cmd(args: argparse.Namespace) -> None:
@@ -157,10 +156,7 @@ class ChatCommand(CLISubcommand):
156
157 class CompleteCommand(CLISubcommand):
158 """The `complete` subcommand for the vLLM CLI. """
160 -
161 - def __init__(self):
162 - self.name = "complete"
163 - super().__init__()
159 + name = 'complete'
160
161 @staticmethod
162 def cmd(args: argparse.Namespace) -> None:
vllm/entrypoints/cli/run_batch.py
+19 -12
@@ -1,37 +1,42 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from __future__ import annotations
5 +
6 import argparse
7 import asyncio
6 -
7 -from prometheus_client import start_http_server
8 +import importlib.metadata
9 +import typing
10
11 from vllm.entrypoints.cli.types import CLISubcommand
10 -from vllm.entrypoints.logger import logger
11 -from vllm.entrypoints.openai.run_batch import main as run_batch_main
12 -from vllm.entrypoints.openai.run_batch import make_arg_parser
12 from vllm.entrypoints.utils import (VLLM_SUBCMD_PARSER_EPILOG,
13 show_filtered_argument_or_group_from_help)
15 -from vllm.utils import FlexibleArgumentParser
16 -from vllm.version import __version__ as VLLM_VERSION
14 +from vllm.logger import init_logger
15 +
16 +if typing.TYPE_CHECKING:
17 + from vllm.utils import FlexibleArgumentParser
18 +
19 +logger = init_logger(__name__)
20
21
22 class RunBatchSubcommand(CLISubcommand):
23 """The `run-batch` subcommand for vLLM CLI."""
21 -
22 - def __init__(self):
23 - self.name = "run-batch"
24 - super().__init__()
24 + name = "run-batch"
25
26 @staticmethod
27 def cmd(args: argparse.Namespace) -> None:
28 - logger.info("vLLM batch processing API version %s", VLLM_VERSION)
28 + from vllm.entrypoints.openai.run_batch import main as run_batch_main
29 +
30 + logger.info("vLLM batch processing API version %s",
31 + importlib.metadata.version("vllm"))
32 logger.info("args: %s", args)
33
34 # Start the Prometheus metrics server.
35 # LLMEngine uses the Prometheus client
36 # to publish metrics at the /metrics endpoint.
37 if args.enable_metrics:
38 + from prometheus_client import start_http_server
39 +
40 logger.info("Prometheus metrics enabled")
41 start_http_server(port=args.port, addr=args.url)
42 else:
@@ -42,6 +47,8 @@ class RunBatchSubcommand(CLISubcommand):
47 def subparser_init(
48 self,
49 subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
50 + from vllm.entrypoints.openai.run_batch import make_arg_parser
51 +
52 run_batch_parser = subparsers.add_parser(
53 "run-batch",
54 help="Run batch prompts and write results to file.",
vllm/entrypoints/cli/serve.py
+4 -7
@@ -9,8 +9,8 @@ import sys
9 import uvloop
10 import zmq
11
12 +import vllm
13 import vllm.envs as envs
13 -from vllm import AsyncEngineArgs
14 from vllm.entrypoints.cli.types import CLISubcommand
15 from vllm.entrypoints.openai.api_server import (run_server, run_server_worker,
16 setup_server)
@@ -38,10 +38,7 @@ logger = init_logger(__name__)
38
39 class ServeSubcommand(CLISubcommand):
40 """The `serve` subcommand for the vLLM CLI. """
41 -
42 - def __init__(self):
43 - self.name = "serve"
44 - super().__init__()
41 + name = "serve"
42
43 @staticmethod
44 def cmd(args: argparse.Namespace) -> None:
@@ -115,7 +112,7 @@ def run_headless(args: argparse.Namespace):
112 raise ValueError("api_server_count can't be set in headless mode")
113
114 # Create the EngineConfig.
118 - engine_args = AsyncEngineArgs.from_cli_args(args)
115 + engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
116 usage_context = UsageContext.OPENAI_API_SERVER
117 vllm_config = engine_args.create_engine_config(usage_context=usage_context)
118
@@ -175,7 +172,7 @@ def run_multi_api_server(args: argparse.Namespace):
172
173 listen_address, sock = setup_server(args)
174
178 - engine_args = AsyncEngineArgs.from_cli_args(args)
175 + engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
176 usage_context = UsageContext.OPENAI_API_SERVER
177 vllm_config = engine_args.create_engine_config(usage_context=usage_context)
178 model_config = vllm_config.model_config
vllm/entrypoints/cli/types.py
+5 -1
@@ -1,9 +1,13 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +from __future__ import annotations
5 +
6 import argparse
7 +import typing
8
6 -from vllm.utils import FlexibleArgumentParser
9 +if typing.TYPE_CHECKING:
10 + from vllm.utils import FlexibleArgumentParser
11
12
13 class CLISubcommand:
vllm/entrypoints/openai/run_batch.py
+11 -5
@@ -15,7 +15,7 @@ from tqdm import tqdm
15
16 from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
17 from vllm.engine.async_llm_engine import AsyncLLMEngine
18 -from vllm.entrypoints.logger import RequestLogger, logger
18 +from vllm.entrypoints.logger import RequestLogger
19 # yapf: disable
20 from vllm.entrypoints.openai.protocol import (BatchRequestInput,
21 BatchRequestOutput,
@@ -29,10 +29,13 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
29 from vllm.entrypoints.openai.serving_models import (BaseModelPath,
30 OpenAIServingModels)
31 from vllm.entrypoints.openai.serving_score import ServingScores
32 +from vllm.logger import init_logger
33 from vllm.usage.usage_lib import UsageContext
34 from vllm.utils import FlexibleArgumentParser, random_uuid
35 from vllm.version import __version__ as VLLM_VERSION
36
37 +logger = init_logger(__name__)
38 +
39
40 def make_arg_parser(parser: FlexibleArgumentParser):
41 parser.add_argument(
@@ -201,13 +204,16 @@ async def upload_data(output_url: str, data_or_file: str,
204 except Exception as e:
205 if attempt < max_retries:
206 logger.error(
204 - f"Failed to upload data (attempt {attempt}). "
205 - f"Error message: {str(e)}.\nRetrying in {delay} seconds..."
207 + "Failed to upload data (attempt %d). Error message: %s.\nRetrying in %d seconds...", # noqa: E501
208 + attempt,
209 + e,
210 + delay,
211 )
212 await asyncio.sleep(delay)
213 else:
209 - raise Exception(f"Failed to upload data (attempt {attempt}). "
210 - f"Error message: {str(e)}.") from e
214 + raise Exception(
215 + f"Failed to upload data (attempt {attempt}). Error message: {str(e)}." # noqa: E501
216 + ) from e
217
218
219 async def write_file(path_or_url: str, batch_outputs: list[BatchRequestOutput],
vllm/utils.py
-3
@@ -67,9 +67,6 @@ from torch.library import Library
67 from typing_extensions import Never, ParamSpec, TypeIs, assert_never
68
69 import vllm.envs as envs
70 -# NOTE: import triton_utils to make TritonPlaceholderModule work
71 -# if triton is unavailable
72 -import vllm.triton_utils # noqa: F401
70 from vllm.logger import enable_trace_function_call, init_logger
71
72 if TYPE_CHECKING: