@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 159 lines 5.8 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import logging
5 from collections.abc import Callable
6 from typing import TYPE_CHECKING, Any
7
8 import vllm.envs as envs
9
10 if TYPE_CHECKING:
11 from vllm.plugins.endpoint_plugins.interface import EndpointPlugin
12 from vllm.tasks import SupportedTask
13
14 logger = logging.getLogger(__name__)
15
16 # Default plugins group will be loaded in all processes(process0, engine core
17 # process and worker processes)
18 DEFAULT_PLUGINS_GROUP = "vllm.general_plugins"
19 # IO processor plugins group will be loaded in process0 only
20 IO_PROCESSOR_PLUGINS_GROUP = "vllm.io_processor_plugins"
21 # Platform plugins group will be loaded in all processes when
22 # `vllm.platforms.current_platform` is called and the value not initialized,
23 PLATFORM_PLUGINS_GROUP = "vllm.platform_plugins"
24 # Stat logger plugins group will be loaded in process0 only when serve vLLM with
25 # async mode.
26 STAT_LOGGER_PLUGINS_GROUP = "vllm.stat_logger_plugins"
27 # Endpoint plugins group is loaded in the API server front end process only.
28 # Each entry point resolves to a factory returning an `EndpointPlugin`
29 # (see `vllm/plugins/endpoint_plugins/interface.py`).
30 ENDPOINT_PLUGINS_GROUP = "vllm.endpoint_plugins"
31
32 # make sure one process only loads plugins once
33 plugins_loaded = False
34
35
36 def load_plugins_by_group(group: str) -> dict[str, Callable[[], Any]]:
37 """Load plugins registered under the given entry point group."""
38 from importlib.metadata import entry_points
39
40 allowed_plugins = envs.VLLM_PLUGINS
41
42 discovered_plugins = entry_points(group=group)
43 if len(discovered_plugins) == 0:
44 logger.debug("No plugins for group %s found.", group)
45 return {}
46
47 # Check if the only discovered plugin is the default one
48 is_default_group = group == DEFAULT_PLUGINS_GROUP
49 # Use INFO for non-default groups and DEBUG for the default group
50 log_level = logger.debug if is_default_group else logger.info
51
52 log_level("Available plugins for group %s:", group)
53 for plugin in discovered_plugins:
54 log_level("- %s -> %s", plugin.name, plugin.value)
55
56 if allowed_plugins is None:
57 log_level(
58 "All plugins in this group will be loaded. "
59 "Set `VLLM_PLUGINS` to control which plugins to load."
60 )
61
62 plugins = dict[str, Callable[[], Any]]()
63 for plugin in discovered_plugins:
64 if allowed_plugins is None or plugin.name in allowed_plugins:
65 if allowed_plugins is not None:
66 log_level("Loading plugin %s", plugin.name)
67
68 try:
69 func = plugin.load()
70 plugins[plugin.name] = func
71 except Exception:
72 logger.exception("Failed to load plugin %s", plugin.name)
73
74 return plugins
75
76
77 def load_general_plugins():
78 """WARNING: plugins can be loaded for multiple times in different
79 processes. They should be designed in a way that they can be loaded
80 multiple times without causing issues.
81 """
82 global plugins_loaded
83 if plugins_loaded:
84 return
85 plugins_loaded = True
86
87 plugins = load_plugins_by_group(group=DEFAULT_PLUGINS_GROUP)
88 # general plugins, we only need to execute the loaded functions
89 for func in plugins.values():
90 func()
91
92
93 def load_endpoint_plugins(
94 supported_tasks: "tuple[SupportedTask, ...] | None" = None,
95 ) -> "list[EndpointPlugin]":
96 """Discover, gate and instantiate `vllm.endpoint_plugins` entry points.
97
98 Endpoint plugins add HTTP routes to the API server, so they default to
99 not loading. Unlike other plugin groups, a plugin here is only
100 considered when it is explicitly named in `VLLM_PLUGINS`. This is a
101 stricter posture than `load_plugins_by_group` which "load everything unless
102 an allowlist says otherwise". This posture is taken to handle potentially
103 larger exposed network surface.
104
105 A discovered plugin is loaded only if both hold:
106 - it is named in `VLLM_PLUGINS` (enforced by not calling the loader
107 at all when `VLLM_PLUGINS` is unset). Note that `VLLM_PLUGINS=""`
108 parses to `[""]`, not `None`, so it is treated as a (non strict)
109 allowlist that matches no plugin name, not as "unset".
110 - its `required_tasks` is `None` or intersects `supported_tasks`.
111
112 Args:
113 supported_tasks: Tasks the server supports. `None` means no plugin
114 with a non `None` `required_tasks` will be loaded.
115
116 Returns:
117 Instantiated plugins that passed gating in discovery order.
118
119 """
120 from importlib.metadata import entry_points
121
122 if envs.VLLM_PLUGINS is None:
123 discovered = entry_points(group=ENDPOINT_PLUGINS_GROUP)
124 if discovered:
125 logger.warning(
126 "Found endpoint plugin(s) %s but VLLM_PLUGINS is not set. "
127 "Endpoint plugins add HTTP routes and must be explicitly "
128 "allowlisted via VLLM_PLUGINS to be loaded.",
129 [p.name for p in discovered],
130 )
131 return []
132
133 factories = load_plugins_by_group(ENDPOINT_PLUGINS_GROUP)
134
135 endpoint_plugins: list[EndpointPlugin] = []
136 for name, factory in factories.items():
137 try:
138 plugin = factory()
139 except Exception:
140 logger.exception("Failed to instantiate endpoint plugin %s", name)
141 continue
142
143 required_tasks = plugin.required_tasks
144 if required_tasks is not None and (
145 supported_tasks is None or not set(required_tasks) & set(supported_tasks)
146 ):
147 logger.info(
148 "Skipping endpoint plugin %s: requires one of tasks %s, "
149 "server supports %s",
150 name,
151 required_tasks,
152 supported_tasks,
153 )
154 continue
155
156 logger.info("Loaded endpoint plugin %s", name)
157 endpoint_plugins.append(plugin)
158
159 return endpoint_plugins