| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Contract for `vllm.endpoint_plugins` entry points. |
| 4 | |
| 5 | An endpoint plugin adds HTTP routes to the OpenAI compatible API server. |
| 6 | Its scope is HTTP surface only. It registers routes and optionally |
| 7 | per app state used by those routes. It must not open new paths into the |
| 8 | engine by reaching the engine the same way an in-tree serving handler does |
| 9 | via `EngineClient` (e.g. `engine_client.collective_rpc(...)`). |
| 10 | |
| 11 | If a plugin also needs engine side behavior (a new worker side RPC method, |
| 12 | a custom stat, etc.) pair this entry point with one registered under |
| 13 | `vllm.general_plugins` (see `vllm/plugins/__init__.py`). The |
| 14 | `general_plugins` entry installs the engine side method and the |
| 15 | `endpoint_plugins` entry exposes it over HTTP. The two are registered and |
| 16 | loaded independently where neither implies the other. |
| 17 | |
| 18 | Plugins are opt-in. See `load_endpoint_plugins` in `vllm/plugins/__init__.py` |
| 19 | for the loading/gating rules and `docs/usage/security.md` for the security |
| 20 | posture of exposing plugin defined routes. |
| 21 | |
| 22 | The CPU only render server (see `build_and_serve_renderer` in |
| 23 | `vllm/entrypoints/launchers/render`) has no `EngineClient`. A plugin |
| 24 | eligible for the `render` task (`required_tasks` is `None` or includes |
| 25 | `"render"`) still gets `attach_router` called but `init_state` receives |
| 26 | `engine_client=None`. Plugins that cannot function without an engine should |
| 27 | either exclude `"render"` from `required_tasks` or check for `None` in |
| 28 | `init_state`/their route handlers and degrade gracefully. |
| 29 | """ |
| 30 | |
| 31 | from argparse import Namespace |
| 32 | from typing import TYPE_CHECKING, Protocol, runtime_checkable |
| 33 | |
| 34 | from fastapi import FastAPI |
| 35 | from starlette.datastructures import State |
| 36 | |
| 37 | from vllm.engine.protocol import EngineClient |
| 38 | |
| 39 | if TYPE_CHECKING: |
| 40 | from vllm.tasks import SupportedTask |
| 41 | |
| 42 | |
| 43 | @runtime_checkable |
| 44 | class EndpointPlugin(Protocol): |
| 45 | """Protocol implemented by `vllm.endpoint_plugins` entry point factories. |
| 46 | |
| 47 | An entry point registered under the `vllm.endpoint_plugins` group must |
| 48 | resolve to a zero argument callable (a class or factory function) that |
| 49 | returns an object satisfying this protocol. |
| 50 | """ |
| 51 | |
| 52 | name: str |
| 53 | """Unique plugin name used in logs and for `VLLM_PLUGINS` allowlisting.""" |
| 54 | |
| 55 | required_tasks: "tuple[SupportedTask, ...] | None" |
| 56 | """Tasks the server must support for this plugin to be loaded. |
| 57 | |
| 58 | The plugin is loaded only if this set intersects the server's |
| 59 | `supported_tasks`. `None` means the plugin has no task requirement and |
| 60 | is always eligible (subject to the `VLLM_PLUGINS` allowlist). |
| 61 | """ |
| 62 | |
| 63 | def attach_router(self, app: FastAPI) -> None: |
| 64 | """Register this plugin's routes on `app`. |
| 65 | |
| 66 | Called once during `build_app()` after all core routers have been |
| 67 | attached. Routes attached here can shadow core routes with the same |
| 68 | path. There is currently no conflict enforcement (see RFC #46565 follow ups). |
| 69 | """ |
| 70 | ... |
| 71 | |
| 72 | async def init_state( |
| 73 | self, engine_client: "EngineClient | None", state: State, args: Namespace |
| 74 | ) -> None: |
| 75 | """Initialize per app state consumed by this plugin's routes. |
| 76 | |
| 77 | Called once during `init_app_state()` after core state has been |
| 78 | initialized. Use `engine_client` (e.g. `collective_rpc`) to reach |
| 79 | the engine. Do not open new engine access paths. |
| 80 | |
| 81 | `engine_client` is `None` on the CPU only render server which has |
| 82 | no engine. This only happens for plugins eligible for the `render` |
| 83 | task (`required_tasks` is `None` or includes `"render"`). Handle |
| 84 | `None` explicitly (e.g. skip engine dependent setup, or have route |
| 85 | handlers return an error) if the plugin is loadable for `render` but |
| 86 | cannot function without an engine. |
| 87 | """ |
| 88 | ... |
| 89 | |
| 90 | |
| 91 | def attach_endpoint_plugins( |
| 92 | app: FastAPI, supported_tasks: tuple["SupportedTask", ...] |
| 93 | ) -> None: |
| 94 | """Phase A of endpoint plugin wiring: discover, gate and attach routes. |
| 95 | |
| 96 | Attached last after all core routers. This is so endpoint plugin routes can |
| 97 | shadow core routes with the same path (see `EndpointPlugin.attach_router` |
| 98 | docstring). No-ops when no plugins are discovered/allowlisted. |
| 99 | """ |
| 100 | from vllm.plugins import load_endpoint_plugins |
| 101 | |
| 102 | endpoint_plugins = load_endpoint_plugins(supported_tasks) |
| 103 | for plugin in endpoint_plugins: |
| 104 | plugin.attach_router(app) |
| 105 | app.state.endpoint_plugins = endpoint_plugins |
| 106 | |
| 107 | |
| 108 | async def init_endpoint_plugins_state( |
| 109 | engine_client: EngineClient | None, state: State, args: Namespace |
| 110 | ) -> None: |
| 111 | """Phase B of endpoint plugin wiring: initialize per app plugin state. |
| 112 | |
| 113 | `state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A) |
| 114 | in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`) |
| 115 | build their own bare `State` without going through `build_app`. As a result |
| 116 | `endpoint_plugins` may be absent and are treated that the same as "none attached". |
| 117 | |
| 118 | `engine_client` is `None` for the CPU only render server which has no |
| 119 | engine (see `init_render_app_state`). Plugins must handle a `None` |
| 120 | `engine_client` themselves (see `EndpointPlugin.init_state`). |
| 121 | """ |
| 122 | for plugin in getattr(state, "endpoint_plugins", []): |
| 123 | await plugin.init_state(engine_client, state, args) |