| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from asyncio import Lock |
| 5 | from collections import defaultdict |
| 6 | from http import HTTPStatus |
| 7 | |
| 8 | from vllm.config import ModelConfig |
| 9 | from vllm.engine.protocol import EngineClient |
| 10 | from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath |
| 11 | from vllm.entrypoints.serve import create_error_response |
| 12 | from vllm.entrypoints.serve.engine.protocol import ( |
| 13 | ErrorResponse, |
| 14 | ModelCard, |
| 15 | ModelList, |
| 16 | ModelPermission, |
| 17 | ) |
| 18 | from vllm.entrypoints.serve.lora.protocol import ( |
| 19 | LoadLoRAAdapterRequest, |
| 20 | UnloadLoRAAdapterRequest, |
| 21 | ) |
| 22 | from vllm.exceptions import LoRAAdapterNotFoundError |
| 23 | from vllm.logger import init_logger |
| 24 | from vllm.lora.request import LoRARequest |
| 25 | from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry |
| 26 | from vllm.utils.counter import AtomicCounter |
| 27 | |
| 28 | logger = init_logger(__name__) |
| 29 | |
| 30 | |
| 31 | class OpenAIModelRegistry: |
| 32 | """Read-only view of the loaded base models with no engine dependency. |
| 33 | |
| 34 | Suitable for CPU-only / render-only contexts that have no engine client |
| 35 | and no LoRA support. |
| 36 | """ |
| 37 | |
| 38 | def __init__( |
| 39 | self, |
| 40 | model_config: ModelConfig, |
| 41 | base_model_paths: list[BaseModelPath], |
| 42 | ) -> None: |
| 43 | self.model_config = model_config |
| 44 | self.base_model_paths = base_model_paths |
| 45 | self.lora_requests: dict[str, LoRARequest] = {} |
| 46 | |
| 47 | def model_name(self, lora_request: LoRARequest | None = None) -> str: |
| 48 | return self.base_model_paths[0].name |
| 49 | |
| 50 | def is_base_model(self, model_name: str) -> bool: |
| 51 | return any(model.name == model_name for model in self.base_model_paths) |
| 52 | |
| 53 | async def check_model(self, model_name: str | None) -> ErrorResponse | None: |
| 54 | """Return an ErrorResponse if model_name is not served, else None.""" |
| 55 | if not model_name or self.is_base_model(model_name): |
| 56 | return None |
| 57 | return create_error_response( |
| 58 | message=f"The model `{model_name}` does not exist.", |
| 59 | err_type="NotFoundError", |
| 60 | status_code=HTTPStatus.NOT_FOUND, |
| 61 | param="model", |
| 62 | ) |
| 63 | |
| 64 | async def show_available_models(self) -> ModelList: |
| 65 | """Show available models (base models only).""" |
| 66 | max_model_len = self.model_config.max_model_len |
| 67 | return ModelList( |
| 68 | data=[ |
| 69 | ModelCard( |
| 70 | id=base_model.name, |
| 71 | max_model_len=max_model_len, |
| 72 | root=base_model.model_path, |
| 73 | permission=[ModelPermission()], |
| 74 | ) |
| 75 | for base_model in self.base_model_paths |
| 76 | ] |
| 77 | ) |
| 78 | |
| 79 | async def resolve_lora(self, lora_name: str): |
| 80 | raise RuntimeError("The OpenAIModelRegistry has no LoRA support.") |
| 81 | |
| 82 | |
| 83 | class OpenAIServingModels: |
| 84 | """Shared instance to hold data about the loaded base model(s) and adapters. |
| 85 | |
| 86 | Handles the routes: |
| 87 | - /v1/models |
| 88 | - /v1/load_lora_adapter |
| 89 | - /v1/unload_lora_adapter |
| 90 | """ |
| 91 | |
| 92 | def __init__( |
| 93 | self, |
| 94 | engine_client: EngineClient, |
| 95 | base_model_paths: list[BaseModelPath], |
| 96 | *, |
| 97 | lora_modules: list[LoRAModulePath] | None = None, |
| 98 | ): |
| 99 | super().__init__() |
| 100 | |
| 101 | self.registry = OpenAIModelRegistry( |
| 102 | model_config=engine_client.model_config, |
| 103 | base_model_paths=base_model_paths, |
| 104 | ) |
| 105 | |
| 106 | self.engine_client = engine_client |
| 107 | self.base_model_paths = base_model_paths |
| 108 | |
| 109 | self.static_lora_modules = lora_modules |
| 110 | self.lora_requests: dict[str, LoRARequest] = {} |
| 111 | self.lora_id_counter = AtomicCounter(0) |
| 112 | |
| 113 | self.lora_resolvers: list[LoRAResolver] = [] |
| 114 | for lora_resolver_name in LoRAResolverRegistry.get_supported_resolvers(): |
| 115 | self.lora_resolvers.append( |
| 116 | LoRAResolverRegistry.get_resolver(lora_resolver_name) |
| 117 | ) |
| 118 | self.lora_resolver_lock: dict[str, Lock] = defaultdict(Lock) |
| 119 | |
| 120 | self.model_config = self.engine_client.model_config |
| 121 | self.renderer = self.engine_client.renderer |
| 122 | self.input_processor = self.engine_client.input_processor |
| 123 | |
| 124 | async def init_static_loras(self): |
| 125 | """Loads all static LoRA modules. |
| 126 | Raises if any fail to load""" |
| 127 | if self.static_lora_modules is None: |
| 128 | return |
| 129 | for lora in self.static_lora_modules: |
| 130 | load_request = LoadLoRAAdapterRequest( |
| 131 | lora_path=lora.path, |
| 132 | lora_name=lora.name, |
| 133 | is_3d_lora_weight=lora.is_3d_lora_weight, |
| 134 | ) |
| 135 | load_result = await self.load_lora_adapter( |
| 136 | request=load_request, base_model_name=lora.base_model_name |
| 137 | ) |
| 138 | if isinstance(load_result, ErrorResponse): |
| 139 | raise ValueError(load_result.error.message) |
| 140 | |
| 141 | def is_base_model(self, model_name: str) -> bool: |
| 142 | return self.registry.is_base_model(model_name) |
| 143 | |
| 144 | def model_name(self, lora_request: LoRARequest | None = None) -> str: |
| 145 | if lora_request is not None: |
| 146 | return lora_request.lora_name |
| 147 | return self.base_model_paths[0].name |
| 148 | |
| 149 | async def show_available_models(self) -> ModelList: |
| 150 | """Show available models. This includes the base model and all |
| 151 | adapters.""" |
| 152 | model_list = await self.registry.show_available_models() |
| 153 | lora_cards = [ |
| 154 | ModelCard( |
| 155 | id=lora.lora_name, |
| 156 | root=lora.path, |
| 157 | parent=lora.base_model_name |
| 158 | if lora.base_model_name |
| 159 | else self.base_model_paths[0].name, |
| 160 | permission=[ModelPermission()], |
| 161 | ) |
| 162 | for lora in self.lora_requests.values() |
| 163 | ] |
| 164 | model_list.data.extend(lora_cards) |
| 165 | return model_list |
| 166 | |
| 167 | async def load_lora_adapter( |
| 168 | self, request: LoadLoRAAdapterRequest, base_model_name: str | None = None |
| 169 | ) -> ErrorResponse | str: |
| 170 | lora_name = request.lora_name |
| 171 | |
| 172 | # Ensure atomicity based on the lora name |
| 173 | async with self.lora_resolver_lock[lora_name]: |
| 174 | error_check_ret = await self._check_load_lora_adapter_request(request) |
| 175 | if error_check_ret is not None: |
| 176 | return error_check_ret |
| 177 | |
| 178 | lora_path = request.lora_path |
| 179 | lora_int_id = ( |
| 180 | self.lora_requests[lora_name].lora_int_id |
| 181 | if lora_name in self.lora_requests |
| 182 | else self.lora_id_counter.inc(1) |
| 183 | ) |
| 184 | lora_request = LoRARequest( |
| 185 | lora_name=lora_name, |
| 186 | lora_int_id=lora_int_id, |
| 187 | lora_path=lora_path, |
| 188 | load_inplace=request.load_inplace, |
| 189 | is_3d_lora_weight=request.is_3d_lora_weight, |
| 190 | ) |
| 191 | if base_model_name is not None and self.is_base_model(base_model_name): |
| 192 | lora_request.base_model_name = base_model_name |
| 193 | |
| 194 | # Validate that the adapter can be loaded into the engine |
| 195 | # This will also preload it for incoming requests |
| 196 | try: |
| 197 | await self.engine_client.add_lora(lora_request) |
| 198 | except Exception as e: |
| 199 | if str( |
| 200 | LoRAAdapterNotFoundError( |
| 201 | lora_request.lora_name, lora_request.lora_path |
| 202 | ) |
| 203 | ) in str(e): |
| 204 | return create_error_response( |
| 205 | LoRAAdapterNotFoundError( |
| 206 | lora_request.lora_name, lora_request.lora_path |
| 207 | ) |
| 208 | ) |
| 209 | return create_error_response( |
| 210 | message=str(e), |
| 211 | err_type="InternalServerError", |
| 212 | status_code=HTTPStatus.INTERNAL_SERVER_ERROR, |
| 213 | ) |
| 214 | |
| 215 | previous = self.lora_requests.get(lora_name) |
| 216 | if previous is not None and previous.lora_path == lora_path: |
| 217 | logger.warning( |
| 218 | "Reloaded LoRA adapter '%s' in place from the same path '%s'. " |
| 219 | "Prefix-cache blocks computed with its previous weights are " |
| 220 | "not invalidated; load changed weights from a new path to " |
| 221 | "avoid reusing them.", |
| 222 | lora_name, |
| 223 | lora_path, |
| 224 | ) |
| 225 | self.lora_requests[lora_name] = lora_request |
| 226 | logger.info( |
| 227 | "Loaded new LoRA adapter: name '%s', path '%s'", lora_name, lora_path |
| 228 | ) |
| 229 | return f"Success: LoRA adapter '{lora_name}' added successfully." |
| 230 | |
| 231 | async def unload_lora_adapter( |
| 232 | self, request: UnloadLoRAAdapterRequest |
| 233 | ) -> ErrorResponse | str: |
| 234 | lora_name = request.lora_name |
| 235 | |
| 236 | # Ensure atomicity based on the lora name |
| 237 | async with self.lora_resolver_lock[lora_name]: |
| 238 | error_check_ret = await self._check_unload_lora_adapter_request(request) |
| 239 | if error_check_ret is not None: |
| 240 | return error_check_ret |
| 241 | |
| 242 | # Safe to delete now since we hold the lock |
| 243 | del self.lora_requests[lora_name] |
| 244 | logger.info("Removed LoRA adapter: name '%s'", lora_name) |
| 245 | return f"Success: LoRA adapter '{lora_name}' removed successfully." |
| 246 | |
| 247 | async def _check_load_lora_adapter_request( |
| 248 | self, request: LoadLoRAAdapterRequest |
| 249 | ) -> ErrorResponse | None: |
| 250 | # Check if both 'lora_name' and 'lora_path' are provided |
| 251 | if not request.lora_name or not request.lora_path: |
| 252 | return create_error_response( |
| 253 | message="Both 'lora_name' and 'lora_path' must be provided.", |
| 254 | err_type="InvalidUserInput", |
| 255 | status_code=HTTPStatus.BAD_REQUEST, |
| 256 | ) |
| 257 | |
| 258 | if self.is_base_model(request.lora_name): |
| 259 | return create_error_response( |
| 260 | message=f"The lora adapter '{request.lora_name}' conflicts with a " |
| 261 | "served base model.", |
| 262 | err_type="InvalidUserInput", |
| 263 | status_code=HTTPStatus.BAD_REQUEST, |
| 264 | ) |
| 265 | |
| 266 | # If not loading inplace |
| 267 | # Check if the lora adapter with the given name already exists |
| 268 | if not request.load_inplace and request.lora_name in self.lora_requests: |
| 269 | return create_error_response( |
| 270 | message=f"The lora adapter '{request.lora_name}' has already been " |
| 271 | "loaded. If you want to load the adapter in place, set 'load_inplace'" |
| 272 | " to True.", |
| 273 | err_type="InvalidUserInput", |
| 274 | status_code=HTTPStatus.BAD_REQUEST, |
| 275 | ) |
| 276 | |
| 277 | return None |
| 278 | |
| 279 | async def _check_unload_lora_adapter_request( |
| 280 | self, request: UnloadLoRAAdapterRequest |
| 281 | ) -> ErrorResponse | None: |
| 282 | # Check if 'lora_name' is not provided return an error |
| 283 | if not request.lora_name: |
| 284 | return create_error_response( |
| 285 | message="'lora_name' needs to be provided to unload a LoRA adapter.", |
| 286 | err_type="InvalidUserInput", |
| 287 | status_code=HTTPStatus.BAD_REQUEST, |
| 288 | ) |
| 289 | |
| 290 | # Check if the lora adapter with the given name exists |
| 291 | if request.lora_name not in self.lora_requests: |
| 292 | return create_error_response( |
| 293 | message=f"The lora adapter '{request.lora_name}' cannot be found.", |
| 294 | err_type="NotFoundError", |
| 295 | status_code=HTTPStatus.NOT_FOUND, |
| 296 | ) |
| 297 | |
| 298 | return None |
| 299 | |
| 300 | async def resolve_lora(self, lora_name: str) -> LoRARequest | ErrorResponse: |
| 301 | """Attempt to resolve a LoRA adapter using available resolvers. |
| 302 | |
| 303 | Args: |
| 304 | lora_name: Name/identifier of the LoRA adapter |
| 305 | |
| 306 | Returns: |
| 307 | LoRARequest if found and loaded successfully. |
| 308 | ErrorResponse (404) if no resolver finds the adapter. |
| 309 | ErrorResponse (400) if adapter(s) are found but none load. |
| 310 | |
| 311 | """ |
| 312 | async with self.lora_resolver_lock[lora_name]: |
| 313 | # First check if this LoRA is already loaded |
| 314 | if lora_name in self.lora_requests: |
| 315 | return self.lora_requests[lora_name] |
| 316 | |
| 317 | base_model_name = self.model_config.model |
| 318 | unique_id = self.lora_id_counter.inc(1) |
| 319 | found_adapter = False |
| 320 | |
| 321 | # Try to resolve using available resolvers |
| 322 | for resolver in self.lora_resolvers: |
| 323 | lora_request = await resolver.resolve_lora(base_model_name, lora_name) |
| 324 | |
| 325 | if lora_request is not None: |
| 326 | found_adapter = True |
| 327 | lora_request.lora_int_id = unique_id |
| 328 | |
| 329 | try: |
| 330 | await self.engine_client.add_lora(lora_request) |
| 331 | self.lora_requests[lora_name] = lora_request |
| 332 | logger.info( |
| 333 | "Resolved and loaded LoRA adapter '%s' using %s", |
| 334 | lora_name, |
| 335 | resolver.__class__.__name__, |
| 336 | ) |
| 337 | return lora_request |
| 338 | except BaseException as e: |
| 339 | logger.warning( |
| 340 | "Failed to load LoRA '%s' resolved by %s: %s. " |
| 341 | "Trying next resolver.", |
| 342 | lora_name, |
| 343 | resolver.__class__.__name__, |
| 344 | e, |
| 345 | ) |
| 346 | continue |
| 347 | |
| 348 | if found_adapter: |
| 349 | # An adapter was found, but all attempts to load it failed. |
| 350 | return create_error_response( |
| 351 | message=( |
| 352 | f"LoRA adapter '{lora_name}' was found but could not be loaded." |
| 353 | ), |
| 354 | err_type="BadRequestError", |
| 355 | status_code=HTTPStatus.BAD_REQUEST, |
| 356 | ) |
| 357 | else: |
| 358 | # No adapter was found |
| 359 | return create_error_response( |
| 360 | message=f"LoRA adapter {lora_name} does not exist", |
| 361 | err_type="NotFoundError", |
| 362 | status_code=HTTPStatus.NOT_FOUND, |
| 363 | ) |