| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from http import HTTPStatus |
| 4 | |
| 5 | from fastapi import Request |
| 6 | |
| 7 | from vllm import PromptType, SamplingParams, envs |
| 8 | from vllm.config import ModelConfig |
| 9 | from vllm.entrypoints.openai.models.serving import ( |
| 10 | OpenAIModelRegistry, |
| 11 | OpenAIServingModels, |
| 12 | ) |
| 13 | from vllm.entrypoints.pooling.typing import AnyPoolingRequest |
| 14 | from vllm.entrypoints.serve import create_error_response |
| 15 | from vllm.entrypoints.serve.engine.protocol import ErrorResponse |
| 16 | from vllm.entrypoints.serve.engine.typing import AnyRequest |
| 17 | from vllm.entrypoints.serve.utils.request_logger import RequestLogger |
| 18 | from vllm.exceptions import VLLMNotFoundError |
| 19 | from vllm.inputs import EngineInput |
| 20 | from vllm.lora.request import LoRARequest |
| 21 | from vllm.renderers.inputs.preprocess import ( |
| 22 | extract_prompt_components, |
| 23 | extract_prompt_len, |
| 24 | ) |
| 25 | from vllm.sampling_params import BeamSearchParams |
| 26 | from vllm.utils import random_uuid |
| 27 | |
| 28 | |
| 29 | class BaseServing: |
| 30 | def __init__( |
| 31 | self, |
| 32 | models: OpenAIServingModels | OpenAIModelRegistry, |
| 33 | model_config: ModelConfig, |
| 34 | request_logger: RequestLogger | None = None, |
| 35 | ): |
| 36 | self.models = models |
| 37 | self.model_config = model_config |
| 38 | self.request_logger = request_logger |
| 39 | |
| 40 | async def _check_model( |
| 41 | self, |
| 42 | request: AnyRequest | AnyPoolingRequest, |
| 43 | ) -> ErrorResponse | None: |
| 44 | if self.request_logger is not None: |
| 45 | self.request_logger.log_request_body(request) |
| 46 | |
| 47 | error_response = None |
| 48 | |
| 49 | if self._is_model_supported(request.model): |
| 50 | return None |
| 51 | if request.model in self.models.lora_requests: |
| 52 | return None |
| 53 | if ( |
| 54 | envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING |
| 55 | and request.model |
| 56 | and (load_result := await self.models.resolve_lora(request.model)) |
| 57 | ): |
| 58 | if isinstance(load_result, LoRARequest): |
| 59 | return None |
| 60 | if ( |
| 61 | isinstance(load_result, ErrorResponse) |
| 62 | and load_result.error.code == HTTPStatus.BAD_REQUEST.value |
| 63 | ): |
| 64 | error_response = load_result |
| 65 | |
| 66 | return error_response or self.create_error_response( |
| 67 | message=f"The model `{request.model}` does not exist.", |
| 68 | err_type="NotFoundError", |
| 69 | status_code=HTTPStatus.NOT_FOUND, |
| 70 | param="model", |
| 71 | ) |
| 72 | |
| 73 | def _is_model_supported(self, model_name: str | None) -> bool: |
| 74 | if not model_name: |
| 75 | return True |
| 76 | if envs.VLLM_SKIP_MODEL_NAME_VALIDATION: |
| 77 | return True |
| 78 | return self.models.is_base_model(model_name) |
| 79 | |
| 80 | @staticmethod |
| 81 | def create_error_response( |
| 82 | message: str | Exception, |
| 83 | err_type: str = "BadRequestError", |
| 84 | status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, |
| 85 | param: str | None = None, |
| 86 | ) -> ErrorResponse: |
| 87 | return create_error_response(message, err_type, status_code, param) |
| 88 | |
| 89 | def _extract_prompt_components(self, prompt: PromptType | EngineInput): |
| 90 | return extract_prompt_components(self.model_config, prompt) |
| 91 | |
| 92 | def _extract_prompt_text(self, prompt: PromptType | EngineInput): |
| 93 | return self._extract_prompt_components(prompt).text |
| 94 | |
| 95 | def _extract_prompt_len(self, prompt: EngineInput): |
| 96 | return extract_prompt_len(self.model_config, prompt) |
| 97 | |
| 98 | def _log_inputs( |
| 99 | self, |
| 100 | request_id: str, |
| 101 | inputs: PromptType | EngineInput, |
| 102 | params: SamplingParams | BeamSearchParams | None, |
| 103 | lora_request: LoRARequest | None, |
| 104 | ) -> None: |
| 105 | if self.request_logger is None: |
| 106 | return |
| 107 | |
| 108 | components = self._extract_prompt_components(inputs) |
| 109 | |
| 110 | self.request_logger.log_inputs( |
| 111 | request_id, |
| 112 | components.text, |
| 113 | components.token_ids, |
| 114 | components.embeds, |
| 115 | params=params, |
| 116 | lora_request=lora_request, |
| 117 | ) |
| 118 | |
| 119 | @staticmethod |
| 120 | def _base_request_id( |
| 121 | raw_request: Request | None, default: str | None = None |
| 122 | ) -> str | None: |
| 123 | """Pulls the request id to use from a header, if provided.""" |
| 124 | if raw_request is not None and ( |
| 125 | (req_id := raw_request.headers.get("X-Request-Id")) is not None |
| 126 | ): |
| 127 | return req_id |
| 128 | |
| 129 | return random_uuid() if default is None else default |
| 130 | |
| 131 | def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]: |
| 132 | """Retrieve the set of types from message content dicts up |
| 133 | until `_`; we use this to match potential multimodal data |
| 134 | with default per modality loras. |
| 135 | """ |
| 136 | message_types: set[str] = set() |
| 137 | |
| 138 | if not hasattr(request, "messages"): |
| 139 | return message_types |
| 140 | |
| 141 | messages = request.messages |
| 142 | if messages is None or isinstance(messages, (str, bytes)): |
| 143 | return message_types |
| 144 | |
| 145 | for message in messages: |
| 146 | if ( |
| 147 | isinstance(message, dict) |
| 148 | and "content" in message |
| 149 | and isinstance(message["content"], list) |
| 150 | ): |
| 151 | for content_dict in message["content"]: |
| 152 | if "type" in content_dict: |
| 153 | message_types.add(content_dict["type"].split("_")[0]) |
| 154 | return message_types |
| 155 | |
| 156 | def _get_active_default_mm_loras( |
| 157 | self, request: AnyRequest | AnyPoolingRequest |
| 158 | ) -> LoRARequest | None: |
| 159 | """Determine if there are any active default multimodal loras.""" |
| 160 | # TODO: Currently this is only enabled for chat completions |
| 161 | # to be better aligned with only being enabled for .generate |
| 162 | # when run offline. It would be nice to support additional |
| 163 | # tasks types in the future. |
| 164 | message_types = self._get_message_types(request) |
| 165 | default_mm_loras = set() |
| 166 | |
| 167 | for lora in self.models.lora_requests.values(): |
| 168 | # Best effort match for default multimodal lora adapters; |
| 169 | # There is probably a better way to do this, but currently |
| 170 | # this matches against the set of 'types' in any content lists |
| 171 | # up until '_', e.g., to match audio_url -> audio |
| 172 | if lora.lora_name in message_types: |
| 173 | default_mm_loras.add(lora) |
| 174 | |
| 175 | # Currently only support default modality specific loras if |
| 176 | # we have exactly one lora matched on the request. |
| 177 | if len(default_mm_loras) == 1: |
| 178 | return default_mm_loras.pop() |
| 179 | return None |
| 180 | |
| 181 | def _maybe_get_adapters( |
| 182 | self, |
| 183 | request: AnyRequest | AnyPoolingRequest, |
| 184 | supports_default_mm_loras: bool = False, |
| 185 | ) -> LoRARequest | None: |
| 186 | if request.model in self.models.lora_requests: |
| 187 | return self.models.lora_requests[request.model] |
| 188 | |
| 189 | # Currently only support default modality specific loras |
| 190 | # if we have exactly one lora matched on the request. |
| 191 | if supports_default_mm_loras: |
| 192 | default_mm_lora = self._get_active_default_mm_loras(request) |
| 193 | if default_mm_lora is not None: |
| 194 | return default_mm_lora |
| 195 | |
| 196 | if self._is_model_supported(request.model): |
| 197 | return None |
| 198 | |
| 199 | # if _check_model has been called earlier, this will be unreachable |
| 200 | raise VLLMNotFoundError(f"The model `{request.model}` does not exist.") |