@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 66ded03067
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Allow model to be served under multiple names (#2894)

Co-authored-by: Alexandre Payot <alexandrep@graphcore.ai>

Harry Mellor committed Apr 18, 2024 at 08:16 UTC 66ded030677c7a0ca696f8d64e41637f4a358c00
5 files changed +26 -21
vllm/entrypoints/openai/api_server.py
+4 -4
@@ -150,18 +150,18 @@ if __name__ == "__main__":
150 logger.info(f"args: {args}")
151
152 if args.served_model_name is not None:
153 - served_model = args.served_model_name
153 + served_model_names = args.served_model_name
154 else:
155 - served_model = args.model
155 + served_model_names = [args.model]
156 engine_args = AsyncEngineArgs.from_cli_args(args)
157 engine = AsyncLLMEngine.from_engine_args(
158 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
159 - openai_serving_chat = OpenAIServingChat(engine, served_model,
159 + openai_serving_chat = OpenAIServingChat(engine, served_model_names,
160 args.response_role,
161 args.lora_modules,
162 args.chat_template)
163 openai_serving_completion = OpenAIServingCompletion(
164 - engine, served_model, args.lora_modules)
164 + engine, served_model_names, args.lora_modules)
165
166 app.root_path = args.root_path
167 uvicorn.run(app,
vllm/entrypoints/openai/cli_args.py
+7 -3
@@ -54,11 +54,15 @@ def make_arg_parser():
54 help="If provided, the server will require this key "
55 "to be presented in the header.")
56 parser.add_argument("--served-model-name",
57 + nargs="+",
58 type=str,
59 default=None,
59 - help="The model name used in the API. If not "
60 - "specified, the model name will be the same as "
61 - "the huggingface name.")
60 + help="The model name(s) used in the API. If multiple "
61 + "names are provided, the server will respond to any "
62 + "of the provided names. The model name in the model "
63 + "field of a response will be the first name in this "
64 + "list. If not specified, the model name will be the "
65 + "same as the `--model` argument.")
66 parser.add_argument(
67 "--lora-modules",
68 type=str,
vllm/entrypoints/openai/serving_chat.py
+4 -4
@@ -24,12 +24,12 @@ class OpenAIServingChat(OpenAIServing):
24
25 def __init__(self,
26 engine: AsyncLLMEngine,
27 - served_model: str,
27 + served_model_names: List[str],
28 response_role: str,
29 lora_modules: Optional[List[LoRA]] = None,
30 chat_template=None):
31 super().__init__(engine=engine,
32 - served_model=served_model,
32 + served_model_names=served_model_names,
33 lora_modules=lora_modules)
34 self.response_role = response_role
35 self._load_chat_template(chat_template)
@@ -109,7 +109,7 @@ class OpenAIServingChat(OpenAIServing):
109 result_generator: AsyncIterator[RequestOutput], request_id: str
110 ) -> Union[ErrorResponse, AsyncGenerator[str, None]]:
111
112 - model_name = request.model
112 + model_name = self.served_model_names[0]
113 created_time = int(time.time())
114 chunk_object_type = "chat.completion.chunk"
115 first_iteration = True
@@ -251,7 +251,7 @@ class OpenAIServingChat(OpenAIServing):
251 result_generator: AsyncIterator[RequestOutput],
252 request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]:
253
254 - model_name = request.model
254 + model_name = self.served_model_names[0]
255 created_time = int(time.time())
256 final_res: RequestOutput = None
257
vllm/entrypoints/openai/serving_completion.py
+3 -3
@@ -53,10 +53,10 @@ class OpenAIServingCompletion(OpenAIServing):
53
54 def __init__(self,
55 engine: AsyncLLMEngine,
56 - served_model: str,
56 + served_model_names: List[str],
57 lora_modules: Optional[List[LoRA]] = None):
58 super().__init__(engine=engine,
59 - served_model=served_model,
59 + served_model_names=served_model_names,
60 lora_modules=lora_modules)
61
62 async def create_completion(self, request: CompletionRequest,
@@ -79,7 +79,7 @@ class OpenAIServingCompletion(OpenAIServing):
79 return self.create_error_response(
80 "suffix is not currently supported")
81
82 - model_name = request.model
82 + model_name = self.served_model_names[0]
83 request_id = f"cmpl-{random_uuid()}"
84 created_time = int(time.time())
85
vllm/entrypoints/openai/serving_engine.py
+8 -7
@@ -29,10 +29,10 @@ class OpenAIServing:
29
30 def __init__(self,
31 engine: AsyncLLMEngine,
32 - served_model: str,
32 + served_model_names: List[str],
33 lora_modules=Optional[List[LoRA]]):
34 self.engine = engine
35 - self.served_model = served_model
35 + self.served_model_names = served_model_names
36 if lora_modules is None:
37 self.lora_requests = []
38 else:
@@ -74,13 +74,14 @@ class OpenAIServing:
74 async def show_available_models(self) -> ModelList:
75 """Show available models. Right now we only have one model."""
76 model_cards = [
77 - ModelCard(id=self.served_model,
78 - root=self.served_model,
77 + ModelCard(id=served_model_name,
78 + root=self.served_model_names[0],
79 permission=[ModelPermission()])
80 + for served_model_name in self.served_model_names
81 ]
82 lora_cards = [
83 ModelCard(id=lora.lora_name,
83 - root=self.served_model,
84 + root=self.served_model_names[0],
85 permission=[ModelPermission()])
86 for lora in self.lora_requests
87 ]
@@ -150,7 +151,7 @@ class OpenAIServing:
151 return json_str
152
153 async def _check_model(self, request) -> Optional[ErrorResponse]:
153 - if request.model == self.served_model:
154 + if request.model in self.served_model_names:
155 return
156 if request.model in [lora.lora_name for lora in self.lora_requests]:
157 return
@@ -160,7 +161,7 @@ class OpenAIServing:
161 status_code=HTTPStatus.NOT_FOUND)
162
163 def _maybe_get_lora(self, request) -> Optional[LoRARequest]:
163 - if request.model == self.served_model:
164 + if request.model in self.served_model_names:
165 return
166 for lora in self.lora_requests:
167 if request.model == lora.lora_name: