@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f36444c4f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

multi-LoRA as extra models in OpenAI server (#2775)

how to serve the loras (mimicking the [multilora inference example](https://github.com/vllm-project/vllm/blob/main/examples/multilora_inference.py)): ```terminal $ export LORA_PATH=~/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/ $ python -m vllm.entrypoints.api_server \ --model meta-llama/Llama-2-7b-hf \ --enable-lora \ --lora-modules sql-lora=$LORA_PATH sql-lora2=$LORA_PATH ``` the above server will list 3 separate values if the user queries `/models`: one for the base served model, and one each for the specified lora modules. in this case sql-lora and sql-lora2 point to the same underlying lora, but this need not be the case. lora config values take the same values they do in EngineArgs no work has been done here to scope client permissions to specific models

jvmncs committed Feb 17, 2024 at 15:00 UTC 8f36444c4f9a55669bcb64e20b5588c0dd72bd93
7 files changed +200 -27
docs/source/models/lora.rst
+40 -1
@@ -49,4 +49,43 @@ the third parameter is the path to the LoRA adapter.
49
50
51 Check out `examples/multilora_inference.py <https://github.com/vllm-project/vllm/blob/main/examples/multilora_inference.py>`_
52 -for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
\ No newline at end of file
52 +for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
53 +
54 +Serving LoRA Adapters
55 +---------------------
56 +LoRA adapted models can also be served with the Open-AI compatible vLLM server. To do so, we use
57 +``--lora-modules {name}={path} {name}={path}`` to specify each LoRA module when we kickoff the server:
58 +
59 +.. code-block:: bash
60 +
61 + python -m vllm.entrypoints.api_server \
62 + --model meta-llama/Llama-2-7b-hf \
63 + --enable-lora \
64 + --lora-modules sql-lora=~/.cache/huggingface/hub/models--yard1--llama-2-7b-sql-lora-test/
65 +
66 +The server entrypoint accepts all other LoRA configuration parameters (``max_loras``, ``max_lora_rank``, ``max_cpu_loras``,
67 +etc.), which will apply to all forthcoming requests. Upon querying the ``/models`` endpoint, we should see our LoRA along
68 +with its base model:
69 +
70 +.. code-block:: bash
71 +
72 + curl localhost:8000/v1/models | jq .
73 + {
74 + "object": "list",
75 + "data": [
76 + {
77 + "id": "meta-llama/Llama-2-7b-hf",
78 + "object": "model",
79 + ...
80 + },
81 + {
82 + "id": "sql-lora",
83 + "object": "model",
84 + ...
85 + }
86 + ]
87 + }
88 +
89 +Requests can specify the LoRA adapter as if it were any other model via the ``model`` request parameter. The requests will be
90 +processed according to the server-wide LoRA configuration (i.e. in parallel with base model requests, and potentially other
91 +LoRA adapter requests if they were provided and ``max_loras`` is set high enough).
examples/multilora_inference.py
+3 -1
@@ -12,7 +12,9 @@ from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
12 from vllm.lora.request import LoRARequest
13
14
15 -def create_test_prompts(lora_path: str) -> List[Tuple[str, SamplingParams]]:
15 +def create_test_prompts(
16 + lora_path: str
17 +) -> List[Tuple[str, SamplingParams, Optional[LoRARequest]]]:
18 """Create a list of test prompts with their sampling parameters.
19
20 2 requests for base model, 4 requests for the LoRA. We define 2
tests/entrypoints/test_openai_server.py
+74 -15
@@ -7,9 +7,11 @@ import pytest
7 import requests
8 import ray # using Ray for overall ease of process management, parallel requests, and debugging.
9 import openai # use the official client for correctness check
10 +from huggingface_hub import snapshot_download # downloading lora to test lora requests
11
12 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
13 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
14 +LORA_NAME = "typeof/zephyr-7b-beta-lora" # technically this needs Mistral-7B-v0.1 as base, but we're not testing generation quality here
15
16 pytestmark = pytest.mark.asyncio
17
@@ -54,7 +56,12 @@ class ServerRunner:
56
57
58 @pytest.fixture(scope="session")
57 -def server():
59 +def zephyr_lora_files():
60 + return snapshot_download(repo_id=LORA_NAME)
61 +
62 +
63 +@pytest.fixture(scope="session")
64 +def server(zephyr_lora_files):
65 ray.init()
66 server_runner = ServerRunner.remote([
67 "--model",
@@ -64,6 +71,17 @@ def server():
71 "--max-model-len",
72 "8192",
73 "--enforce-eager",
74 + # lora config below
75 + "--enable-lora",
76 + "--lora-modules",
77 + f"zephyr-lora={zephyr_lora_files}",
78 + f"zephyr-lora2={zephyr_lora_files}",
79 + "--max-lora-rank",
80 + "64",
81 + "--max-cpu-loras",
82 + "2",
83 + "--max-num-seqs",
84 + "128"
85 ])
86 ray.get(server_runner.ready.remote())
87 yield server_runner
@@ -79,8 +97,25 @@ def client():
97 yield client
98
99
82 -async def test_single_completion(server, client: openai.AsyncOpenAI):
83 - completion = await client.completions.create(model=MODEL_NAME,
100 +async def test_check_models(server, client: openai.AsyncOpenAI):
101 + models = await client.models.list()
102 + models = models.data
103 + served_model = models[0]
104 + lora_models = models[1:]
105 + assert served_model.id == MODEL_NAME
106 + assert all(model.root == MODEL_NAME for model in models)
107 + assert lora_models[0].id == "zephyr-lora"
108 + assert lora_models[1].id == "zephyr-lora2"
109 +
110 +
111 +@pytest.mark.parametrize(
112 + # first test base model, then test loras
113 + "model_name",
114 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
115 +)
116 +async def test_single_completion(server, client: openai.AsyncOpenAI,
117 + model_name: str):
118 + completion = await client.completions.create(model=model_name,
119 prompt="Hello, my name is",
120 max_tokens=5,
121 temperature=0.0)
@@ -104,7 +139,13 @@ async def test_single_completion(server, client: openai.AsyncOpenAI):
139 completion.choices[0].text) >= 5
140
141
107 -async def test_single_chat_session(server, client: openai.AsyncOpenAI):
142 +@pytest.mark.parametrize(
143 + # just test 1 lora hereafter
144 + "model_name",
145 + [MODEL_NAME, "zephyr-lora"],
146 +)
147 +async def test_single_chat_session(server, client: openai.AsyncOpenAI,
148 + model_name: str):
149 messages = [{
150 "role": "system",
151 "content": "you are a helpful assistant"
@@ -115,7 +156,7 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI):
156
157 # test single completion
158 chat_completion = await client.chat.completions.create(
118 - model=MODEL_NAME,
159 + model=model_name,
160 messages=messages,
161 max_tokens=10,
162 )
@@ -139,11 +180,17 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI):
180 assert message.content is not None and len(message.content) >= 0
181
182
142 -async def test_completion_streaming(server, client: openai.AsyncOpenAI):
183 +@pytest.mark.parametrize(
184 + # just test 1 lora hereafter
185 + "model_name",
186 + [MODEL_NAME, "zephyr-lora"],
187 +)
188 +async def test_completion_streaming(server, client: openai.AsyncOpenAI,
189 + model_name: str):
190 prompt = "What is an LLM?"
191
192 single_completion = await client.completions.create(
146 - model=MODEL_NAME,
193 + model=model_name,
194 prompt=prompt,
195 max_tokens=5,
196 temperature=0.0,
@@ -152,7 +199,7 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI):
199 single_usage = single_completion.usage
200
201 stream = await client.completions.create(
155 - model=MODEL_NAME,
202 + model=model_name,
203 prompt=prompt,
204 max_tokens=5,
205 temperature=0.0,
@@ -166,7 +213,13 @@ async def test_completion_streaming(server, client: openai.AsyncOpenAI):
213 assert "".join(chunks) == single_output
214
215
169 -async def test_chat_streaming(server, client: openai.AsyncOpenAI):
216 +@pytest.mark.parametrize(
217 + # just test 1 lora hereafter
218 + "model_name",
219 + [MODEL_NAME, "zephyr-lora"],
220 +)
221 +async def test_chat_streaming(server, client: openai.AsyncOpenAI,
222 + model_name: str):
223 messages = [{
224 "role": "system",
225 "content": "you are a helpful assistant"
@@ -177,7 +230,7 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI):
230
231 # test single completion
232 chat_completion = await client.chat.completions.create(
180 - model=MODEL_NAME,
233 + model=model_name,
234 messages=messages,
235 max_tokens=10,
236 temperature=0.0,
@@ -187,7 +240,7 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI):
240
241 # test streaming
242 stream = await client.chat.completions.create(
190 - model=MODEL_NAME,
243 + model=model_name,
244 messages=messages,
245 max_tokens=10,
246 temperature=0.0,
@@ -204,10 +257,16 @@ async def test_chat_streaming(server, client: openai.AsyncOpenAI):
257 assert "".join(chunks) == output
258
259
207 -async def test_batch_completions(server, client: openai.AsyncOpenAI):
260 +@pytest.mark.parametrize(
261 + # just test 1 lora hereafter
262 + "model_name",
263 + [MODEL_NAME, "zephyr-lora"],
264 +)
265 +async def test_batch_completions(server, client: openai.AsyncOpenAI,
266 + model_name: str):
267 # test simple list
268 batch = await client.completions.create(
210 - model=MODEL_NAME,
269 + model=model_name,
270 prompt=["Hello, my name is", "Hello, my name is"],
271 max_tokens=5,
272 temperature=0.0,
@@ -217,7 +276,7 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI):
276
277 # test n = 2
278 batch = await client.completions.create(
220 - model=MODEL_NAME,
279 + model=model_name,
280 prompt=["Hello, my name is", "Hello, my name is"],
281 n=2,
282 max_tokens=5,
@@ -236,7 +295,7 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI):
295
296 # test streaming
297 batch = await client.completions.create(
239 - model=MODEL_NAME,
298 + model=model_name,
299 prompt=["Hello, my name is", "Hello, my name is"],
300 max_tokens=5,
301 temperature=0.0,
vllm/entrypoints/openai/api_server.py
+23 -1
@@ -23,6 +23,7 @@ from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRe
23 from vllm.logger import init_logger
24 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
25 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
26 +from vllm.entrypoints.openai.serving_engine import LoRA
27
28 TIMEOUT_KEEP_ALIVE = 5 # seconds
29
@@ -48,6 +49,16 @@ async def lifespan(app: fastapi.FastAPI):
49 app = fastapi.FastAPI(lifespan=lifespan)
50
51
52 +class LoRAParserAction(argparse.Action):
53 +
54 + def __call__(self, parser, namespace, values, option_string=None):
55 + lora_list = []
56 + for item in values:
57 + name, path = item.split('=')
58 + lora_list.append(LoRA(name, path))
59 + setattr(namespace, self.dest, lora_list)
60 +
61 +
62 def parse_args():
63 parser = argparse.ArgumentParser(
64 description="vLLM OpenAI-Compatible RESTful API server.")
@@ -81,6 +92,15 @@ def parse_args():
92 help="The model name used in the API. If not "
93 "specified, the model name will be the same as "
94 "the huggingface name.")
95 + parser.add_argument(
96 + "--lora-modules",
97 + type=str,
98 + default=None,
99 + nargs='+',
100 + action=LoRAParserAction,
101 + help=
102 + "LoRA module configurations in the format name=path. Multiple modules can be specified."
103 + )
104 parser.add_argument("--chat-template",
105 type=str,
106 default=None,
@@ -217,8 +237,10 @@ if __name__ == "__main__":
237 engine = AsyncLLMEngine.from_engine_args(engine_args)
238 openai_serving_chat = OpenAIServingChat(engine, served_model,
239 args.response_role,
240 + args.lora_modules,
241 args.chat_template)
221 - openai_serving_completion = OpenAIServingCompletion(engine, served_model)
242 + openai_serving_completion = OpenAIServingCompletion(
243 + engine, served_model, args.lora_modules)
244
245 # Register labels for metrics
246 add_global_metrics_labels(model_name=engine_args.model)
vllm/entrypoints/openai/serving_chat.py
+9 -4
@@ -1,7 +1,7 @@
1 import time
2 import codecs
3 from fastapi import Request
4 -from typing import AsyncGenerator, AsyncIterator, Union
4 +from typing import AsyncGenerator, AsyncIterator, Optional, List, Union
5 from vllm.logger import init_logger
6 from vllm.utils import random_uuid
7 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -11,7 +11,7 @@ from vllm.entrypoints.openai.protocol import (
11 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
12 UsageInfo)
13 from vllm.outputs import RequestOutput
14 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
14 +from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
15
16 logger = init_logger(__name__)
17
@@ -22,8 +22,11 @@ class OpenAIServingChat(OpenAIServing):
22 engine: AsyncLLMEngine,
23 served_model: str,
24 response_role: str,
25 + lora_modules: Optional[List[LoRA]] = None,
26 chat_template=None):
26 - super().__init__(engine=engine, served_model=served_model)
27 + super().__init__(engine=engine,
28 + served_model=served_model,
29 + lora_modules=lora_modules)
30 self.response_role = response_role
31 self._load_chat_template(chat_template)
32
@@ -64,11 +67,13 @@ class OpenAIServingChat(OpenAIServing):
67 token_ids = self._validate_prompt_and_tokenize(request,
68 prompt=prompt)
69 sampling_params = request.to_sampling_params()
70 + lora_request = self._maybe_get_lora(request)
71 except ValueError as e:
72 return self.create_error_response(str(e))
73
74 result_generator = self.engine.generate(prompt, sampling_params,
71 - request_id, token_ids)
75 + request_id, token_ids,
76 + lora_request)
77 # Streaming response
78 if request.stream:
79 return self.chat_completion_stream_generator(
vllm/entrypoints/openai/serving_completion.py
+11 -4
@@ -15,7 +15,7 @@ from .protocol import (
15 UsageInfo,
16 )
17 from vllm.outputs import RequestOutput
18 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
18 +from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
19
20 logger = init_logger(__name__)
21
@@ -249,8 +249,13 @@ def merge_async_iterators(*iterators):
249
250 class OpenAIServingCompletion(OpenAIServing):
251
252 - def __init__(self, engine: AsyncLLMEngine, served_model: str):
253 - super().__init__(engine=engine, served_model=served_model)
252 + def __init__(self,
253 + engine: AsyncLLMEngine,
254 + served_model: str,
255 + lora_modules: Optional[List[LoRA]] = None):
256 + super().__init__(engine=engine,
257 + served_model=served_model,
258 + lora_modules=lora_modules)
259
260 async def create_completion(self, request: CompletionRequest,
261 raw_request: Request):
@@ -284,6 +289,7 @@ class OpenAIServingCompletion(OpenAIServing):
289 generators = []
290 try:
291 sampling_params = request.to_sampling_params()
292 + lora_request = self._maybe_get_lora(request)
293 prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
294
295 for i, prompt in enumerate(prompts):
@@ -298,7 +304,8 @@ class OpenAIServingCompletion(OpenAIServing):
304 self.engine.generate(None,
305 sampling_params,
306 f"{request_id}-{i}",
301 - prompt_token_ids=input_ids))
307 + prompt_token_ids=input_ids,
308 + lora_request=lora_request))
309 except ValueError as e:
310 return self.create_error_response(str(e))
311
vllm/entrypoints/openai/serving_engine.py
+40 -1
@@ -1,4 +1,5 @@
1 import asyncio
2 +from dataclasses import dataclass
3 from http import HTTPStatus
4 from typing import Dict, List, Optional, Union
5 from vllm.logger import init_logger
@@ -9,15 +10,35 @@ from vllm.entrypoints.openai.protocol import (CompletionRequest,
10 ErrorResponse, LogProbs,
11 ModelCard, ModelList,
12 ModelPermission)
13 +from vllm.lora.request import LoRARequest
14
15 logger = init_logger(__name__)
16
17
18 +@dataclass
19 +class LoRA:
20 + name: str
21 + local_path: str
22 +
23 +
24 class OpenAIServing:
25
18 - def __init__(self, engine: AsyncLLMEngine, served_model: str):
26 + def __init__(self,
27 + engine: AsyncLLMEngine,
28 + served_model: str,
29 + lora_modules=Optional[List[LoRA]]):
30 self.engine = engine
31 self.served_model = served_model
32 + if lora_modules is None:
33 + self.lora_requests = []
34 + else:
35 + self.lora_requests = [
36 + LoRARequest(
37 + lora_name=lora.name,
38 + lora_int_id=i,
39 + lora_local_path=lora.local_path,
40 + ) for i, lora in enumerate(lora_modules, start=1)
41 + ]
42
43 self.max_model_len = 0
44 self.tokenizer = None
@@ -50,6 +71,13 @@ class OpenAIServing:
71 root=self.served_model,
72 permission=[ModelPermission()])
73 ]
74 + lora_cards = [
75 + ModelCard(id=lora.lora_name,
76 + root=self.served_model,
77 + permission=[ModelPermission()])
78 + for lora in self.lora_requests
79 + ]
80 + model_cards.extend(lora_cards)
81 return ModelList(data=model_cards)
82
83 def _create_logprobs(
@@ -99,11 +127,22 @@ class OpenAIServing:
127 async def _check_model(self, request) -> Optional[ErrorResponse]:
128 if request.model == self.served_model:
129 return
130 + if request.model in [lora.lora_name for lora in self.lora_requests]:
131 + return
132 return self.create_error_response(
133 message=f"The model `{request.model}` does not exist.",
134 err_type="NotFoundError",
135 status_code=HTTPStatus.NOT_FOUND)
136
137 + def _maybe_get_lora(self, request) -> Optional[LoRARequest]:
138 + if request.model == self.served_model:
139 + return
140 + for lora in self.lora_requests:
141 + if request.model == lora.lora_name:
142 + return lora
143 + # if _check_model has been called earlier, this will be unreachable
144 + raise ValueError("The model `{request.model}` does not exist.")
145 +
146 def _validate_prompt_and_tokenize(
147 self,
148 request: Union[ChatCompletionRequest, CompletionRequest],