A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1] LoRA Support (#10957)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Feb 6, 2025 at 23:02 UTC
467a96a5415dc896170cecc0bb83d9c49c2f3c5e
16 files changed
+453
-56
tests/lora/conftest.py
+17
@@ -306,3 +306,20 @@ def llama_2_7b_engine_extra_embeddings():
306
def llama_2_7b_model_extra_embeddings(llama_2_7b_engine_extra_embeddings):
307
yield (llama_2_7b_engine_extra_embeddings.model_executor.driver_worker.
308
model_runner.model)
309
+
310
+
311
+@pytest.fixture(params=[True, False])
312
+def run_with_both_engines_lora(request, monkeypatch):
313
+ # Automatically runs tests twice, once with V1 and once without
314
+ use_v1 = request.param
315
+ # Tests decorated with `@skip_v1` are only run without v1
316
+ skip_v1 = request.node.get_closest_marker("skip_v1")
317
+
318
+ if use_v1:
319
+ if skip_v1:
320
+ pytest.skip("Skipping test on vllm V1")
321
+ monkeypatch.setenv('VLLM_USE_V1', '1')
322
+ else:
323
+ monkeypatch.setenv('VLLM_USE_V1', '0')
324
+
325
+ yield
tests/lora/test_baichuan.py
+8
@@ -42,6 +42,14 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
42
return generated_texts
43
44
45
+@pytest.fixture(autouse=True)
46
+def v1(run_with_both_engines_lora):
47
+ # Simple autouse wrapper to run both engines for each test
48
+ # This can be promoted up to conftest.py to run for every
49
+ # test in a package
50
+ pass
51
+
52
+
53
def test_baichuan_lora(baichuan_lora_files):
54
llm = vllm.LLM(MODEL_PATH,
55
max_model_len=1024,
tests/lora/test_chatglm3_tp.py
+13
@@ -2,6 +2,8 @@
2
3
from typing import List
4
5
+import pytest
6
+
7
import vllm
8
from tests.utils import fork_new_process_for_each_test
9
from vllm.lora.request import LoRARequest
@@ -47,6 +49,15 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
49
return generated_texts
50
51
52
+@pytest.fixture(autouse=True)
53
+def v1(run_with_both_engines_lora):
54
+ # Simple autouse wrapper to run both engines for each test
55
+ # This can be promoted up to conftest.py to run for every
56
+ # test in a package
57
+ pass
58
+
59
+
60
+@pytest.mark.skip_v1
61
@fork_new_process_for_each_test
62
def test_chatglm3_lora(chatglm3_lora_files):
63
llm = vllm.LLM(MODEL_PATH,
@@ -66,6 +77,7 @@ def test_chatglm3_lora(chatglm3_lora_files):
77
assert output2[i] == EXPECTED_LORA_OUTPUT[i]
78
79
80
+@pytest.mark.skip_v1
81
@multi_gpu_test(num_gpus=4)
82
@fork_new_process_for_each_test
83
def test_chatglm3_lora_tp4(chatglm3_lora_files):
@@ -87,6 +99,7 @@ def test_chatglm3_lora_tp4(chatglm3_lora_files):
99
assert output2[i] == EXPECTED_LORA_OUTPUT[i]
100
101
102
+@pytest.mark.skip_v1
103
@multi_gpu_test(num_gpus=4)
104
@fork_new_process_for_each_test
105
def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
tests/lora/test_gemma.py
+8
@@ -33,6 +33,14 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
33
return generated_texts
34
35
36
+@pytest.fixture(autouse=True)
37
+def v1(run_with_both_engines_lora):
38
+ # Simple autouse wrapper to run both engines for each test
39
+ # This can be promoted up to conftest.py to run for every
40
+ # test in a package
41
+ pass
42
+
43
+
44
@pytest.mark.xfail(current_platform.is_rocm(),
45
reason="There can be output mismatch on ROCm")
46
def test_gemma_lora(gemma_lora_files):
tests/lora/test_llama_tp.py
+12
@@ -2,6 +2,7 @@
2
3
from typing import List
4
5
+import pytest
6
import ray
7
8
import vllm
@@ -73,6 +74,14 @@ def generate_and_test(llm, sql_lora_files):
74
print("removing lora")
75
76
77
+@pytest.fixture(autouse=True)
78
+def v1(run_with_both_engines_lora):
79
+ # Simple autouse wrapper to run both engines for each test
80
+ # This can be promoted up to conftest.py to run for every
81
+ # test in a package
82
+ pass
83
+
84
+
85
@fork_new_process_for_each_test
86
def test_llama_lora(sql_lora_files):
87
@@ -85,6 +94,9 @@ def test_llama_lora(sql_lora_files):
94
generate_and_test(llm, sql_lora_files)
95
96
97
+# Skipping for v1 as v1 doesn't have a good way to expose the num_gpu_blocks
98
+# used by the engine yet.
99
+@pytest.mark.skip_v1
100
@fork_new_process_for_each_test
101
def test_llama_lora_warmup(sql_lora_files):
102
"""Test that the LLM initialization works with a warmup LORA path and
tests/lora/test_lora_bias_e2e.py
+11
@@ -30,6 +30,17 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
30
return generated_texts
31
32
33
+@pytest.fixture(autouse=True)
34
+def v1(run_with_both_engines_lora):
35
+ # Simple autouse wrapper to run both engines for each test
36
+ # This can be promoted up to conftest.py to run for every
37
+ # test in a package
38
+ pass
39
+
40
+
41
+# Skipping for V1 for now as we are hitting,
42
+# "Head size 80 is not supported by FlashAttention." error.
43
+@pytest.mark.skip_v1
44
@pytest.mark.parametrize("lora_bias", [True])
45
@pytest.mark.parametrize("fully_sharded", [True, False])
46
def test_lora_bias(lora_bias_files: str, lora_bias: bool, fully_sharded: bool):
tests/lora/test_phi.py
+13
@@ -2,6 +2,8 @@
2
3
from typing import List
4
5
+import pytest
6
+
7
import vllm
8
from vllm.lora.request import LoRARequest
9
@@ -48,6 +50,17 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
50
return generated_texts
51
52
53
+@pytest.fixture(autouse=True)
54
+def v1(run_with_both_engines_lora):
55
+ # Simple autouse wrapper to run both engines for each test
56
+ # This can be promoted up to conftest.py to run for every
57
+ # test in a package
58
+ pass
59
+
60
+
61
+# Skipping for V1 for now as we are hitting,
62
+# "Head size 80 is not supported by FlashAttention." error.
63
+@pytest.mark.skip_v1
64
def test_phi2_lora(phi2_lora_files):
65
# We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
66
# Otherwise, the lora-test will fail due to CUDA OOM.
tests/lora/test_quant_model.py
+8
@@ -70,6 +70,14 @@ def do_sample(llm: vllm.LLM,
70
return generated_texts
71
72
73
+@pytest.fixture(autouse=True)
74
+def v1(run_with_both_engines_lora):
75
+ # Simple autouse wrapper to run both engines for each test
76
+ # This can be promoted up to conftest.py to run for every
77
+ # test in a package
78
+ pass
79
+
80
+
81
@pytest.mark.parametrize("model", MODELS)
82
@pytest.mark.parametrize("tp_size", [1])
83
def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
tests/v1/core/test_kv_cache_utils.py
+1
-1
@@ -163,7 +163,7 @@ def test_generate_block_hash_extra_keys():
163
164
# Test with no overlap
165
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 6, 10, 0)
166
- assert extra_keys == ()
166
+ assert extra_keys is None
167
assert next_mm_idx == 1
168
169
# Test with multiple extra keys
vllm/lora/layers.py
+5
-3
@@ -16,8 +16,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
16
get_tensor_model_parallel_world_size,
17
split_tensor_along_last_dim,
18
tensor_model_parallel_all_gather,
19
- tensor_model_parallel_all_reduce,
20
- tensor_model_parallel_gather)
19
+ tensor_model_parallel_all_reduce)
20
from vllm.distributed.utils import divide
21
# yapf: disable
22
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -1043,7 +1042,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1042
logits = lm_head.linear_method.apply(lm_head, hidden_states)
1043
if embedding_bias is not None:
1044
logits += embedding_bias
1046
- logits = tensor_model_parallel_gather(logits)
1045
+
1046
+ # Gather logits for TP
1047
+ logits = self.base_layer._gather_logits(logits)
1048
+
1049
if logits is None:
1050
return None
1051
vllm/model_executor/layers/logits_processor.py
+17
-11
@@ -51,7 +51,6 @@ class LogitsProcessor(nn.Module):
51
# Soft cap the logits. Used in Gemma 2.
52
self.soft_cap = soft_cap
53
# Whether to use gather or all-gather to gather the logits.
54
-
54
parallel_config = get_current_vllm_config().parallel_config
55
self.use_all_gather = current_platform.is_tpu() \
56
or envs.VLLM_USE_V1 \
@@ -88,6 +87,20 @@ class LogitsProcessor(nn.Module):
87
88
return logits
89
90
+ def _gather_logits(self, logits: torch.Tensor) -> torch.Tensor:
91
+ """gather/all-gather the logits tensor across model parallel group."""
92
+ if self.use_all_gather:
93
+ # Gather is not supported for some devices such as TPUs.
94
+ # Use all-gather instead.
95
+ # NOTE(woosuk): Here, the outputs of every device should not be None
96
+ # because XLA requires strict SPMD among all devices. Every device
97
+ # should execute the same operations after gathering the logits.
98
+ logits = tensor_model_parallel_all_gather(logits)
99
+ else:
100
+ # None may be returned for rank > 0
101
+ logits = tensor_model_parallel_gather(logits)
102
+ return logits
103
+
104
def _get_logits(
105
self,
106
hidden_states: torch.Tensor,
@@ -99,16 +112,9 @@ class LogitsProcessor(nn.Module):
112
hidden_states,
113
bias=embedding_bias)
114
102
- if self.use_all_gather:
103
- # Gather is not supported for some devices such as TPUs.
104
- # Use all-gather instead.
105
- # NOTE(woosuk): Here, the outputs of every device should not be None
106
- # because XLA requires strict SPMD among all devices. Every device
107
- # should execute the same operations after gathering the logits.
108
- logits = tensor_model_parallel_all_gather(logits)
109
- else:
110
- # None may be returned for rank > 0
111
- logits = tensor_model_parallel_gather(logits)
115
+ # Gather logits for TP
116
+ logits = self._gather_logits(logits)
117
+
118
# Remove paddings in vocab (if any).
119
if logits is not None:
120
logits = logits[..., :self.org_vocab_size]
vllm/v1/core/kv_cache_utils.py
+75
-26
@@ -170,14 +170,28 @@ class FreeKVCacheBlockQueue:
170
return ret
171
172
173
-def generate_block_hash_extra_keys(
174
- request: Request, start_token_idx: int, end_token_idx: int,
175
- start_mm_idx: int) -> Tuple[Optional[Tuple[Any, ...]], int]:
176
- """Generate extra keys for the block hash. The extra keys can come from
177
- the multi-modal inputs and request specific metadata (e.g., LoRA ID).
178
- For multi-modal inputs, the extra keys are (mm_hash, start_offset) that
179
- indicate a mm input contained in the block and its starting offset in
180
- the block tokens.
173
+def need_extra_keys(request: Request) -> bool:
174
+ """Check whether the blocks allocated to this request need extra hash keys.
175
+
176
+ Args:
177
+ request (Request): The request.
178
+
179
+ Returns:
180
+ bool: Whether blocks allocated to this request need extra hash keys.
181
+ """
182
+
183
+ # Multimodal requests need to include the MM hash.
184
+ # LoRA requests need to include the LoRA ID.
185
+ return bool(request.mm_positions) or (request.lora_request is not None)
186
+
187
+
188
+def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
189
+ end_token_idx: int,
190
+ start_mm_idx: int) -> Tuple[List[Any], int]:
191
+ """Generate extra keys related to MultiModal request for block hash
192
+ computation. For multi-modal inputs, the extra keys are
193
+ (mm_hash, start_offset) that indicate a mm input contained in the
194
+ block and its starting offset in the block tokens.
195
196
Args:
197
request: The request object.
@@ -188,10 +202,11 @@ def generate_block_hash_extra_keys(
202
Returns:
203
A tuple of extra keys and the next multi-modal index.
204
"""
205
+ extra_keys: List[Any] = []
206
207
mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
208
if not mm_positions:
194
- return None, start_mm_idx
209
+ return extra_keys, start_mm_idx
210
211
if mm_positions and len(mm_positions) != len(mm_hashes):
212
raise ValueError(
@@ -204,14 +219,13 @@ def generate_block_hash_extra_keys(
219
# range. This usually happens in the late prefill phase and decoding phase.
220
if mm_positions[-1]["offset"] + mm_positions[-1][
221
"length"] < start_token_idx:
207
- return None, start_mm_idx
222
+ return extra_keys, start_mm_idx
223
224
# Support start_mm_idx == -1 to indicate the last mm input.
225
if start_mm_idx < 0:
226
assert -start_mm_idx <= len(mm_positions)
227
start_mm_idx = len(mm_positions) + start_mm_idx
228
214
- extra_keys = []
229
curr_mm_idx = start_mm_idx
230
while mm_positions and curr_mm_idx < len(mm_positions):
231
assert mm_hashes[curr_mm_idx] is not None
@@ -237,7 +251,50 @@ def generate_block_hash_extra_keys(
251
else:
252
# This block has not reached the current mm input.
253
break
240
- return tuple(extra_keys), curr_mm_idx
254
+ return extra_keys, curr_mm_idx
255
+
256
+
257
+def _gen_lora_extra_hash_keys(request: Request) -> List[int]:
258
+ """Generate extra keys related to LoRA for block hash computation.
259
+
260
+ Args:
261
+ request: The request object.
262
+
263
+ Returns:
264
+ Return LoRA id of the request if it is a LoRA request. Return empty
265
+ list otherwise.
266
+ """
267
+ if not request.lora_request:
268
+ return []
269
+ return [request.lora_request.lora_int_id]
270
+
271
+
272
+def generate_block_hash_extra_keys(
273
+ request: Request, start_token_idx: int, end_token_idx: int,
274
+ start_mm_idx: int) -> Tuple[Optional[Tuple[Any, ...]], int]:
275
+ """Generate extra keys for the block hash. The extra keys can come from
276
+ the multi-modal inputs and request specific metadata (e.g., LoRA ID).
277
+
278
+ Args:
279
+ request: The request object.
280
+ start_token_idx: The start token index of the block.
281
+ end_token_idx: The end token index of the block.
282
+ start_mm_idx: The start multi-modal index of the block.
283
+
284
+ Returns:
285
+ A tuple of extra keys and the next multi-modal index.
286
+ """
287
+ mm_extra_keys: List[Any]
288
+ mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
289
+ request, start_token_idx, end_token_idx, start_mm_idx)
290
+ lora_extra_keys: List[int] = _gen_lora_extra_hash_keys(request)
291
+
292
+ extra_keys: List[Any] = lora_extra_keys + mm_extra_keys
293
+
294
+ if not extra_keys:
295
+ return None, new_start_mm_idx
296
+
297
+ return tuple(extra_keys), new_start_mm_idx
298
299
300
def hash_block_tokens(
@@ -249,9 +306,6 @@ def hash_block_tokens(
306
prefix caching. We use LRU cache for this function to avoid recomputing
307
hash values for the same block contents.
308
252
- TODO: Support arbitrary metadata so that we could support more
253
- features such as LoRA adapter.
254
-
309
Args:
310
parent_block_hash: The hash of the parent block. None
311
if this is the first block.
@@ -291,14 +345,9 @@ def hash_request_tokens(block_size: int,
345
The list of computed hash values.
346
"""
347
token_ids = request.all_token_ids
294
- mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
295
- if mm_positions and len(mm_positions) != len(mm_hashes):
296
- raise ValueError(
297
- "The number of multi-modal positions and hashes must match.")
348
299
- # TODO: Extend this to support other features such as LoRA.
300
- need_extra_keys = bool(mm_positions)
301
- extra_keys = None
349
+ req_need_extra_keys = need_extra_keys(request)
350
+ req_extra_keys = None
351
curr_mm_idx = 0
352
353
ret = []
@@ -310,13 +359,13 @@ def hash_request_tokens(block_size: int,
359
if len(block_token_ids) < block_size:
360
break
361
313
- # Add extra keys if the block is a multi-modal block.
314
- if need_extra_keys:
315
- extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
362
+ if req_need_extra_keys:
363
+ # MM and LoRA requests need extra keys for block-hash computation.
364
+ req_extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
365
request, start, end, curr_mm_idx)
366
367
block_hash = hash_block_tokens(parent_block_hash_value,
319
- block_token_ids, extra_keys)
368
+ block_token_ids, req_extra_keys)
369
ret.append(block_hash)
370
parent_block_hash_value = block_hash.hash_value
371
return ret
vllm/v1/core/scheduler.py
+30
-2
@@ -7,6 +7,7 @@ from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
7
8
from vllm.config import CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig
9
from vllm.logger import init_logger
10
+from vllm.lora.request import LoRARequest
11
from vllm.sampling_params import SamplingParams
12
from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
13
compute_encoder_budget)
@@ -35,8 +36,6 @@ class Scheduler:
36
self.scheduler_config = scheduler_config
37
self.cache_config = cache_config
38
self.lora_config = lora_config
38
- # TODO: Support LoRA.
39
- assert lora_config is None, "V1 does not support LoRA yet."
39
40
# Scheduling constraints.
41
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
@@ -180,6 +179,14 @@ class Scheduler:
179
self.encoder_cache_manager.allocate(request, i)
180
encoder_budget = new_encoder_budget
181
182
+ # Record the LoRAs in scheduled_running_reqs
183
+ requested_loras: Set[int] = set()
184
+ if self.lora_config:
185
+ requested_loras = set(
186
+ req.lora_request.lora_int_id for req in scheduled_running_reqs
187
+ if req.lora_request and req.lora_request.lora_int_id > 0)
188
+ assert len(requested_loras) <= self.lora_config.max_loras
189
+
190
# Next, schedule the WAITING requests.
191
if not preempted_reqs:
192
while self.waiting and token_budget > 0:
@@ -187,6 +194,23 @@ class Scheduler:
194
break
195
196
request = self.waiting[0]
197
+
198
+ # Check that adding the request still respects the max_loras
199
+ # constraint.
200
+ if self.lora_config and request.lora_request:
201
+ req_lora_id = request.lora_request.lora_int_id
202
+ if len(requested_loras) == self.lora_config.max_loras and (
203
+ req_lora_id not in requested_loras):
204
+ # Cannot schedule.
205
+ # TODO (varun): This means all the other requests in
206
+ # the WAITING queue will be blocked by this request,
207
+ # even if,
208
+ # 1. these other requests do not use LoRA, or,
209
+ # 2. these other requests use the already requested
210
+ # LoRAs.
211
+ # This is too conservative and could be optimized.
212
+ break
213
+
214
# Get already-cached tokens.
215
computed_blocks, num_computed_tokens = \
216
self.kv_cache_manager.get_computed_blocks(request)
@@ -234,6 +258,8 @@ class Scheduler:
258
raise RuntimeError(
259
f"Invalid request status: {request.status}")
260
261
+ if self.lora_config and request.lora_request:
262
+ requested_loras.add(request.lora_request.lora_int_id)
263
req_to_new_block_ids[request.request_id] = [
264
b.block_id for b in computed_blocks + new_blocks
265
]
@@ -568,6 +594,7 @@ class NewRequestData:
594
sampling_params: SamplingParams
595
block_ids: List[int]
596
num_computed_tokens: int
597
+ lora_request: Optional[LoRARequest]
598
599
@classmethod
600
def from_request(
@@ -586,6 +613,7 @@ class NewRequestData:
613
sampling_params=request.sampling_params,
614
block_ids=block_ids,
615
num_computed_tokens=num_computed_tokens,
616
+ lora_request=request.lora_request,
617
)
618
619
vllm/v1/worker/gpu_input_batch.py
+62
-1
@@ -3,11 +3,12 @@
3
# Datastructures defining an input batch
4
5
from dataclasses import dataclass
6
-from typing import TYPE_CHECKING, Dict, List, Optional, Set
6
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
7
8
import numpy as np
9
import torch
10
11
+from vllm.lora.request import LoRARequest
12
from vllm.multimodal import MultiModalKwargs
13
from vllm.sampling_params import SamplingParams, SamplingType
14
from vllm.v1.sample.metadata import SamplingMetadata
@@ -35,6 +36,8 @@ class CachedRequestState:
36
mrope_positions: Optional[torch.Tensor] = None
37
mrope_position_delta: Optional[int] = None
38
39
+ lora_request: Optional[LoRARequest] = None
40
+
41
@property
42
def num_tokens(self) -> int:
43
return len(self.prompt_token_ids) + len(self.output_token_ids)
@@ -161,6 +164,12 @@ class InputBatch:
164
]
165
self.prompt_token_ids: Optional[torch.Tensor] = None
166
167
+ # lora related
168
+ self.request_lora_mapping = np.zeros((self.max_num_reqs, ),
169
+ dtype=np.int32)
170
+ self.lora_id_to_request_ids: Dict[int, Set[str]] = {}
171
+ self.lora_id_to_lora_request: Dict[int, LoRARequest] = {}
172
+
173
# req_index -> generator
174
# NOTE(woosuk): The indices of the requests that do not have their own
175
# generator should not be included in the dictionary.
@@ -235,6 +244,19 @@ class InputBatch:
244
if sampling_params.prompt_logprobs:
245
self.prompt_logprob_reqs.add(req_id)
246
247
+ # Add request lora ID
248
+ if request.lora_request:
249
+ lora_id = request.lora_request.lora_int_id
250
+ if lora_id not in self.lora_id_to_request_ids:
251
+ self.lora_id_to_request_ids[lora_id] = set()
252
+
253
+ self.request_lora_mapping[req_index] = lora_id
254
+ self.lora_id_to_request_ids[lora_id].add(request.req_id)
255
+ self.lora_id_to_lora_request[lora_id] = request.lora_request
256
+ else:
257
+ # No LoRA
258
+ self.request_lora_mapping[req_index] = 0
259
+
260
def remove_request(self, req_id: str) -> Optional[int]:
261
req_index = self.req_id_to_index.pop(req_id, None)
262
if req_index is None:
@@ -251,6 +273,16 @@ class InputBatch:
273
self.generators.pop(req_index, None)
274
self.num_logprobs.pop(req_id, None)
275
self.prompt_logprob_reqs.discard(req_id)
276
+
277
+ # LoRA
278
+ lora_id = self.request_lora_mapping[req_index]
279
+ if lora_id != 0:
280
+ self.lora_id_to_request_ids[lora_id].discard(req_id)
281
+ if len(self.lora_id_to_request_ids[lora_id]) == 0:
282
+ self.lora_id_to_request_ids.pop(lora_id)
283
+ self.lora_id_to_lora_request.pop(lora_id)
284
+ self.request_lora_mapping[req_index] = 0
285
+
286
return req_index
287
288
def clear(self) -> None:
@@ -266,6 +298,9 @@ class InputBatch:
298
self.generators.clear()
299
self.num_logprobs.clear()
300
self.prompt_logprob_reqs.clear()
301
+ self.request_lora_mapping.fill(0)
302
+ self.lora_id_to_lora_request.clear()
303
+ self.lora_id_to_request_ids.clear()
304
305
def condense(self, empty_req_indices: List[int]) -> None:
306
if self.num_reqs == 0:
@@ -318,6 +353,9 @@ class InputBatch:
353
if generator is not None:
354
self.generators[empty_index] = generator
355
356
+ self.request_lora_mapping[empty_index] = self.request_lora_mapping[
357
+ last_req_index]
358
+
359
# Decrement last_req_index since it is now empty.
360
last_req_index -= 1
361
@@ -401,6 +439,29 @@ class InputBatch:
439
return prompt_token_ids_cpu_tensor.to(device=self.device,
440
non_blocking=True)
441
442
+ def make_lora_inputs(
443
+ self, num_scheduled_tokens: np.ndarray
444
+ ) -> Tuple[Tuple[int, ...], Tuple[int, ...], Set[LoRARequest]]:
445
+ """
446
+ Given the num_scheduled_tokens for each request in the batch, return
447
+ datastructures used to activate the current LoRAs.
448
+ Returns:
449
+ 1. prompt_lora_mapping: A tuple of size self.num_reqs where,
450
+ prompt_lora_mapping[i] is the LoRA id to use for the ith prompt.
451
+ 2. token_lora_mapping: A tuple of size np.sum(num_scheduled_tokens)
452
+ where, token_lora_mapping[i] is the LoRA id to use for ith token.
453
+ 3. lora_requests: Set of relevant LoRA requests.
454
+ """
455
+
456
+ req_lora_mapping = self.request_lora_mapping[:self.num_reqs]
457
+ prompt_lora_mapping = tuple(req_lora_mapping)
458
+ token_lora_mapping = tuple(
459
+ req_lora_mapping.repeat(num_scheduled_tokens))
460
+ active_lora_requests: Set[LoRARequest] = set(
461
+ self.lora_id_to_lora_request.values())
462
+
463
+ return prompt_lora_mapping, token_lora_mapping, active_lora_requests
464
+
465
@property
466
def num_reqs(self) -> int:
467
return len(self.req_id_to_index)
vllm/v1/worker/gpu_model_runner.py
+44
-12
@@ -33,6 +33,7 @@ from vllm.v1.outputs import ModelRunnerOutput
33
from vllm.v1.sample.metadata import SamplingMetadata
34
from vllm.v1.utils import bind_kv_cache
35
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
36
+from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
37
38
if TYPE_CHECKING:
39
from vllm.v1.core.scheduler import SchedulerOutput
@@ -40,7 +41,7 @@ if TYPE_CHECKING:
41
logger = init_logger(__name__)
42
43
43
-class GPUModelRunner:
44
+class GPUModelRunner(LoRAModelRunnerMixin):
45
46
def __init__(
47
self,
@@ -279,6 +280,7 @@ class GPUModelRunner:
280
block_ids=new_req_data.block_ids,
281
num_computed_tokens=new_req_data.num_computed_tokens,
282
output_token_ids=[],
283
+ lora_request=new_req_data.lora_request,
284
)
285
286
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
@@ -372,15 +374,16 @@ class GPUModelRunner:
374
375
# Get the number of scheduled tokens for each request.
376
# TODO: The Python loop can be slow. Optimize.
375
- num_scheduled_tokens = []
377
+ num_scheduled_tokens_list: List[int] = []
378
max_num_scheduled_tokens = 0
379
for req_id in self.input_batch.req_ids[:num_reqs]:
380
assert req_id is not None
381
num_tokens = scheduler_output.num_scheduled_tokens[req_id]
380
- num_scheduled_tokens.append(num_tokens)
382
+ num_scheduled_tokens_list.append(num_tokens)
383
max_num_scheduled_tokens = max(max_num_scheduled_tokens,
384
num_tokens)
383
- num_scheduled_tokens = np.array(num_scheduled_tokens, dtype=np.int32)
385
+ num_scheduled_tokens: np.ndarray = np.array(num_scheduled_tokens_list,
386
+ dtype=np.int32)
387
assert max_num_scheduled_tokens > 0
388
389
# Get request indices.
@@ -565,6 +568,11 @@ class GPUModelRunner:
568
prefix_kv_lens=prefix_kv_lens,
569
suffix_kv_lens=suffix_kv_lens,
570
)
571
+
572
+ # Hot-Swap lora model
573
+ if self.lora_config:
574
+ self.set_active_loras(self.input_batch, num_scheduled_tokens)
575
+
576
# NOTE(woosuk): Due to chunked prefills, the batch may contain partial
577
# requests. While we should not sample any token from these partial
578
# requests, we do so for simplicity. We will ignore the sampled
@@ -867,6 +875,12 @@ class GPUModelRunner:
875
logger.info("Starting to load model %s...", self.model_config.model)
876
with DeviceMemoryProfiler() as m: # noqa: SIM117
877
self.model = get_model(vllm_config=self.vllm_config)
878
+ if self.lora_config:
879
+ self.model = self.load_lora_model(self.model,
880
+ self.model_config,
881
+ self.scheduler_config,
882
+ self.lora_config,
883
+ self.device)
884
885
self.model_memory_usage = m.consumed_memory
886
logger.info("Loading model weights took %.4f GB",
@@ -1005,14 +1019,32 @@ class GPUModelRunner:
1019
# Cache the dummy encoder outputs.
1020
self.encoder_cache["tmp"] = dict(enumerate(dummy_encoder_outputs))
1021
1008
- # Trigger compilation for general shape.
1009
- hidden_states = self._dummy_run(self.max_num_tokens, dummy_kv_caches)
1010
- logits = self.model.compute_logits(hidden_states, None)
1011
- logits = logits[:self.max_num_tokens]
1012
- # TODO(woosuk): Consider the memory usage of the sampler.
1013
- torch.cuda.synchronize()
1014
- del hidden_states, logits
1015
- self.encoder_cache.clear()
1022
+ # For profile, have maximum num_reqs and that collectively have
1023
+ # maximum num_tokens.
1024
+ num_reqs = self.scheduler_config.max_num_seqs
1025
+ num_tokens = self.max_num_tokens
1026
+ min_tokens_per_req: int = num_tokens // num_reqs
1027
+
1028
+ num_scheduled_tokens_list: List[int] = [min_tokens_per_req] * num_reqs
1029
+ num_scheduled_tokens_list[-1] += num_tokens % num_reqs
1030
+ assert sum(num_scheduled_tokens_list) == num_tokens
1031
+ assert len(num_scheduled_tokens_list) == num_reqs
1032
+
1033
+ num_scheduled_tokens: np.ndarray = np.array(num_scheduled_tokens_list,
1034
+ dtype=np.int32)
1035
+ logit_indices = np.cumsum(num_scheduled_tokens) - 1
1036
+
1037
+ with self.maybe_profile_with_lora(self.lora_config,
1038
+ num_scheduled_tokens):
1039
+ # Trigger compilation for general shape.
1040
+ hidden_states = self._dummy_run(self.max_num_tokens,
1041
+ dummy_kv_caches)
1042
+ hidden_states = hidden_states[logit_indices]
1043
+ logits = self.model.compute_logits(hidden_states, None)
1044
+ # TODO(woosuk): Consider the memory usage of the sampler.
1045
+ torch.cuda.synchronize()
1046
+ del hidden_states, logits
1047
+ self.encoder_cache.clear()
1048
gc.collect()
1049
1050
def capture_model(self) -> None:
vllm/v1/worker/lora_model_runner_mixin.py
new
+129
@@ -0,0 +1,129 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""
3
+Define LoRA functionality mixin for model runners.
4
+"""
5
+
6
+from contextlib import contextmanager
7
+from typing import Set, Tuple
8
+
9
+import numpy as np
10
+import torch.nn as nn
11
+
12
+from vllm.config import LoRAConfig, ModelConfig, SchedulerConfig
13
+from vllm.logger import init_logger
14
+from vllm.lora.layers import LoRAMapping
15
+from vllm.lora.request import LoRARequest
16
+from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
17
+from vllm.model_executor.models import supports_lora, supports_multimodal
18
+from vllm.v1.worker.gpu_input_batch import InputBatch
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+# Defined as a mixin for GPUModelRunner
24
+class LoRAModelRunnerMixin:
25
+
26
+ LORA_WARMUP_RANK = 8
27
+
28
+ def load_lora_model(self, model: nn.Module, model_config: ModelConfig,
29
+ scheduler_config: SchedulerConfig,
30
+ lora_config: LoRAConfig, device: str) -> nn.Module:
31
+
32
+ assert supports_lora(
33
+ model), f"{model.__class__.__name__} does not support LoRA yet."
34
+
35
+ if supports_multimodal(model):
36
+ logger.warning("Regarding multimodal models, vLLM currently "
37
+ "only supports adding LoRA to language model.")
38
+
39
+ # It's necessary to distinguish between the max_position_embeddings
40
+ # of VLMs and LLMs.
41
+ if hasattr(model.config, "max_position_embeddings"):
42
+ max_pos_embeddings = model.config.max_position_embeddings
43
+ else:
44
+ max_pos_embeddings = (
45
+ model.config.text_config.max_position_embeddings)
46
+
47
+ # Add LoRA Manager to the Model Runner
48
+ self.lora_manager = LRUCacheWorkerLoRAManager(
49
+ scheduler_config.max_num_seqs,
50
+ scheduler_config.max_num_batched_tokens,
51
+ model_config.get_vocab_size(),
52
+ lora_config,
53
+ device,
54
+ model.embedding_modules,
55
+ model.embedding_padding_modules,
56
+ max_position_embeddings=max_pos_embeddings,
57
+ )
58
+ return self.lora_manager.create_lora_manager(model)
59
+
60
+ def _set_active_loras(self, prompt_lora_mapping: Tuple[int, ...],
61
+ token_lora_mapping: Tuple[int, ...],
62
+ lora_requests: Set[LoRARequest]) -> None:
63
+ if not self.lora_manager:
64
+ raise RuntimeError("LoRA is not enabled.")
65
+
66
+ # We dont make any distinction between prefills and decodes in the
67
+ # scheduler. To that effect, set is_prefill to True so we use the
68
+ # sgmv punica kernels always.
69
+ lora_mapping = LoRAMapping(token_lora_mapping,
70
+ prompt_lora_mapping,
71
+ is_prefill=True)
72
+ self.lora_manager.set_active_adapters(lora_requests, lora_mapping)
73
+
74
+ def set_active_loras(self, input_batch: InputBatch,
75
+ num_scheduled_tokens: np.ndarray) -> None:
76
+
77
+ prompt_lora_mapping: Tuple[int, ...] # of size input_batch.num_reqs
78
+ token_lora_mapping: Tuple[int,
79
+ ...] # of size np.sum(num_scheduled_tokens)
80
+ lora_requests: Set[LoRARequest]
81
+ prompt_lora_mapping, token_lora_mapping, lora_requests = \
82
+ input_batch.make_lora_inputs(num_scheduled_tokens)
83
+ return self._set_active_loras(prompt_lora_mapping, token_lora_mapping,
84
+ lora_requests)
85
+
86
+ @contextmanager
87
+ def maybe_profile_with_lora(self, lora_config: LoRAConfig,
88
+ num_scheduled_tokens: np.ndarray):
89
+ if lora_config is None:
90
+ yield
91
+ else:
92
+ # __enter__ code
93
+ assert self.lora_manager is not None, "LoRA is not enabled"
94
+
95
+ num_reqs = len(num_scheduled_tokens)
96
+ num_loras = lora_config.max_loras
97
+
98
+ # Make prompt lora mapping
99
+ # Assign LoRA IDs cyclically to simulate a worst-case scenario.
100
+ prompt_lora_mapping = (np.arange(num_reqs, dtype=np.int32) %
101
+ num_loras) + 1
102
+
103
+ # Make token lora mapping
104
+ token_lora_mapping = np.repeat(prompt_lora_mapping,
105
+ num_scheduled_tokens)
106
+
107
+ # Make dummy lora requests
108
+ lora_requests: Set[LoRARequest] = {
109
+ LoRARequest(lora_name=f"warmup_{lora_id}",
110
+ lora_int_id=lora_id,
111
+ lora_path="/not/a/real/path")
112
+ for lora_id in range(1, num_loras + 1)
113
+ }
114
+
115
+ with self.lora_manager.dummy_lora_cache():
116
+ # Add the dummy LoRAs here so _set_active_loras doesn't try to
117
+ # load from disk.
118
+ for lr in lora_requests:
119
+ self.lora_manager.add_dummy_lora(
120
+ lr, rank=self.LORA_WARMUP_RANK)
121
+
122
+ self._set_active_loras(tuple(prompt_lora_mapping),
123
+ tuple(token_lora_mapping),
124
+ lora_requests)
125
+
126
+ yield
127
+
128
+ # __exit__ code
129
+ self.lora_manager.remove_all_adapters()