@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 467a96a541
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1] LoRA Support (#10957)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Feb 6, 2025 at 23:02 UTC 467a96a5415dc896170cecc0bb83d9c49c2f3c5e
16 files changed +453 -56
tests/lora/conftest.py
+17
@@ -306,3 +306,20 @@ def llama_2_7b_engine_extra_embeddings():
306 def llama_2_7b_model_extra_embeddings(llama_2_7b_engine_extra_embeddings):
307 yield (llama_2_7b_engine_extra_embeddings.model_executor.driver_worker.
308 model_runner.model)
309 +
310 +
311 +@pytest.fixture(params=[True, False])
312 +def run_with_both_engines_lora(request, monkeypatch):
313 + # Automatically runs tests twice, once with V1 and once without
314 + use_v1 = request.param
315 + # Tests decorated with `@skip_v1` are only run without v1
316 + skip_v1 = request.node.get_closest_marker("skip_v1")
317 +
318 + if use_v1:
319 + if skip_v1:
320 + pytest.skip("Skipping test on vllm V1")
321 + monkeypatch.setenv('VLLM_USE_V1', '1')
322 + else:
323 + monkeypatch.setenv('VLLM_USE_V1', '0')
324 +
325 + yield
tests/lora/test_baichuan.py
+8
@@ -42,6 +42,14 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
42 return generated_texts
43
44
45 +@pytest.fixture(autouse=True)
46 +def v1(run_with_both_engines_lora):
47 + # Simple autouse wrapper to run both engines for each test
48 + # This can be promoted up to conftest.py to run for every
49 + # test in a package
50 + pass
51 +
52 +
53 def test_baichuan_lora(baichuan_lora_files):
54 llm = vllm.LLM(MODEL_PATH,
55 max_model_len=1024,
tests/lora/test_chatglm3_tp.py
+13
@@ -2,6 +2,8 @@
2
3 from typing import List
4
5 +import pytest
6 +
7 import vllm
8 from tests.utils import fork_new_process_for_each_test
9 from vllm.lora.request import LoRARequest
@@ -47,6 +49,15 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
49 return generated_texts
50
51
52 +@pytest.fixture(autouse=True)
53 +def v1(run_with_both_engines_lora):
54 + # Simple autouse wrapper to run both engines for each test
55 + # This can be promoted up to conftest.py to run for every
56 + # test in a package
57 + pass
58 +
59 +
60 +@pytest.mark.skip_v1
61 @fork_new_process_for_each_test
62 def test_chatglm3_lora(chatglm3_lora_files):
63 llm = vllm.LLM(MODEL_PATH,
@@ -66,6 +77,7 @@ def test_chatglm3_lora(chatglm3_lora_files):
77 assert output2[i] == EXPECTED_LORA_OUTPUT[i]
78
79
80 +@pytest.mark.skip_v1
81 @multi_gpu_test(num_gpus=4)
82 @fork_new_process_for_each_test
83 def test_chatglm3_lora_tp4(chatglm3_lora_files):
@@ -87,6 +99,7 @@ def test_chatglm3_lora_tp4(chatglm3_lora_files):
99 assert output2[i] == EXPECTED_LORA_OUTPUT[i]
100
101
102 +@pytest.mark.skip_v1
103 @multi_gpu_test(num_gpus=4)
104 @fork_new_process_for_each_test
105 def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
tests/lora/test_gemma.py
+8
@@ -33,6 +33,14 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
33 return generated_texts
34
35
36 +@pytest.fixture(autouse=True)
37 +def v1(run_with_both_engines_lora):
38 + # Simple autouse wrapper to run both engines for each test
39 + # This can be promoted up to conftest.py to run for every
40 + # test in a package
41 + pass
42 +
43 +
44 @pytest.mark.xfail(current_platform.is_rocm(),
45 reason="There can be output mismatch on ROCm")
46 def test_gemma_lora(gemma_lora_files):
tests/lora/test_llama_tp.py
+12
@@ -2,6 +2,7 @@
2
3 from typing import List
4
5 +import pytest
6 import ray
7
8 import vllm
@@ -73,6 +74,14 @@ def generate_and_test(llm, sql_lora_files):
74 print("removing lora")
75
76
77 +@pytest.fixture(autouse=True)
78 +def v1(run_with_both_engines_lora):
79 + # Simple autouse wrapper to run both engines for each test
80 + # This can be promoted up to conftest.py to run for every
81 + # test in a package
82 + pass
83 +
84 +
85 @fork_new_process_for_each_test
86 def test_llama_lora(sql_lora_files):
87
@@ -85,6 +94,9 @@ def test_llama_lora(sql_lora_files):
94 generate_and_test(llm, sql_lora_files)
95
96
97 +# Skipping for v1 as v1 doesn't have a good way to expose the num_gpu_blocks
98 +# used by the engine yet.
99 +@pytest.mark.skip_v1
100 @fork_new_process_for_each_test
101 def test_llama_lora_warmup(sql_lora_files):
102 """Test that the LLM initialization works with a warmup LORA path and
tests/lora/test_lora_bias_e2e.py
+11
@@ -30,6 +30,17 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
30 return generated_texts
31
32
33 +@pytest.fixture(autouse=True)
34 +def v1(run_with_both_engines_lora):
35 + # Simple autouse wrapper to run both engines for each test
36 + # This can be promoted up to conftest.py to run for every
37 + # test in a package
38 + pass
39 +
40 +
41 +# Skipping for V1 for now as we are hitting,
42 +# "Head size 80 is not supported by FlashAttention." error.
43 +@pytest.mark.skip_v1
44 @pytest.mark.parametrize("lora_bias", [True])
45 @pytest.mark.parametrize("fully_sharded", [True, False])
46 def test_lora_bias(lora_bias_files: str, lora_bias: bool, fully_sharded: bool):
tests/lora/test_phi.py
+13
@@ -2,6 +2,8 @@
2
3 from typing import List
4
5 +import pytest
6 +
7 import vllm
8 from vllm.lora.request import LoRARequest
9
@@ -48,6 +50,17 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
50 return generated_texts
51
52
53 +@pytest.fixture(autouse=True)
54 +def v1(run_with_both_engines_lora):
55 + # Simple autouse wrapper to run both engines for each test
56 + # This can be promoted up to conftest.py to run for every
57 + # test in a package
58 + pass
59 +
60 +
61 +# Skipping for V1 for now as we are hitting,
62 +# "Head size 80 is not supported by FlashAttention." error.
63 +@pytest.mark.skip_v1
64 def test_phi2_lora(phi2_lora_files):
65 # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
66 # Otherwise, the lora-test will fail due to CUDA OOM.
tests/lora/test_quant_model.py
+8
@@ -70,6 +70,14 @@ def do_sample(llm: vllm.LLM,
70 return generated_texts
71
72
73 +@pytest.fixture(autouse=True)
74 +def v1(run_with_both_engines_lora):
75 + # Simple autouse wrapper to run both engines for each test
76 + # This can be promoted up to conftest.py to run for every
77 + # test in a package
78 + pass
79 +
80 +
81 @pytest.mark.parametrize("model", MODELS)
82 @pytest.mark.parametrize("tp_size", [1])
83 def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
tests/v1/core/test_kv_cache_utils.py
+1 -1
@@ -163,7 +163,7 @@ def test_generate_block_hash_extra_keys():
163
164 # Test with no overlap
165 extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 6, 10, 0)
166 - assert extra_keys == ()
166 + assert extra_keys is None
167 assert next_mm_idx == 1
168
169 # Test with multiple extra keys
vllm/lora/layers.py
+5 -3
@@ -16,8 +16,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
16 get_tensor_model_parallel_world_size,
17 split_tensor_along_last_dim,
18 tensor_model_parallel_all_gather,
19 - tensor_model_parallel_all_reduce,
20 - tensor_model_parallel_gather)
19 + tensor_model_parallel_all_reduce)
20 from vllm.distributed.utils import divide
21 # yapf: disable
22 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -1043,7 +1042,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1042 logits = lm_head.linear_method.apply(lm_head, hidden_states)
1043 if embedding_bias is not None:
1044 logits += embedding_bias
1046 - logits = tensor_model_parallel_gather(logits)
1045 +
1046 + # Gather logits for TP
1047 + logits = self.base_layer._gather_logits(logits)
1048 +
1049 if logits is None:
1050 return None
1051
vllm/model_executor/layers/logits_processor.py
+17 -11
@@ -51,7 +51,6 @@ class LogitsProcessor(nn.Module):
51 # Soft cap the logits. Used in Gemma 2.
52 self.soft_cap = soft_cap
53 # Whether to use gather or all-gather to gather the logits.
54 -
54 parallel_config = get_current_vllm_config().parallel_config
55 self.use_all_gather = current_platform.is_tpu() \
56 or envs.VLLM_USE_V1 \
@@ -88,6 +87,20 @@ class LogitsProcessor(nn.Module):
87
88 return logits
89
90 + def _gather_logits(self, logits: torch.Tensor) -> torch.Tensor:
91 + """gather/all-gather the logits tensor across model parallel group."""
92 + if self.use_all_gather:
93 + # Gather is not supported for some devices such as TPUs.
94 + # Use all-gather instead.
95 + # NOTE(woosuk): Here, the outputs of every device should not be None
96 + # because XLA requires strict SPMD among all devices. Every device
97 + # should execute the same operations after gathering the logits.
98 + logits = tensor_model_parallel_all_gather(logits)
99 + else:
100 + # None may be returned for rank > 0
101 + logits = tensor_model_parallel_gather(logits)
102 + return logits
103 +
104 def _get_logits(
105 self,
106 hidden_states: torch.Tensor,
@@ -99,16 +112,9 @@ class LogitsProcessor(nn.Module):
112 hidden_states,
113 bias=embedding_bias)
114
102 - if self.use_all_gather:
103 - # Gather is not supported for some devices such as TPUs.
104 - # Use all-gather instead.
105 - # NOTE(woosuk): Here, the outputs of every device should not be None
106 - # because XLA requires strict SPMD among all devices. Every device
107 - # should execute the same operations after gathering the logits.
108 - logits = tensor_model_parallel_all_gather(logits)
109 - else:
110 - # None may be returned for rank > 0
111 - logits = tensor_model_parallel_gather(logits)
115 + # Gather logits for TP
116 + logits = self._gather_logits(logits)
117 +
118 # Remove paddings in vocab (if any).
119 if logits is not None:
120 logits = logits[..., :self.org_vocab_size]
vllm/v1/core/kv_cache_utils.py
+75 -26
@@ -170,14 +170,28 @@ class FreeKVCacheBlockQueue:
170 return ret
171
172
173 -def generate_block_hash_extra_keys(
174 - request: Request, start_token_idx: int, end_token_idx: int,
175 - start_mm_idx: int) -> Tuple[Optional[Tuple[Any, ...]], int]:
176 - """Generate extra keys for the block hash. The extra keys can come from
177 - the multi-modal inputs and request specific metadata (e.g., LoRA ID).
178 - For multi-modal inputs, the extra keys are (mm_hash, start_offset) that
179 - indicate a mm input contained in the block and its starting offset in
180 - the block tokens.
173 +def need_extra_keys(request: Request) -> bool:
174 + """Check whether the blocks allocated to this request need extra hash keys.
175 +
176 + Args:
177 + request (Request): The request.
178 +
179 + Returns:
180 + bool: Whether blocks allocated to this request need extra hash keys.
181 + """
182 +
183 + # Multimodal requests need to include the MM hash.
184 + # LoRA requests need to include the LoRA ID.
185 + return bool(request.mm_positions) or (request.lora_request is not None)
186 +
187 +
188 +def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
189 + end_token_idx: int,
190 + start_mm_idx: int) -> Tuple[List[Any], int]:
191 + """Generate extra keys related to MultiModal request for block hash
192 + computation. For multi-modal inputs, the extra keys are
193 + (mm_hash, start_offset) that indicate a mm input contained in the
194 + block and its starting offset in the block tokens.
195
196 Args:
197 request: The request object.
@@ -188,10 +202,11 @@ def generate_block_hash_extra_keys(
202 Returns:
203 A tuple of extra keys and the next multi-modal index.
204 """
205 + extra_keys: List[Any] = []
206
207 mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
208 if not mm_positions:
194 - return None, start_mm_idx
209 + return extra_keys, start_mm_idx
210
211 if mm_positions and len(mm_positions) != len(mm_hashes):
212 raise ValueError(
@@ -204,14 +219,13 @@ def generate_block_hash_extra_keys(
219 # range. This usually happens in the late prefill phase and decoding phase.
220 if mm_positions[-1]["offset"] + mm_positions[-1][
221 "length"] < start_token_idx:
207 - return None, start_mm_idx
222 + return extra_keys, start_mm_idx
223
224 # Support start_mm_idx == -1 to indicate the last mm input.
225 if start_mm_idx < 0:
226 assert -start_mm_idx <= len(mm_positions)
227 start_mm_idx = len(mm_positions) + start_mm_idx
228
214 - extra_keys = []
229 curr_mm_idx = start_mm_idx
230 while mm_positions and curr_mm_idx < len(mm_positions):
231 assert mm_hashes[curr_mm_idx] is not None
@@ -237,7 +251,50 @@ def generate_block_hash_extra_keys(
251 else:
252 # This block has not reached the current mm input.
253 break
240 - return tuple(extra_keys), curr_mm_idx
254 + return extra_keys, curr_mm_idx
255 +
256 +
257 +def _gen_lora_extra_hash_keys(request: Request) -> List[int]:
258 + """Generate extra keys related to LoRA for block hash computation.
259 +
260 + Args:
261 + request: The request object.
262 +
263 + Returns:
264 + Return LoRA id of the request if it is a LoRA request. Return empty
265 + list otherwise.
266 + """
267 + if not request.lora_request:
268 + return []
269 + return [request.lora_request.lora_int_id]
270 +
271 +
272 +def generate_block_hash_extra_keys(
273 + request: Request, start_token_idx: int, end_token_idx: int,
274 + start_mm_idx: int) -> Tuple[Optional[Tuple[Any, ...]], int]:
275 + """Generate extra keys for the block hash. The extra keys can come from
276 + the multi-modal inputs and request specific metadata (e.g., LoRA ID).
277 +
278 + Args:
279 + request: The request object.
280 + start_token_idx: The start token index of the block.
281 + end_token_idx: The end token index of the block.
282 + start_mm_idx: The start multi-modal index of the block.
283 +
284 + Returns:
285 + A tuple of extra keys and the next multi-modal index.
286 + """
287 + mm_extra_keys: List[Any]
288 + mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
289 + request, start_token_idx, end_token_idx, start_mm_idx)
290 + lora_extra_keys: List[int] = _gen_lora_extra_hash_keys(request)
291 +
292 + extra_keys: List[Any] = lora_extra_keys + mm_extra_keys
293 +
294 + if not extra_keys:
295 + return None, new_start_mm_idx
296 +
297 + return tuple(extra_keys), new_start_mm_idx
298
299
300 def hash_block_tokens(
@@ -249,9 +306,6 @@ def hash_block_tokens(
306 prefix caching. We use LRU cache for this function to avoid recomputing
307 hash values for the same block contents.
308
252 - TODO: Support arbitrary metadata so that we could support more
253 - features such as LoRA adapter.
254 -
309 Args:
310 parent_block_hash: The hash of the parent block. None
311 if this is the first block.
@@ -291,14 +345,9 @@ def hash_request_tokens(block_size: int,
345 The list of computed hash values.
346 """
347 token_ids = request.all_token_ids
294 - mm_positions, mm_hashes = request.mm_positions, request.mm_hashes
295 - if mm_positions and len(mm_positions) != len(mm_hashes):
296 - raise ValueError(
297 - "The number of multi-modal positions and hashes must match.")
348
299 - # TODO: Extend this to support other features such as LoRA.
300 - need_extra_keys = bool(mm_positions)
301 - extra_keys = None
349 + req_need_extra_keys = need_extra_keys(request)
350 + req_extra_keys = None
351 curr_mm_idx = 0
352
353 ret = []
@@ -310,13 +359,13 @@ def hash_request_tokens(block_size: int,
359 if len(block_token_ids) < block_size:
360 break
361
313 - # Add extra keys if the block is a multi-modal block.
314 - if need_extra_keys:
315 - extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
362 + if req_need_extra_keys:
363 + # MM and LoRA requests need extra keys for block-hash computation.
364 + req_extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
365 request, start, end, curr_mm_idx)
366
367 block_hash = hash_block_tokens(parent_block_hash_value,
319 - block_token_ids, extra_keys)
368 + block_token_ids, req_extra_keys)
369 ret.append(block_hash)
370 parent_block_hash_value = block_hash.hash_value
371 return ret
vllm/v1/core/scheduler.py
+30 -2
@@ -7,6 +7,7 @@ from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
7
8 from vllm.config import CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig
9 from vllm.logger import init_logger
10 +from vllm.lora.request import LoRARequest
11 from vllm.sampling_params import SamplingParams
12 from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
13 compute_encoder_budget)
@@ -35,8 +36,6 @@ class Scheduler:
36 self.scheduler_config = scheduler_config
37 self.cache_config = cache_config
38 self.lora_config = lora_config
38 - # TODO: Support LoRA.
39 - assert lora_config is None, "V1 does not support LoRA yet."
39
40 # Scheduling constraints.
41 self.max_num_running_reqs = self.scheduler_config.max_num_seqs
@@ -180,6 +179,14 @@ class Scheduler:
179 self.encoder_cache_manager.allocate(request, i)
180 encoder_budget = new_encoder_budget
181
182 + # Record the LoRAs in scheduled_running_reqs
183 + requested_loras: Set[int] = set()
184 + if self.lora_config:
185 + requested_loras = set(
186 + req.lora_request.lora_int_id for req in scheduled_running_reqs
187 + if req.lora_request and req.lora_request.lora_int_id > 0)
188 + assert len(requested_loras) <= self.lora_config.max_loras
189 +
190 # Next, schedule the WAITING requests.
191 if not preempted_reqs:
192 while self.waiting and token_budget > 0:
@@ -187,6 +194,23 @@ class Scheduler:
194 break
195
196 request = self.waiting[0]
197 +
198 + # Check that adding the request still respects the max_loras
199 + # constraint.
200 + if self.lora_config and request.lora_request:
201 + req_lora_id = request.lora_request.lora_int_id
202 + if len(requested_loras) == self.lora_config.max_loras and (
203 + req_lora_id not in requested_loras):
204 + # Cannot schedule.
205 + # TODO (varun): This means all the other requests in
206 + # the WAITING queue will be blocked by this request,
207 + # even if,
208 + # 1. these other requests do not use LoRA, or,
209 + # 2. these other requests use the already requested
210 + # LoRAs.
211 + # This is too conservative and could be optimized.
212 + break
213 +
214 # Get already-cached tokens.
215 computed_blocks, num_computed_tokens = \
216 self.kv_cache_manager.get_computed_blocks(request)
@@ -234,6 +258,8 @@ class Scheduler:
258 raise RuntimeError(
259 f"Invalid request status: {request.status}")
260
261 + if self.lora_config and request.lora_request:
262 + requested_loras.add(request.lora_request.lora_int_id)
263 req_to_new_block_ids[request.request_id] = [
264 b.block_id for b in computed_blocks + new_blocks
265 ]
@@ -568,6 +594,7 @@ class NewRequestData:
594 sampling_params: SamplingParams
595 block_ids: List[int]
596 num_computed_tokens: int
597 + lora_request: Optional[LoRARequest]
598
599 @classmethod
600 def from_request(
@@ -586,6 +613,7 @@ class NewRequestData:
613 sampling_params=request.sampling_params,
614 block_ids=block_ids,
615 num_computed_tokens=num_computed_tokens,
616 + lora_request=request.lora_request,
617 )
618
619
vllm/v1/worker/gpu_input_batch.py
+62 -1
@@ -3,11 +3,12 @@
3 # Datastructures defining an input batch
4
5 from dataclasses import dataclass
6 -from typing import TYPE_CHECKING, Dict, List, Optional, Set
6 +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
7
8 import numpy as np
9 import torch
10
11 +from vllm.lora.request import LoRARequest
12 from vllm.multimodal import MultiModalKwargs
13 from vllm.sampling_params import SamplingParams, SamplingType
14 from vllm.v1.sample.metadata import SamplingMetadata
@@ -35,6 +36,8 @@ class CachedRequestState:
36 mrope_positions: Optional[torch.Tensor] = None
37 mrope_position_delta: Optional[int] = None
38
39 + lora_request: Optional[LoRARequest] = None
40 +
41 @property
42 def num_tokens(self) -> int:
43 return len(self.prompt_token_ids) + len(self.output_token_ids)
@@ -161,6 +164,12 @@ class InputBatch:
164 ]
165 self.prompt_token_ids: Optional[torch.Tensor] = None
166
167 + # lora related
168 + self.request_lora_mapping = np.zeros((self.max_num_reqs, ),
169 + dtype=np.int32)
170 + self.lora_id_to_request_ids: Dict[int, Set[str]] = {}
171 + self.lora_id_to_lora_request: Dict[int, LoRARequest] = {}
172 +
173 # req_index -> generator
174 # NOTE(woosuk): The indices of the requests that do not have their own
175 # generator should not be included in the dictionary.
@@ -235,6 +244,19 @@ class InputBatch:
244 if sampling_params.prompt_logprobs:
245 self.prompt_logprob_reqs.add(req_id)
246
247 + # Add request lora ID
248 + if request.lora_request:
249 + lora_id = request.lora_request.lora_int_id
250 + if lora_id not in self.lora_id_to_request_ids:
251 + self.lora_id_to_request_ids[lora_id] = set()
252 +
253 + self.request_lora_mapping[req_index] = lora_id
254 + self.lora_id_to_request_ids[lora_id].add(request.req_id)
255 + self.lora_id_to_lora_request[lora_id] = request.lora_request
256 + else:
257 + # No LoRA
258 + self.request_lora_mapping[req_index] = 0
259 +
260 def remove_request(self, req_id: str) -> Optional[int]:
261 req_index = self.req_id_to_index.pop(req_id, None)
262 if req_index is None:
@@ -251,6 +273,16 @@ class InputBatch:
273 self.generators.pop(req_index, None)
274 self.num_logprobs.pop(req_id, None)
275 self.prompt_logprob_reqs.discard(req_id)
276 +
277 + # LoRA
278 + lora_id = self.request_lora_mapping[req_index]
279 + if lora_id != 0:
280 + self.lora_id_to_request_ids[lora_id].discard(req_id)
281 + if len(self.lora_id_to_request_ids[lora_id]) == 0:
282 + self.lora_id_to_request_ids.pop(lora_id)
283 + self.lora_id_to_lora_request.pop(lora_id)
284 + self.request_lora_mapping[req_index] = 0
285 +
286 return req_index
287
288 def clear(self) -> None:
@@ -266,6 +298,9 @@ class InputBatch:
298 self.generators.clear()
299 self.num_logprobs.clear()
300 self.prompt_logprob_reqs.clear()
301 + self.request_lora_mapping.fill(0)
302 + self.lora_id_to_lora_request.clear()
303 + self.lora_id_to_request_ids.clear()
304
305 def condense(self, empty_req_indices: List[int]) -> None:
306 if self.num_reqs == 0:
@@ -318,6 +353,9 @@ class InputBatch:
353 if generator is not None:
354 self.generators[empty_index] = generator
355
356 + self.request_lora_mapping[empty_index] = self.request_lora_mapping[
357 + last_req_index]
358 +
359 # Decrement last_req_index since it is now empty.
360 last_req_index -= 1
361
@@ -401,6 +439,29 @@ class InputBatch:
439 return prompt_token_ids_cpu_tensor.to(device=self.device,
440 non_blocking=True)
441
442 + def make_lora_inputs(
443 + self, num_scheduled_tokens: np.ndarray
444 + ) -> Tuple[Tuple[int, ...], Tuple[int, ...], Set[LoRARequest]]:
445 + """
446 + Given the num_scheduled_tokens for each request in the batch, return
447 + datastructures used to activate the current LoRAs.
448 + Returns:
449 + 1. prompt_lora_mapping: A tuple of size self.num_reqs where,
450 + prompt_lora_mapping[i] is the LoRA id to use for the ith prompt.
451 + 2. token_lora_mapping: A tuple of size np.sum(num_scheduled_tokens)
452 + where, token_lora_mapping[i] is the LoRA id to use for ith token.
453 + 3. lora_requests: Set of relevant LoRA requests.
454 + """
455 +
456 + req_lora_mapping = self.request_lora_mapping[:self.num_reqs]
457 + prompt_lora_mapping = tuple(req_lora_mapping)
458 + token_lora_mapping = tuple(
459 + req_lora_mapping.repeat(num_scheduled_tokens))
460 + active_lora_requests: Set[LoRARequest] = set(
461 + self.lora_id_to_lora_request.values())
462 +
463 + return prompt_lora_mapping, token_lora_mapping, active_lora_requests
464 +
465 @property
466 def num_reqs(self) -> int:
467 return len(self.req_id_to_index)
vllm/v1/worker/gpu_model_runner.py
+44 -12
@@ -33,6 +33,7 @@ from vllm.v1.outputs import ModelRunnerOutput
33 from vllm.v1.sample.metadata import SamplingMetadata
34 from vllm.v1.utils import bind_kv_cache
35 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
36 +from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
37
38 if TYPE_CHECKING:
39 from vllm.v1.core.scheduler import SchedulerOutput
@@ -40,7 +41,7 @@ if TYPE_CHECKING:
41 logger = init_logger(__name__)
42
43
43 -class GPUModelRunner:
44 +class GPUModelRunner(LoRAModelRunnerMixin):
45
46 def __init__(
47 self,
@@ -279,6 +280,7 @@ class GPUModelRunner:
280 block_ids=new_req_data.block_ids,
281 num_computed_tokens=new_req_data.num_computed_tokens,
282 output_token_ids=[],
283 + lora_request=new_req_data.lora_request,
284 )
285
286 # Only relevant for models using M-RoPE (e.g, Qwen2-VL)
@@ -372,15 +374,16 @@ class GPUModelRunner:
374
375 # Get the number of scheduled tokens for each request.
376 # TODO: The Python loop can be slow. Optimize.
375 - num_scheduled_tokens = []
377 + num_scheduled_tokens_list: List[int] = []
378 max_num_scheduled_tokens = 0
379 for req_id in self.input_batch.req_ids[:num_reqs]:
380 assert req_id is not None
381 num_tokens = scheduler_output.num_scheduled_tokens[req_id]
380 - num_scheduled_tokens.append(num_tokens)
382 + num_scheduled_tokens_list.append(num_tokens)
383 max_num_scheduled_tokens = max(max_num_scheduled_tokens,
384 num_tokens)
383 - num_scheduled_tokens = np.array(num_scheduled_tokens, dtype=np.int32)
385 + num_scheduled_tokens: np.ndarray = np.array(num_scheduled_tokens_list,
386 + dtype=np.int32)
387 assert max_num_scheduled_tokens > 0
388
389 # Get request indices.
@@ -565,6 +568,11 @@ class GPUModelRunner:
568 prefix_kv_lens=prefix_kv_lens,
569 suffix_kv_lens=suffix_kv_lens,
570 )
571 +
572 + # Hot-Swap lora model
573 + if self.lora_config:
574 + self.set_active_loras(self.input_batch, num_scheduled_tokens)
575 +
576 # NOTE(woosuk): Due to chunked prefills, the batch may contain partial
577 # requests. While we should not sample any token from these partial
578 # requests, we do so for simplicity. We will ignore the sampled
@@ -867,6 +875,12 @@ class GPUModelRunner:
875 logger.info("Starting to load model %s...", self.model_config.model)
876 with DeviceMemoryProfiler() as m: # noqa: SIM117
877 self.model = get_model(vllm_config=self.vllm_config)
878 + if self.lora_config:
879 + self.model = self.load_lora_model(self.model,
880 + self.model_config,
881 + self.scheduler_config,
882 + self.lora_config,
883 + self.device)
884
885 self.model_memory_usage = m.consumed_memory
886 logger.info("Loading model weights took %.4f GB",
@@ -1005,14 +1019,32 @@ class GPUModelRunner:
1019 # Cache the dummy encoder outputs.
1020 self.encoder_cache["tmp"] = dict(enumerate(dummy_encoder_outputs))
1021
1008 - # Trigger compilation for general shape.
1009 - hidden_states = self._dummy_run(self.max_num_tokens, dummy_kv_caches)
1010 - logits = self.model.compute_logits(hidden_states, None)
1011 - logits = logits[:self.max_num_tokens]
1012 - # TODO(woosuk): Consider the memory usage of the sampler.
1013 - torch.cuda.synchronize()
1014 - del hidden_states, logits
1015 - self.encoder_cache.clear()
1022 + # For profile, have maximum num_reqs and that collectively have
1023 + # maximum num_tokens.
1024 + num_reqs = self.scheduler_config.max_num_seqs
1025 + num_tokens = self.max_num_tokens
1026 + min_tokens_per_req: int = num_tokens // num_reqs
1027 +
1028 + num_scheduled_tokens_list: List[int] = [min_tokens_per_req] * num_reqs
1029 + num_scheduled_tokens_list[-1] += num_tokens % num_reqs
1030 + assert sum(num_scheduled_tokens_list) == num_tokens
1031 + assert len(num_scheduled_tokens_list) == num_reqs
1032 +
1033 + num_scheduled_tokens: np.ndarray = np.array(num_scheduled_tokens_list,
1034 + dtype=np.int32)
1035 + logit_indices = np.cumsum(num_scheduled_tokens) - 1
1036 +
1037 + with self.maybe_profile_with_lora(self.lora_config,
1038 + num_scheduled_tokens):
1039 + # Trigger compilation for general shape.
1040 + hidden_states = self._dummy_run(self.max_num_tokens,
1041 + dummy_kv_caches)
1042 + hidden_states = hidden_states[logit_indices]
1043 + logits = self.model.compute_logits(hidden_states, None)
1044 + # TODO(woosuk): Consider the memory usage of the sampler.
1045 + torch.cuda.synchronize()
1046 + del hidden_states, logits
1047 + self.encoder_cache.clear()
1048 gc.collect()
1049
1050 def capture_model(self) -> None:
vllm/v1/worker/lora_model_runner_mixin.py new
+129
@@ -0,0 +1,129 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""
3 +Define LoRA functionality mixin for model runners.
4 +"""
5 +
6 +from contextlib import contextmanager
7 +from typing import Set, Tuple
8 +
9 +import numpy as np
10 +import torch.nn as nn
11 +
12 +from vllm.config import LoRAConfig, ModelConfig, SchedulerConfig
13 +from vllm.logger import init_logger
14 +from vllm.lora.layers import LoRAMapping
15 +from vllm.lora.request import LoRARequest
16 +from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
17 +from vllm.model_executor.models import supports_lora, supports_multimodal
18 +from vllm.v1.worker.gpu_input_batch import InputBatch
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +# Defined as a mixin for GPUModelRunner
24 +class LoRAModelRunnerMixin:
25 +
26 + LORA_WARMUP_RANK = 8
27 +
28 + def load_lora_model(self, model: nn.Module, model_config: ModelConfig,
29 + scheduler_config: SchedulerConfig,
30 + lora_config: LoRAConfig, device: str) -> nn.Module:
31 +
32 + assert supports_lora(
33 + model), f"{model.__class__.__name__} does not support LoRA yet."
34 +
35 + if supports_multimodal(model):
36 + logger.warning("Regarding multimodal models, vLLM currently "
37 + "only supports adding LoRA to language model.")
38 +
39 + # It's necessary to distinguish between the max_position_embeddings
40 + # of VLMs and LLMs.
41 + if hasattr(model.config, "max_position_embeddings"):
42 + max_pos_embeddings = model.config.max_position_embeddings
43 + else:
44 + max_pos_embeddings = (
45 + model.config.text_config.max_position_embeddings)
46 +
47 + # Add LoRA Manager to the Model Runner
48 + self.lora_manager = LRUCacheWorkerLoRAManager(
49 + scheduler_config.max_num_seqs,
50 + scheduler_config.max_num_batched_tokens,
51 + model_config.get_vocab_size(),
52 + lora_config,
53 + device,
54 + model.embedding_modules,
55 + model.embedding_padding_modules,
56 + max_position_embeddings=max_pos_embeddings,
57 + )
58 + return self.lora_manager.create_lora_manager(model)
59 +
60 + def _set_active_loras(self, prompt_lora_mapping: Tuple[int, ...],
61 + token_lora_mapping: Tuple[int, ...],
62 + lora_requests: Set[LoRARequest]) -> None:
63 + if not self.lora_manager:
64 + raise RuntimeError("LoRA is not enabled.")
65 +
66 + # We dont make any distinction between prefills and decodes in the
67 + # scheduler. To that effect, set is_prefill to True so we use the
68 + # sgmv punica kernels always.
69 + lora_mapping = LoRAMapping(token_lora_mapping,
70 + prompt_lora_mapping,
71 + is_prefill=True)
72 + self.lora_manager.set_active_adapters(lora_requests, lora_mapping)
73 +
74 + def set_active_loras(self, input_batch: InputBatch,
75 + num_scheduled_tokens: np.ndarray) -> None:
76 +
77 + prompt_lora_mapping: Tuple[int, ...] # of size input_batch.num_reqs
78 + token_lora_mapping: Tuple[int,
79 + ...] # of size np.sum(num_scheduled_tokens)
80 + lora_requests: Set[LoRARequest]
81 + prompt_lora_mapping, token_lora_mapping, lora_requests = \
82 + input_batch.make_lora_inputs(num_scheduled_tokens)
83 + return self._set_active_loras(prompt_lora_mapping, token_lora_mapping,
84 + lora_requests)
85 +
86 + @contextmanager
87 + def maybe_profile_with_lora(self, lora_config: LoRAConfig,
88 + num_scheduled_tokens: np.ndarray):
89 + if lora_config is None:
90 + yield
91 + else:
92 + # __enter__ code
93 + assert self.lora_manager is not None, "LoRA is not enabled"
94 +
95 + num_reqs = len(num_scheduled_tokens)
96 + num_loras = lora_config.max_loras
97 +
98 + # Make prompt lora mapping
99 + # Assign LoRA IDs cyclically to simulate a worst-case scenario.
100 + prompt_lora_mapping = (np.arange(num_reqs, dtype=np.int32) %
101 + num_loras) + 1
102 +
103 + # Make token lora mapping
104 + token_lora_mapping = np.repeat(prompt_lora_mapping,
105 + num_scheduled_tokens)
106 +
107 + # Make dummy lora requests
108 + lora_requests: Set[LoRARequest] = {
109 + LoRARequest(lora_name=f"warmup_{lora_id}",
110 + lora_int_id=lora_id,
111 + lora_path="/not/a/real/path")
112 + for lora_id in range(1, num_loras + 1)
113 + }
114 +
115 + with self.lora_manager.dummy_lora_cache():
116 + # Add the dummy LoRAs here so _set_active_loras doesn't try to
117 + # load from disk.
118 + for lr in lora_requests:
119 + self.lora_manager.add_dummy_lora(
120 + lr, rank=self.LORA_WARMUP_RANK)
121 +
122 + self._set_active_loras(tuple(prompt_lora_mapping),
123 + tuple(token_lora_mapping),
124 + lora_requests)
125 +
126 + yield
127 +
128 + # __exit__ code
129 + self.lora_manager.remove_all_adapters()