A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Mypy] Typing lora folder (#4337)
SangBin Cho committed
Apr 26, 2024 at 04:13 UTC
b5b4a398a794e3276729a525e58eaa92f5fc0212
7 files changed
+91
-70
.github/workflows/mypy.yaml
+3
-4
@@ -33,8 +33,6 @@ jobs:
33
- name: Mypy
34
run: |
35
mypy vllm/attention --config-file pyproject.toml
36
- # TODO(sang): Fix nested dir
37
- mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
36
mypy vllm/distributed --config-file pyproject.toml
37
mypy vllm/entrypoints --config-file pyproject.toml
38
mypy vllm/executor --config-file pyproject.toml
@@ -44,8 +42,9 @@ jobs:
42
mypy vllm/engine --config-file pyproject.toml
43
mypy vllm/worker --config-file pyproject.toml
44
mypy vllm/spec_decode --config-file pyproject.toml
45
+ mypy vllm/lora --config-file pyproject.toml
46
+
47
# TODO(sang): Fix nested dir
48
mypy vllm/model_executor/*.py --config-file pyproject.toml
49
- # TODO(sang): Fix nested dir
50
- # mypy vllm/lora/*.py --config-file pyproject.toml
49
+ mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
50
format.sh
+1
-1
@@ -106,7 +106,7 @@ mypy vllm/engine --config-file pyproject.toml
106
mypy vllm/worker --config-file pyproject.toml
107
mypy vllm/spec_decode --config-file pyproject.toml
108
mypy vllm/model_executor/*.py --config-file pyproject.toml
109
-# mypy vllm/lora/*.py --config-file pyproject.toml
109
+mypy vllm/lora --config-file pyproject.toml
110
111
112
CODESPELL_EXCLUDES=(
vllm/lora/layers.py
+22
-13
@@ -176,6 +176,8 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
176
def __init__(self, base_layer: VocabParallelEmbedding) -> None:
177
super().__init__()
178
self.base_layer = base_layer
179
+ self.embeddings_slice: Optional[Tuple[int, int]]
180
+ self.embeddings_weights: Optional[torch.Tensor]
181
182
def create_lora_weights(
183
self,
@@ -233,9 +235,10 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
235
self.lora_a_stacked.shape[0] * self.lora_a_stacked.shape[1],
236
self.lora_a_stacked.shape[2],
237
)
236
- self.indices: Optional[torch.Tensor] = None
237
- self.indices_len: Optional[List[int]] = None
238
- self.embeddings_indices = None
238
+ # Lazily initialized.
239
+ self.indices: torch.Tensor
240
+ self.indices_len: List[int]
241
+ self.embeddings_indices: torch.Tensor
242
243
def reset_lora(self, index: int):
244
self.lora_a_stacked[index] = 0
@@ -267,6 +270,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
270
self.embeddings_tensors.shape[1],
271
self.embeddings_tensors.shape[2]
272
)[self.embeddings_slice[0]:self.embeddings_slice[1]]
273
+ assert self.embeddings_weights is not None
274
self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
275
276
def set_mapping(
@@ -343,11 +347,12 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
347
dtype=lora_config.lora_dtype,
348
device=self.device,
349
)
346
-
347
- self.indices: Optional[torch.Tensor] = None
348
- self.indices_len: Optional[List[int]] = None
350
self.output_dim = self.lora_b_stacked.shape[2]
351
352
+ # lazily initialized.
353
+ self.indices: torch.Tensor
354
+ self.indices_len: List[int]
355
+
356
def reset_lora(self, index: int):
357
self.lora_a_stacked[index] = 0
358
self.lora_b_stacked[index] = 0
@@ -475,8 +480,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
480
device=self.device,
481
) for _ in range(n_slices))
482
478
- self.indices: Optional[torch.Tensor] = None
483
self.output_dim = self.lora_b_stacked[0].shape[2]
484
+ # Lazily initialized.
485
+ self.indices: torch.Tensor
486
487
def reset_lora(self, index: int):
488
self.lora_a_stacked[0][index] = 0
@@ -690,7 +696,8 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
696
self.kv_proj_shard_size)
697
self.packed_indices: Optional[torch.Tensor] = None
698
self.standard_indices: Optional[torch.Tensor] = None
693
- self.indices_len: Optional[List[int]] = None
699
+ # lazily initialized.
700
+ self.indices_len: List[int]
701
702
def reset_lora(self, index: int):
703
self.lora_a_stacked[0][index] = 0
@@ -814,8 +821,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
821
dtype=lora_config.lora_dtype,
822
device=self.device,
823
)
817
- self.indices: Optional[torch.Tensor] = None
818
- self.indices_len: Optional[List[int]] = None
824
+ # Lazily initialized
825
+ self.indices: torch.Tensor
826
+ self.indices_len: List[int]
827
828
def reset_lora(self, index: int):
829
self.lora_a_stacked[index] = 0
@@ -991,9 +999,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
999
dtype=self.dtype,
1000
device=self.device,
1001
)
994
- self.indices = None
995
- self.indices_padded = None
996
- self.indices_len = None
1002
+ # Lazily initialized.
1003
+ self.indices: torch.Tensor
1004
+ self.indices_len: List[int]
1005
+ self.indices_padded: torch.Tensor
1006
1007
def reset_lora(self, index: int):
1008
self.lora_a_stacked[index] = 0
vllm/lora/lora.py
+17
-11
@@ -97,9 +97,9 @@ class PackedLoRALayerWeights(LoRALayerWeights):
97
self,
98
module_name: str,
99
rank: int,
100
- lora_alphas: List[int],
101
- lora_a: List[torch.Tensor],
102
- lora_b: List[torch.Tensor],
100
+ lora_alphas: List[Optional[int]],
101
+ lora_a: List[Optional[torch.Tensor]],
102
+ lora_b: List[Optional[torch.Tensor]],
103
scaling: Optional[List[float]] = None,
104
) -> None:
105
super().__init__(
@@ -108,17 +108,20 @@ class PackedLoRALayerWeights(LoRALayerWeights):
108
lora_alpha=0,
109
lora_a=lora_a,
110
lora_b=lora_b,
111
- scaling=scaling,
111
+ scaling=scaling, # type: ignore
112
embeddings_tensor=None,
113
)
114
self.lora_alphas = lora_alphas
115
if scaling is None:
116
- self.scaling = [
117
- lora_alpha / self.rank for lora_alpha in self.lora_alphas
116
+ self.scaling = [ # type: ignore
117
+ lora_alpha / self.rank # type: ignore # noqa
118
+ for lora_alpha in self.lora_alphas
119
]
120
121
@classmethod
121
- def pack(cls, loras: List["LoRALayerWeights"]) -> "PackedLoRALayerWeights":
122
+ def pack(
123
+ cls, loras: List[Optional["LoRALayerWeights"]]
124
+ ) -> "PackedLoRALayerWeights":
125
"""Pack a list of LoRAs into a single LoRA.
126
127
If LoRA is None, it signifies that the submodule does not have a LoRA.
@@ -136,16 +139,19 @@ class PackedLoRALayerWeights(LoRALayerWeights):
139
[lora.lora_alpha if lora is not None else None for lora in loras],
140
[lora.lora_a if lora is not None else None for lora in loras],
141
[lora.lora_b if lora is not None else None for lora in loras],
139
- scaling=[1 if lora is not None else None for lora in loras])
142
+ scaling=[
143
+ 1 if lora is not None else None # type: ignore
144
+ for lora in loras
145
+ ])
146
return obj
147
148
def optimize(self) -> "PackedLoRALayerWeights":
149
"""Optimize the LoRA by merging the scaling into lora_b."""
150
for i in range(len(self.lora_b)):
145
- if self.scaling[i] == 1 or self.lora_b[i] is None:
151
+ if self.scaling[i] == 1 or self.lora_b[i] is None: # type: ignore
152
continue
147
- self.lora_b[i] *= self.scaling[i]
148
- self.scaling[i] = 1
153
+ self.lora_b[i] *= self.scaling[i] # type: ignore
154
+ self.scaling[i] = 1 # type: ignore
155
return self
156
157
@property
vllm/lora/models.py
+34
-30
@@ -3,7 +3,7 @@ import json
3
import math
4
import os
5
import re
6
-from typing import Callable, Dict, Hashable, List, Optional, Tuple, Type
6
+from typing import Callable, Dict, List, Optional, Tuple, Type
7
8
import safetensors.torch
9
import torch
@@ -53,44 +53,46 @@ def convert_mapping(
53
embeddings.
54
indices_len: List of lengths of the above tensors.
55
"""
56
- indices = list(mapping.index_mapping).copy()
57
- embedding_indices = indices.copy()
58
- lora_indices = indices.copy()
59
- prompt_mapping = [
56
+ index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
57
+ embedding_indices = index_mapping_indices.copy()
58
+ lora_indices = index_mapping_indices.copy()
59
+ prompt_mapping: List[int] = [
60
lora_index_to_id.index(x) if x > 0 else -1
61
for x in mapping.prompt_mapping
62
]
63
lora_idx = None
64
- for i in range(len(indices)):
64
+ for i in range(len(index_mapping_indices)):
65
# TODO index can be slow. optimize
66
- lora_idx = (lora_index_to_id.index(indices[i])
67
- if indices[i] > 0 else -1)
68
- embedding_indices[i] = lora_idx if indices[i] > 0 else 0
69
- indices[i] = i
66
+ lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
67
+ if index_mapping_indices[i] > 0 else -1)
68
+ embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
69
+ index_mapping_indices[i] = i
70
lora_indices[i] = lora_idx
71
72
- indices = torch.tensor([indices, lora_indices, embedding_indices],
73
- dtype=torch.long,
74
- device="cuda")
75
- prompt_mapping = torch.tensor(prompt_mapping,
76
- device="cuda",
77
- dtype=torch.long)
72
+ indices = torch.tensor(
73
+ [index_mapping_indices, lora_indices, embedding_indices],
74
+ dtype=torch.long,
75
+ device="cuda")
76
+ prompt_mapping_tensor = torch.tensor(prompt_mapping,
77
+ device="cuda",
78
+ dtype=torch.long)
79
embeddings_indices = torch.stack([
80
indices[2] * extra_vocab_size,
81
indices[2] * (vocab_size + extra_vocab_size)
82
])
83
embeddings_indices[embeddings_indices == -1] = max_loras - 1
84
base_indices = indices[1]
84
- sampler_indices = prompt_mapping
85
+ sampler_indices = prompt_mapping_tensor
86
sampler_indices_padded = sampler_indices.clone()
87
sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
88
sampler_indices_padded = (
89
torch.arange(
90
0, len(sampler_indices_padded), device="cuda", dtype=torch.long) +
91
(sampler_indices_padded * len(sampler_indices_padded)))
91
- indices_len = (base_indices.shape[-1], sampler_indices.shape[-1],
92
- sampler_indices_padded.shape[-1],
93
- embeddings_indices.shape[-1])
92
+ indices_len = [
93
+ base_indices.shape[-1], sampler_indices.shape[-1],
94
+ sampler_indices_padded.shape[-1], embeddings_indices.shape[-1]
95
+ ]
96
97
return (base_indices, sampler_indices, sampler_indices_padded,
98
embeddings_indices, indices_len)
@@ -149,6 +151,7 @@ class LoRAModel:
151
if module_name not in loras:
152
lora_embeddings_tensor = None
153
if embeddings:
154
+ assert embedding_modules is not None
155
embeddings_module = next(
156
(k for k in embedding_modules if k in module_name),
157
None)
@@ -171,6 +174,7 @@ class LoRAModel:
174
else:
175
loras[module_name].lora_b = tensor.to(device=device,
176
dtype=dtype).t()
177
+ assert embedding_padding_modules is not None
178
if any(name in module_name
179
for name in embedding_padding_modules
180
) and target_embedding_padding is not None:
@@ -295,11 +299,10 @@ class LoRAModelManager:
299
self.max_num_batched_tokens,
300
dtype=torch.long,
301
device="cuda")
298
- self.offsets = []
302
# 4 is the number of indicies tensors defined above
303
# base_indices, sampler_indices, sampler_indices_padded,
304
# embeddings_indices
302
- self.indices_len = [None] * 4
305
+ self.indices_len: List[Optional[int]] = [None] * 4
306
307
self.model: nn.Module = model
308
if hasattr(self.model, "supported_lora_modules"):
@@ -312,7 +315,7 @@ class LoRAModelManager:
315
self._registered_loras: Dict[int, LoRAModel] = {}
316
# Dict instead of a Set for compatibility with LRUCache.
317
self._active_loras: Dict[int, None] = {}
315
- self._last_mapping = None
318
+ self._last_mapping: Optional[LoRAMapping] = None
319
self._create_lora_modules()
320
self.model.lora_manager = self
321
@@ -370,7 +373,7 @@ class LoRAModelManager:
373
return True
374
return False
375
373
- def _add_lora(self, lora: LoRAModel) -> bool:
376
+ def _add_lora(self, lora: LoRAModel):
377
self._create_merged_loras_inplace(lora)
378
self._registered_loras[lora.id] = lora
379
@@ -418,7 +421,7 @@ class LoRAModelManager:
421
def get_lora(self, lora_id: int) -> Optional[LoRAModel]:
422
return self._registered_loras.get(lora_id, None)
423
421
- def remove_all_loras(self) -> bool:
424
+ def remove_all_loras(self):
425
"""Remove all LoRAModels from the manager."""
426
self._registered_loras.clear()
427
self.lora_index_to_id = [None] * self.lora_slots
@@ -467,6 +470,7 @@ class LoRAModelManager:
470
continue
471
parts = module_name.split(".")
472
if module_name not in self.packed_modules:
473
+ assert embedding_modules is not None
474
if parts[-1] in embedding_modules:
475
input_dim = (module.base_layer.org_vocab_size +
476
self.lora_config.lora_extra_vocab_size if
@@ -500,7 +504,7 @@ class LoRAModelManager:
504
else:
505
parts = module_name.split(".")
506
replacements = self.packed_modules_mapping[parts[-1]]
503
- subloras = []
507
+ subloras: List[Optional["LoRALayerWeights"]] = []
508
for i, r in enumerate(replacements):
509
lora = LoRALayerWeights.create_dummy_lora_weights(
510
module_name + "." + r,
@@ -538,7 +542,7 @@ class LoRAModelManager:
542
543
def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
544
for module_name, new_module_names in self.packed_modules.items():
541
- replacement_loras = []
545
+ replacement_loras: List[Optional[LoRALayerWeights]] = []
546
has_replacement = False
547
for r in new_module_names:
548
lora = lora_model.get_lora(r)
@@ -557,12 +561,12 @@ class LoRAModelManager:
561
562
class LoRALRUCache(LRUCache[LoRAModel]):
563
560
- def __init__(self, capacity: int, deactivate_lora_fn: Callable[[Hashable],
561
- None]):
564
+ def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int],
565
+ bool]):
566
super().__init__(capacity)
567
self.deactivate_lora_fn = deactivate_lora_fn
568
565
- def _on_remove(self, key: Hashable, value: LoRAModel):
569
+ def _on_remove(self, key: int, value: LoRAModel):
570
logger.debug(f"Removing LoRA. int id: {key}")
571
self.deactivate_lora_fn(key)
572
return super()._on_remove(key, value)
vllm/lora/worker_manager.py
+12
-9
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod, abstractproperty
2
-from typing import Any, Dict, List, Optional, Set, Type
2
+from typing import Any, Dict, List, Set, Type
3
4
import torch
5
@@ -37,7 +37,7 @@ class AbstractWorkerLoRAManager(ABC):
37
...
38
39
@abstractmethod
40
- def set_active_loras(self, lora_requests: List[LoRARequest],
40
+ def set_active_loras(self, lora_requests: Set[LoRARequest],
41
lora_mapping: LoRAMapping) -> None:
42
...
43
@@ -54,7 +54,7 @@ class AbstractWorkerLoRAManager(ABC):
54
...
55
56
@abstractmethod
57
- def remove_all_loras(self) -> bool:
57
+ def remove_all_loras(self):
58
...
59
60
@abstractmethod
@@ -81,10 +81,11 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
81
embedding_padding_modules: List[str],
82
lora_model_cls: Type[LoRAModel] = LoRAModel,
83
):
84
- self._lora_manager: Optional[LoRAModelManager] = None
84
self._lora_model_cls = lora_model_cls
85
self.embedding_modules = embedding_modules
86
self.embedding_padding_modules = embedding_padding_modules
87
+ # Lazily initialized by create_lora_manager.
88
+ self._lora_manager: LoRAModelManager
89
super().__init__(max_num_seqs, max_num_batched_tokens, vocab_size,
90
lora_config, device)
91
@@ -104,7 +105,7 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
105
lora_config=self.lora_config,
106
lora_manager_cls=self._lora_manager_cls,
107
)
107
- self._lora_manager: LoRAModelManager = lora_manager
108
+ self._lora_manager = lora_manager
109
return lora_manager.model
110
111
def set_active_loras(self, lora_requests: Set[LoRARequest],
@@ -188,7 +189,7 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
189
def remove_lora(self, lora_id: int) -> bool:
190
return self._lora_manager.remove_lora(lora_id)
191
191
- def remove_all_loras(self) -> bool:
192
+ def remove_all_loras(self):
193
self._lora_manager.remove_all_loras()
194
195
def list_loras(self) -> Set[int]:
@@ -217,10 +218,10 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
218
lora_config=self.lora_config,
219
max_num_batched_tokens=self.max_num_batched_tokens,
220
)
220
- self._lora_manager: LRUCacheLoRAModelManager = lora_manager
221
+ self._lora_manager = lora_manager
222
return lora_manager.model
223
223
- def _apply_loras(self, lora_requests: List[LoRARequest]) -> None:
224
+ def _apply_loras(self, lora_requests: Set[LoRARequest]) -> None:
225
loras_map = {
226
lora_request.lora_int_id: lora_request
227
for lora_request in lora_requests if lora_request
@@ -237,12 +238,14 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
238
if lora_request.lora_int_id not in self.list_loras():
239
# Remove before we load the new lora to save memory
240
if len(self._lora_manager) + 1 > self._lora_manager.capacity:
241
+ assert isinstance(self._lora_manager, LRUCacheLoRAModelManager)
242
self._lora_manager.remove_oldest_lora()
243
lora = self._load_lora(lora_request)
244
loaded = self._lora_manager.add_lora(lora)
245
else:
246
# If the lora is already loaded, just touch it to
247
# update its position in the caches
246
- loaded = self._lora_manager.get_lora(lora_request.lora_int_id)
248
+ loaded = self._lora_manager.get_lora(
249
+ lora_request.lora_int_id) is not None
250
self._lora_manager.activate_lora(lora_request.lora_int_id)
251
return loaded
vllm/worker/model_runner.py
+2
-2
@@ -928,10 +928,10 @@ class ModelRunner:
928
torch.cuda.synchronize()
929
return
930
931
- def remove_all_loras(self) -> bool:
931
+ def remove_all_loras(self):
932
if not self.lora_manager:
933
raise RuntimeError("LoRA is not enabled.")
934
- return self.lora_manager.remove_all_loras()
934
+ self.lora_manager.remove_all_loras()
935
936
def set_active_loras(self, lora_requests: Set[LoRARequest],
937
lora_mapping: LoRAMapping) -> None: