@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b5b4a398a7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Mypy] Typing lora folder (#4337)

SangBin Cho committed Apr 26, 2024 at 04:13 UTC b5b4a398a794e3276729a525e58eaa92f5fc0212
7 files changed +91 -70
.github/workflows/mypy.yaml
+3 -4
@@ -33,8 +33,6 @@ jobs:
33 - name: Mypy
34 run: |
35 mypy vllm/attention --config-file pyproject.toml
36 - # TODO(sang): Fix nested dir
37 - mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
36 mypy vllm/distributed --config-file pyproject.toml
37 mypy vllm/entrypoints --config-file pyproject.toml
38 mypy vllm/executor --config-file pyproject.toml
@@ -44,8 +42,9 @@ jobs:
42 mypy vllm/engine --config-file pyproject.toml
43 mypy vllm/worker --config-file pyproject.toml
44 mypy vllm/spec_decode --config-file pyproject.toml
45 + mypy vllm/lora --config-file pyproject.toml
46 +
47 # TODO(sang): Fix nested dir
48 mypy vllm/model_executor/*.py --config-file pyproject.toml
49 - # TODO(sang): Fix nested dir
50 - # mypy vllm/lora/*.py --config-file pyproject.toml
49 + mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
50
format.sh
+1 -1
@@ -106,7 +106,7 @@ mypy vllm/engine --config-file pyproject.toml
106 mypy vllm/worker --config-file pyproject.toml
107 mypy vllm/spec_decode --config-file pyproject.toml
108 mypy vllm/model_executor/*.py --config-file pyproject.toml
109 -# mypy vllm/lora/*.py --config-file pyproject.toml
109 +mypy vllm/lora --config-file pyproject.toml
110
111
112 CODESPELL_EXCLUDES=(
vllm/lora/layers.py
+22 -13
@@ -176,6 +176,8 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
176 def __init__(self, base_layer: VocabParallelEmbedding) -> None:
177 super().__init__()
178 self.base_layer = base_layer
179 + self.embeddings_slice: Optional[Tuple[int, int]]
180 + self.embeddings_weights: Optional[torch.Tensor]
181
182 def create_lora_weights(
183 self,
@@ -233,9 +235,10 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
235 self.lora_a_stacked.shape[0] * self.lora_a_stacked.shape[1],
236 self.lora_a_stacked.shape[2],
237 )
236 - self.indices: Optional[torch.Tensor] = None
237 - self.indices_len: Optional[List[int]] = None
238 - self.embeddings_indices = None
238 + # Lazily initialized.
239 + self.indices: torch.Tensor
240 + self.indices_len: List[int]
241 + self.embeddings_indices: torch.Tensor
242
243 def reset_lora(self, index: int):
244 self.lora_a_stacked[index] = 0
@@ -267,6 +270,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
270 self.embeddings_tensors.shape[1],
271 self.embeddings_tensors.shape[2]
272 )[self.embeddings_slice[0]:self.embeddings_slice[1]]
273 + assert self.embeddings_weights is not None
274 self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
275
276 def set_mapping(
@@ -343,11 +347,12 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
347 dtype=lora_config.lora_dtype,
348 device=self.device,
349 )
346 -
347 - self.indices: Optional[torch.Tensor] = None
348 - self.indices_len: Optional[List[int]] = None
350 self.output_dim = self.lora_b_stacked.shape[2]
351
352 + # lazily initialized.
353 + self.indices: torch.Tensor
354 + self.indices_len: List[int]
355 +
356 def reset_lora(self, index: int):
357 self.lora_a_stacked[index] = 0
358 self.lora_b_stacked[index] = 0
@@ -475,8 +480,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
480 device=self.device,
481 ) for _ in range(n_slices))
482
478 - self.indices: Optional[torch.Tensor] = None
483 self.output_dim = self.lora_b_stacked[0].shape[2]
484 + # Lazily initialized.
485 + self.indices: torch.Tensor
486
487 def reset_lora(self, index: int):
488 self.lora_a_stacked[0][index] = 0
@@ -690,7 +696,8 @@ class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
696 self.kv_proj_shard_size)
697 self.packed_indices: Optional[torch.Tensor] = None
698 self.standard_indices: Optional[torch.Tensor] = None
693 - self.indices_len: Optional[List[int]] = None
699 + # lazily initialized.
700 + self.indices_len: List[int]
701
702 def reset_lora(self, index: int):
703 self.lora_a_stacked[0][index] = 0
@@ -814,8 +821,9 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
821 dtype=lora_config.lora_dtype,
822 device=self.device,
823 )
817 - self.indices: Optional[torch.Tensor] = None
818 - self.indices_len: Optional[List[int]] = None
824 + # Lazily initialized
825 + self.indices: torch.Tensor
826 + self.indices_len: List[int]
827
828 def reset_lora(self, index: int):
829 self.lora_a_stacked[index] = 0
@@ -991,9 +999,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
999 dtype=self.dtype,
1000 device=self.device,
1001 )
994 - self.indices = None
995 - self.indices_padded = None
996 - self.indices_len = None
1002 + # Lazily initialized.
1003 + self.indices: torch.Tensor
1004 + self.indices_len: List[int]
1005 + self.indices_padded: torch.Tensor
1006
1007 def reset_lora(self, index: int):
1008 self.lora_a_stacked[index] = 0
vllm/lora/lora.py
+17 -11
@@ -97,9 +97,9 @@ class PackedLoRALayerWeights(LoRALayerWeights):
97 self,
98 module_name: str,
99 rank: int,
100 - lora_alphas: List[int],
101 - lora_a: List[torch.Tensor],
102 - lora_b: List[torch.Tensor],
100 + lora_alphas: List[Optional[int]],
101 + lora_a: List[Optional[torch.Tensor]],
102 + lora_b: List[Optional[torch.Tensor]],
103 scaling: Optional[List[float]] = None,
104 ) -> None:
105 super().__init__(
@@ -108,17 +108,20 @@ class PackedLoRALayerWeights(LoRALayerWeights):
108 lora_alpha=0,
109 lora_a=lora_a,
110 lora_b=lora_b,
111 - scaling=scaling,
111 + scaling=scaling, # type: ignore
112 embeddings_tensor=None,
113 )
114 self.lora_alphas = lora_alphas
115 if scaling is None:
116 - self.scaling = [
117 - lora_alpha / self.rank for lora_alpha in self.lora_alphas
116 + self.scaling = [ # type: ignore
117 + lora_alpha / self.rank # type: ignore # noqa
118 + for lora_alpha in self.lora_alphas
119 ]
120
121 @classmethod
121 - def pack(cls, loras: List["LoRALayerWeights"]) -> "PackedLoRALayerWeights":
122 + def pack(
123 + cls, loras: List[Optional["LoRALayerWeights"]]
124 + ) -> "PackedLoRALayerWeights":
125 """Pack a list of LoRAs into a single LoRA.
126
127 If LoRA is None, it signifies that the submodule does not have a LoRA.
@@ -136,16 +139,19 @@ class PackedLoRALayerWeights(LoRALayerWeights):
139 [lora.lora_alpha if lora is not None else None for lora in loras],
140 [lora.lora_a if lora is not None else None for lora in loras],
141 [lora.lora_b if lora is not None else None for lora in loras],
139 - scaling=[1 if lora is not None else None for lora in loras])
142 + scaling=[
143 + 1 if lora is not None else None # type: ignore
144 + for lora in loras
145 + ])
146 return obj
147
148 def optimize(self) -> "PackedLoRALayerWeights":
149 """Optimize the LoRA by merging the scaling into lora_b."""
150 for i in range(len(self.lora_b)):
145 - if self.scaling[i] == 1 or self.lora_b[i] is None:
151 + if self.scaling[i] == 1 or self.lora_b[i] is None: # type: ignore
152 continue
147 - self.lora_b[i] *= self.scaling[i]
148 - self.scaling[i] = 1
153 + self.lora_b[i] *= self.scaling[i] # type: ignore
154 + self.scaling[i] = 1 # type: ignore
155 return self
156
157 @property
vllm/lora/models.py
+34 -30
@@ -3,7 +3,7 @@ import json
3 import math
4 import os
5 import re
6 -from typing import Callable, Dict, Hashable, List, Optional, Tuple, Type
6 +from typing import Callable, Dict, List, Optional, Tuple, Type
7
8 import safetensors.torch
9 import torch
@@ -53,44 +53,46 @@ def convert_mapping(
53 embeddings.
54 indices_len: List of lengths of the above tensors.
55 """
56 - indices = list(mapping.index_mapping).copy()
57 - embedding_indices = indices.copy()
58 - lora_indices = indices.copy()
59 - prompt_mapping = [
56 + index_mapping_indices: List[int] = list(mapping.index_mapping).copy()
57 + embedding_indices = index_mapping_indices.copy()
58 + lora_indices = index_mapping_indices.copy()
59 + prompt_mapping: List[int] = [
60 lora_index_to_id.index(x) if x > 0 else -1
61 for x in mapping.prompt_mapping
62 ]
63 lora_idx = None
64 - for i in range(len(indices)):
64 + for i in range(len(index_mapping_indices)):
65 # TODO index can be slow. optimize
66 - lora_idx = (lora_index_to_id.index(indices[i])
67 - if indices[i] > 0 else -1)
68 - embedding_indices[i] = lora_idx if indices[i] > 0 else 0
69 - indices[i] = i
66 + lora_idx = (lora_index_to_id.index(index_mapping_indices[i])
67 + if index_mapping_indices[i] > 0 else -1)
68 + embedding_indices[i] = lora_idx if index_mapping_indices[i] > 0 else 0
69 + index_mapping_indices[i] = i
70 lora_indices[i] = lora_idx
71
72 - indices = torch.tensor([indices, lora_indices, embedding_indices],
73 - dtype=torch.long,
74 - device="cuda")
75 - prompt_mapping = torch.tensor(prompt_mapping,
76 - device="cuda",
77 - dtype=torch.long)
72 + indices = torch.tensor(
73 + [index_mapping_indices, lora_indices, embedding_indices],
74 + dtype=torch.long,
75 + device="cuda")
76 + prompt_mapping_tensor = torch.tensor(prompt_mapping,
77 + device="cuda",
78 + dtype=torch.long)
79 embeddings_indices = torch.stack([
80 indices[2] * extra_vocab_size,
81 indices[2] * (vocab_size + extra_vocab_size)
82 ])
83 embeddings_indices[embeddings_indices == -1] = max_loras - 1
84 base_indices = indices[1]
84 - sampler_indices = prompt_mapping
85 + sampler_indices = prompt_mapping_tensor
86 sampler_indices_padded = sampler_indices.clone()
87 sampler_indices_padded[sampler_indices_padded == -1] = max_loras - 1
88 sampler_indices_padded = (
89 torch.arange(
90 0, len(sampler_indices_padded), device="cuda", dtype=torch.long) +
91 (sampler_indices_padded * len(sampler_indices_padded)))
91 - indices_len = (base_indices.shape[-1], sampler_indices.shape[-1],
92 - sampler_indices_padded.shape[-1],
93 - embeddings_indices.shape[-1])
92 + indices_len = [
93 + base_indices.shape[-1], sampler_indices.shape[-1],
94 + sampler_indices_padded.shape[-1], embeddings_indices.shape[-1]
95 + ]
96
97 return (base_indices, sampler_indices, sampler_indices_padded,
98 embeddings_indices, indices_len)
@@ -149,6 +151,7 @@ class LoRAModel:
151 if module_name not in loras:
152 lora_embeddings_tensor = None
153 if embeddings:
154 + assert embedding_modules is not None
155 embeddings_module = next(
156 (k for k in embedding_modules if k in module_name),
157 None)
@@ -171,6 +174,7 @@ class LoRAModel:
174 else:
175 loras[module_name].lora_b = tensor.to(device=device,
176 dtype=dtype).t()
177 + assert embedding_padding_modules is not None
178 if any(name in module_name
179 for name in embedding_padding_modules
180 ) and target_embedding_padding is not None:
@@ -295,11 +299,10 @@ class LoRAModelManager:
299 self.max_num_batched_tokens,
300 dtype=torch.long,
301 device="cuda")
298 - self.offsets = []
302 # 4 is the number of indicies tensors defined above
303 # base_indices, sampler_indices, sampler_indices_padded,
304 # embeddings_indices
302 - self.indices_len = [None] * 4
305 + self.indices_len: List[Optional[int]] = [None] * 4
306
307 self.model: nn.Module = model
308 if hasattr(self.model, "supported_lora_modules"):
@@ -312,7 +315,7 @@ class LoRAModelManager:
315 self._registered_loras: Dict[int, LoRAModel] = {}
316 # Dict instead of a Set for compatibility with LRUCache.
317 self._active_loras: Dict[int, None] = {}
315 - self._last_mapping = None
318 + self._last_mapping: Optional[LoRAMapping] = None
319 self._create_lora_modules()
320 self.model.lora_manager = self
321
@@ -370,7 +373,7 @@ class LoRAModelManager:
373 return True
374 return False
375
373 - def _add_lora(self, lora: LoRAModel) -> bool:
376 + def _add_lora(self, lora: LoRAModel):
377 self._create_merged_loras_inplace(lora)
378 self._registered_loras[lora.id] = lora
379
@@ -418,7 +421,7 @@ class LoRAModelManager:
421 def get_lora(self, lora_id: int) -> Optional[LoRAModel]:
422 return self._registered_loras.get(lora_id, None)
423
421 - def remove_all_loras(self) -> bool:
424 + def remove_all_loras(self):
425 """Remove all LoRAModels from the manager."""
426 self._registered_loras.clear()
427 self.lora_index_to_id = [None] * self.lora_slots
@@ -467,6 +470,7 @@ class LoRAModelManager:
470 continue
471 parts = module_name.split(".")
472 if module_name not in self.packed_modules:
473 + assert embedding_modules is not None
474 if parts[-1] in embedding_modules:
475 input_dim = (module.base_layer.org_vocab_size +
476 self.lora_config.lora_extra_vocab_size if
@@ -500,7 +504,7 @@ class LoRAModelManager:
504 else:
505 parts = module_name.split(".")
506 replacements = self.packed_modules_mapping[parts[-1]]
503 - subloras = []
507 + subloras: List[Optional["LoRALayerWeights"]] = []
508 for i, r in enumerate(replacements):
509 lora = LoRALayerWeights.create_dummy_lora_weights(
510 module_name + "." + r,
@@ -538,7 +542,7 @@ class LoRAModelManager:
542
543 def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
544 for module_name, new_module_names in self.packed_modules.items():
541 - replacement_loras = []
545 + replacement_loras: List[Optional[LoRALayerWeights]] = []
546 has_replacement = False
547 for r in new_module_names:
548 lora = lora_model.get_lora(r)
@@ -557,12 +561,12 @@ class LoRAModelManager:
561
562 class LoRALRUCache(LRUCache[LoRAModel]):
563
560 - def __init__(self, capacity: int, deactivate_lora_fn: Callable[[Hashable],
561 - None]):
564 + def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int],
565 + bool]):
566 super().__init__(capacity)
567 self.deactivate_lora_fn = deactivate_lora_fn
568
565 - def _on_remove(self, key: Hashable, value: LoRAModel):
569 + def _on_remove(self, key: int, value: LoRAModel):
570 logger.debug(f"Removing LoRA. int id: {key}")
571 self.deactivate_lora_fn(key)
572 return super()._on_remove(key, value)
vllm/lora/worker_manager.py
+12 -9
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod, abstractproperty
2 -from typing import Any, Dict, List, Optional, Set, Type
2 +from typing import Any, Dict, List, Set, Type
3
4 import torch
5
@@ -37,7 +37,7 @@ class AbstractWorkerLoRAManager(ABC):
37 ...
38
39 @abstractmethod
40 - def set_active_loras(self, lora_requests: List[LoRARequest],
40 + def set_active_loras(self, lora_requests: Set[LoRARequest],
41 lora_mapping: LoRAMapping) -> None:
42 ...
43
@@ -54,7 +54,7 @@ class AbstractWorkerLoRAManager(ABC):
54 ...
55
56 @abstractmethod
57 - def remove_all_loras(self) -> bool:
57 + def remove_all_loras(self):
58 ...
59
60 @abstractmethod
@@ -81,10 +81,11 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
81 embedding_padding_modules: List[str],
82 lora_model_cls: Type[LoRAModel] = LoRAModel,
83 ):
84 - self._lora_manager: Optional[LoRAModelManager] = None
84 self._lora_model_cls = lora_model_cls
85 self.embedding_modules = embedding_modules
86 self.embedding_padding_modules = embedding_padding_modules
87 + # Lazily initialized by create_lora_manager.
88 + self._lora_manager: LoRAModelManager
89 super().__init__(max_num_seqs, max_num_batched_tokens, vocab_size,
90 lora_config, device)
91
@@ -104,7 +105,7 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
105 lora_config=self.lora_config,
106 lora_manager_cls=self._lora_manager_cls,
107 )
107 - self._lora_manager: LoRAModelManager = lora_manager
108 + self._lora_manager = lora_manager
109 return lora_manager.model
110
111 def set_active_loras(self, lora_requests: Set[LoRARequest],
@@ -188,7 +189,7 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
189 def remove_lora(self, lora_id: int) -> bool:
190 return self._lora_manager.remove_lora(lora_id)
191
191 - def remove_all_loras(self) -> bool:
192 + def remove_all_loras(self):
193 self._lora_manager.remove_all_loras()
194
195 def list_loras(self) -> Set[int]:
@@ -217,10 +218,10 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
218 lora_config=self.lora_config,
219 max_num_batched_tokens=self.max_num_batched_tokens,
220 )
220 - self._lora_manager: LRUCacheLoRAModelManager = lora_manager
221 + self._lora_manager = lora_manager
222 return lora_manager.model
223
223 - def _apply_loras(self, lora_requests: List[LoRARequest]) -> None:
224 + def _apply_loras(self, lora_requests: Set[LoRARequest]) -> None:
225 loras_map = {
226 lora_request.lora_int_id: lora_request
227 for lora_request in lora_requests if lora_request
@@ -237,12 +238,14 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
238 if lora_request.lora_int_id not in self.list_loras():
239 # Remove before we load the new lora to save memory
240 if len(self._lora_manager) + 1 > self._lora_manager.capacity:
241 + assert isinstance(self._lora_manager, LRUCacheLoRAModelManager)
242 self._lora_manager.remove_oldest_lora()
243 lora = self._load_lora(lora_request)
244 loaded = self._lora_manager.add_lora(lora)
245 else:
246 # If the lora is already loaded, just touch it to
247 # update its position in the caches
246 - loaded = self._lora_manager.get_lora(lora_request.lora_int_id)
248 + loaded = self._lora_manager.get_lora(
249 + lora_request.lora_int_id) is not None
250 self._lora_manager.activate_lora(lora_request.lora_int_id)
251 return loaded
vllm/worker/model_runner.py
+2 -2
@@ -928,10 +928,10 @@ class ModelRunner:
928 torch.cuda.synchronize()
929 return
930
931 - def remove_all_loras(self) -> bool:
931 + def remove_all_loras(self):
932 if not self.lora_manager:
933 raise RuntimeError("LoRA is not enabled.")
934 - return self.lora_manager.remove_all_loras()
934 + self.lora_manager.remove_all_loras()
935
936 def set_active_loras(self, lora_requests: Set[LoRARequest],
937 lora_mapping: LoRAMapping) -> None: