A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
cleanup: remove adapter commons (#25045)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Simon Mo committed
Sep 17, 2025 at 09:46 UTC
4aa8c7b0477de2cd0f6f1a46437f46e4cb00bae3
11 files changed
+89
-330
pyproject.toml
-1
@@ -115,7 +115,6 @@ follow_imports = "silent"
115
# move the directory here and remove it from tools/mypy.sh
116
files = [
117
"vllm/*.py",
118
- "vllm/adapter_commons",
118
"vllm/assets",
119
"vllm/entrypoints",
120
"vllm/core",
vllm/adapter_commons/__init__.py
vllm/adapter_commons/layers.py
deleted
-16
@@ -1,16 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from dataclasses import dataclass
5
-
6
-
7
-@dataclass
8
-class AdapterMapping:
9
- # Per every token in input_ids:
10
- index_mapping: tuple[int, ...]
11
- # Per sampled token:
12
- prompt_mapping: tuple[int, ...]
13
-
14
- def __post_init__(self):
15
- self.index_mapping = tuple(self.index_mapping)
16
- self.prompt_mapping = tuple(self.prompt_mapping)
\ No newline at end of file
vllm/adapter_commons/models.py
deleted
-106
@@ -1,106 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from abc import ABC, abstractmethod
5
-from typing import Any, Callable, Optional, TypeVar
6
-
7
-from torch import nn
8
-
9
-from vllm.logger import init_logger
10
-from vllm.utils import LRUCache
11
-
12
-logger = init_logger(__name__)
13
-
14
-
15
-class AdapterModel(ABC):
16
-
17
- def __init__(self, model_id=None):
18
- self.id = model_id
19
-
20
- @abstractmethod
21
- def from_local_checkpoint(cls, model_dir, model_id=None, **kwargs):
22
- # Common initialization code
23
- # Load weights or embeddings from local checkpoint
24
- raise NotImplementedError("Subclasses must implement this method.")
25
-
26
-
27
-T = TypeVar('T')
28
-
29
-
30
-class AdapterLRUCache(LRUCache[int, T]):
31
-
32
- def __init__(self, capacity: int, deactivate_fn: Callable[[int], object]):
33
- super().__init__(capacity)
34
- self.deactivate_fn = deactivate_fn
35
-
36
- def _on_remove(self, key: int, value: Optional[T]):
37
- logger.debug("Removing adapter int id: %d", key)
38
- self.deactivate_fn(key)
39
- return super()._on_remove(key, value)
40
-
41
-
42
-class AdapterModelManager(ABC):
43
-
44
- def __init__(
45
- self,
46
- model: nn.Module,
47
- ):
48
- """Create a AdapterModelManager and adapter for a given model.
49
- Args:
50
- model: the model to be adapted.
51
- """
52
- self.model: nn.Module = model
53
- self._registered_adapters: dict[int, Any] = {}
54
- # Dict instead of a Set for compatibility with LRUCache.
55
- self._active_adapters: dict[int, None] = {}
56
- self.adapter_type = 'Adapter'
57
- self._last_mapping = None
58
-
59
- def __len__(self) -> int:
60
- return len(self._registered_adapters)
61
-
62
- @property
63
- @abstractmethod
64
- def adapter_slots(self) -> int:
65
- raise NotImplementedError
66
-
67
- @property
68
- @abstractmethod
69
- def capacity(self) -> int:
70
- raise NotImplementedError
71
-
72
- @abstractmethod
73
- def activate_adapter(self, adapter_id: int) -> bool:
74
- raise NotImplementedError
75
-
76
- @abstractmethod
77
- def deactivate_adapter(self, adapter_id: int) -> bool:
78
- raise NotImplementedError
79
-
80
- @abstractmethod
81
- def add_adapter(self, adapter: Any) -> bool:
82
- raise NotImplementedError
83
-
84
- @abstractmethod
85
- def set_adapter_mapping(self, mapping: Any) -> None:
86
- raise NotImplementedError
87
-
88
- @abstractmethod
89
- def remove_adapter(self, adapter_id: int) -> bool:
90
- raise NotImplementedError
91
-
92
- @abstractmethod
93
- def remove_all_adapters(self) -> None:
94
- raise NotImplementedError
95
-
96
- @abstractmethod
97
- def get_adapter(self, adapter_id: int) -> Optional[Any]:
98
- raise NotImplementedError
99
-
100
- @abstractmethod
101
- def list_adapters(self) -> dict[int, Any]:
102
- raise NotImplementedError
103
-
104
- @abstractmethod
105
- def pin_adapter(self, adapter_id: int) -> bool:
106
- raise NotImplementedError
vllm/adapter_commons/request.py
deleted
-26
@@ -1,26 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from abc import ABC, abstractmethod
5
-
6
-
7
-class AdapterRequest(ABC):
8
- """
9
- Base class for adapter requests.
10
- """
11
-
12
- @property
13
- @abstractmethod
14
- def adapter_id(self) -> int:
15
- raise NotImplementedError
16
-
17
- def __post_init__(self) -> None:
18
- if self.adapter_id < 1:
19
- raise ValueError(f"id must be > 0, got {self.adapter_id}")
20
-
21
- def __eq__(self, value: object) -> bool:
22
- return isinstance(
23
- value, self.__class__) and self.adapter_id == value.adapter_id
24
-
25
- def __hash__(self) -> int:
26
- return hash(self.adapter_id)
vllm/adapter_commons/utils.py
deleted
-93
@@ -1,93 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Any, Callable, Optional
5
-
6
-
7
-## model functions
8
-def deactivate_adapter(adapter_id: int, active_adapters: dict[int, None],
9
- deactivate_func: Callable) -> bool:
10
- if adapter_id in active_adapters:
11
- deactivate_func(adapter_id)
12
- active_adapters.pop(adapter_id)
13
- return True
14
- return False
15
-
16
-
17
-def add_adapter(adapter: Any, registered_adapters: dict[int, Any],
18
- capacity: int, add_func: Callable) -> bool:
19
- if adapter.id not in registered_adapters:
20
- if len(registered_adapters) >= capacity:
21
- raise RuntimeError('No free adapter slots.')
22
- add_func(adapter)
23
- registered_adapters[adapter.id] = adapter
24
- return True
25
- return False
26
-
27
-
28
-def set_adapter_mapping(mapping: Any, last_mapping: Any,
29
- set_mapping_func: Callable) -> Any:
30
- if last_mapping != mapping:
31
- set_mapping_func(mapping)
32
- return mapping
33
- return last_mapping
34
-
35
-
36
-def remove_adapter(adapter_id: int, registered_adapters: dict[int, Any],
37
- deactivate_func: Callable) -> bool:
38
- deactivate_func(adapter_id)
39
- return bool(registered_adapters.pop(adapter_id, None))
40
-
41
-
42
-def list_adapters(registered_adapters: dict[int, Any]) -> dict[int, Any]:
43
- return dict(registered_adapters)
44
-
45
-
46
-def get_adapter(adapter_id: int,
47
- registered_adapters: dict[int, Any]) -> Optional[Any]:
48
- return registered_adapters.get(adapter_id)
49
-
50
-
51
-## worker functions
52
-def set_active_adapters_worker(requests: set[Any], mapping: Optional[Any],
53
- apply_adapters_func,
54
- set_adapter_mapping_func) -> None:
55
- apply_adapters_func(requests)
56
- set_adapter_mapping_func(mapping)
57
-
58
-
59
-def add_adapter_worker(adapter_request: Any, list_adapters_func,
60
- load_adapter_func, add_adapter_func,
61
- activate_adapter_func) -> bool:
62
- if adapter_request.adapter_id in list_adapters_func():
63
- return False
64
- loaded_adapter = load_adapter_func(adapter_request)
65
- loaded = add_adapter_func(loaded_adapter)
66
- activate_adapter_func(loaded_adapter.id)
67
- return loaded
68
-
69
-
70
-def apply_adapters_worker(adapter_requests: set[Any], list_adapters_func,
71
- adapter_slots: int, remove_adapter_func,
72
- add_adapter_func) -> None:
73
- models_that_exist = list_adapters_func()
74
- models_map = {
75
- adapter_request.adapter_id: adapter_request
76
- for adapter_request in adapter_requests if adapter_request
77
- }
78
- if len(models_map) > adapter_slots:
79
- raise RuntimeError(
80
- f"Number of requested models ({len(models_map)}) is greater "
81
- f"than the number of GPU model slots "
82
- f"({adapter_slots}).")
83
- new_models = set(models_map)
84
- models_to_add = new_models - models_that_exist
85
- models_to_remove = models_that_exist - new_models
86
- for adapter_id in models_to_remove:
87
- remove_adapter_func(adapter_id)
88
- for adapter_id in models_to_add:
89
- add_adapter_func(models_map[adapter_id])
90
-
91
-
92
-def list_adapters_worker(adapter_manager_list_adapters_func) -> set[int]:
93
- return set(adapter_manager_list_adapters_func())
vllm/adapter_commons/worker_manager.py
deleted
-39
@@ -1,39 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from abc import ABC, abstractmethod
5
-from typing import Any, Optional
6
-
7
-import torch
8
-
9
-
10
-class AbstractWorkerManager(ABC):
11
-
12
- def __init__(self, device: torch.device):
13
- self.device = device
14
-
15
- @property
16
- @abstractmethod
17
- def is_enabled(self) -> bool:
18
- raise NotImplementedError
19
-
20
- @abstractmethod
21
- def set_active_adapters(self, requests: set[Any],
22
- mapping: Optional[Any]) -> None:
23
- raise NotImplementedError
24
-
25
- @abstractmethod
26
- def add_adapter(self, adapter_request: Any) -> bool:
27
- raise NotImplementedError
28
-
29
- @abstractmethod
30
- def remove_adapter(self, adapter_id: int) -> bool:
31
- raise NotImplementedError
32
-
33
- @abstractmethod
34
- def remove_all_adapters(self) -> None:
35
- raise NotImplementedError
36
-
37
- @abstractmethod
38
- def list_adapters(self) -> set[int]:
39
- raise NotImplementedError
vllm/lora/layers/utils.py
+8
-3
@@ -1,17 +1,22 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
from dataclasses import dataclass
5
6
import torch
7
import torch.nn as nn
8
8
-from vllm.adapter_commons.layers import AdapterMapping
9
-
9
10
@dataclass
12
-class LoRAMapping(AdapterMapping):
11
+class LoRAMapping:
12
+ index_mapping: tuple[int, ...]
13
+ prompt_mapping: tuple[int, ...]
14
is_prefill: bool = False
15
16
+ def __post_init__(self):
17
+ self.index_mapping = tuple(self.index_mapping)
18
+ self.prompt_mapping = tuple(self.prompt_mapping)
19
+
20
21
def _get_lora_device(base_layer: nn.Module) -> torch.device:
22
# code borrowed from https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/vllm/lora/layers.py#L34
vllm/lora/models.py
+52
-25
@@ -4,18 +4,13 @@
4
import math
5
import os
6
from collections.abc import Sequence
7
-from typing import Any, Callable, Optional, Union
7
+from typing import Callable, Optional, TypeVar, Union
8
9
import regex as re
10
import safetensors.torch
11
import torch
12
from torch import nn
13
14
-from vllm.adapter_commons.models import (AdapterLRUCache, AdapterModel,
15
- AdapterModelManager)
16
-from vllm.adapter_commons.utils import (add_adapter, deactivate_adapter,
17
- get_adapter, list_adapters,
18
- remove_adapter, set_adapter_mapping)
14
from vllm.config.lora import LoRAConfig
15
from vllm.logger import init_logger
16
from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping
@@ -33,10 +28,25 @@ from vllm.model_executor.models.interfaces import is_pooling_model
28
from vllm.model_executor.models.module_mapping import MultiModelKeys
29
from vllm.model_executor.models.utils import PPMissingLayer, WeightsMapper
30
from vllm.model_executor.utils import get_packed_modules_mapping
36
-from vllm.utils import is_pin_memory_available
31
+from vllm.utils import LRUCache, is_pin_memory_available
32
33
logger = init_logger(__name__)
34
35
+T = TypeVar("T")
36
+
37
+
38
+class AdapterLRUCache(LRUCache[int, T]):
39
+
40
+ def __init__(self, capacity: int, deactivate_fn: Callable[[int], object]):
41
+ super().__init__(capacity)
42
+ self.deactivate_fn = deactivate_fn
43
+
44
+ def _on_remove(self, key: int, value: Optional[T]):
45
+ logger.debug("Removing adapter int id: %d", key)
46
+ self.deactivate_fn(key)
47
+ return super()._on_remove(key, value)
48
+
49
+
50
_GLOBAL_LORA_ID = 0
51
52
@@ -57,7 +67,7 @@ def is_moe_model(model: nn.Module) -> bool:
67
return False
68
69
60
-class LoRAModel(AdapterModel):
70
+class LoRAModel:
71
"""A LoRA fine-tuned model."""
72
73
def __init__(
@@ -313,7 +323,7 @@ class LoRAModel(AdapterModel):
323
weights_mapper=weights_mapper)
324
325
316
-class LoRAModelManager(AdapterModelManager):
326
+class LoRAModelManager:
327
"""A manager that manages multiple LoRA-fine-tuned models."""
328
329
def __init__(
@@ -336,6 +346,11 @@ class LoRAModelManager(AdapterModelManager):
346
vocab_size: the vocab size of the model.
347
lora_config: the LoRA configuration.
348
"""
349
+ self.model: SupportsLoRA = model
350
+ self._registered_adapters: dict[int, LoRAModel] = {}
351
+ # Dict instead of a set for compatibility with LRUCache.
352
+ self._active_adapters: dict[int, None] = {}
353
+ self.adapter_type = "LoRA"
354
self.lora_config = lora_config
355
self.device = device
356
self.max_num_seqs = max_num_seqs
@@ -347,9 +362,8 @@ class LoRAModelManager(AdapterModelManager):
362
max_num_batched_tokens,
363
max_batches=self.max_num_seqs,
364
device=self.device,
350
- max_loras=self.lora_config.max_loras)
351
-
352
- super().__init__(model)
365
+ max_loras=self.lora_config.max_loras,
366
+ )
367
368
self.supported_lora_modules = get_supported_lora_modules(self.model)
369
assert self.supported_lora_modules, "No supported LoRA modules found in"
@@ -370,7 +384,9 @@ class LoRAModelManager(AdapterModelManager):
384
self._last_mapping: Optional[LoRAMapping] = None
385
self._create_lora_modules()
386
self.model.lora_manager = self
373
- self.adapter_type = 'LoRA'
387
+
388
+ def __len__(self) -> int:
389
+ return len(self._registered_adapters)
390
391
@property
392
def capacity(self) -> int:
@@ -669,28 +685,39 @@ class LoRAModelManager(AdapterModelManager):
685
return lora_model.get_lora(org_module_name)
686
687
def deactivate_adapter(self, adapter_id: int) -> bool:
672
- return deactivate_adapter(adapter_id, self._active_adapters,
673
- self._deactivate_adapter)
688
+ if adapter_id not in self._active_adapters:
689
+ return False
690
+ self._deactivate_adapter(adapter_id)
691
+ self._active_adapters.pop(adapter_id, None)
692
+ return True
693
694
def add_adapter(self, adapter: LoRAModel) -> bool:
695
logger.debug("Adding lora. Model id: %d, "
696
"int id: %d", adapter.id, adapter.id)
678
- return add_adapter(adapter, self._registered_adapters, self.capacity,
679
- self._add_adapter)
697
+ if adapter.id in self._registered_adapters:
698
+ return False
699
+ if len(self._registered_adapters) >= self.capacity:
700
+ raise RuntimeError("No free adapter slots.")
701
+ self._add_adapter(adapter)
702
+ return True
703
704
def set_adapter_mapping(self, mapping: LoRAMapping) -> None:
682
- self._last_mapping = set_adapter_mapping(mapping, self._last_mapping,
683
- self._set_adapter_mapping)
705
+ if self._last_mapping != mapping:
706
+ self._set_adapter_mapping(mapping)
707
+ self._last_mapping = mapping
708
709
def remove_adapter(self, adapter_id: int) -> bool:
686
- return remove_adapter(adapter_id, self._registered_adapters,
687
- self.deactivate_adapter)
710
+ self.deactivate_adapter(adapter_id)
711
+ if adapter_id not in self._registered_adapters:
712
+ return False
713
+ self._registered_adapters.pop(adapter_id, None)
714
+ return True
715
689
- def list_adapters(self) -> dict[int, Any]:
690
- return list_adapters(self._registered_adapters)
716
+ def list_adapters(self) -> dict[int, LoRAModel]:
717
+ return dict(self._registered_adapters)
718
692
- def get_adapter(self, adapter_id: int) -> Optional[Any]:
693
- return get_adapter(adapter_id, self._registered_adapters)
719
+ def get_adapter(self, adapter_id: int) -> Optional[LoRAModel]:
720
+ return self._registered_adapters.get(adapter_id)
721
722
723
class LoRALRUCache(AdapterLRUCache[LoRAModel]):
vllm/lora/request.py
+2
-4
@@ -6,8 +6,6 @@ from typing import Optional
6
7
import msgspec
8
9
-from vllm.adapter_commons.request import AdapterRequest
10
-
9
10
class LoRARequest(
11
msgspec.Struct,
@@ -24,8 +22,6 @@ class LoRARequest(
22
lora_int_id must be globally unique for a given adapter.
23
This is currently not enforced in vLLM.
24
"""
27
- __metaclass__ = AdapterRequest
28
-
25
lora_name: str
26
lora_int_id: int
27
lora_path: str = ""
@@ -35,6 +31,8 @@ class LoRARequest(
31
tensorizer_config_dict: Optional[dict] = None
32
33
def __post_init__(self):
34
+ if self.lora_int_id < 1:
35
+ raise ValueError(f"id must be > 0, got {self.lora_int_id}")
36
if self.lora_local_path:
37
warnings.warn(
38
"The 'lora_local_path' attribute is deprecated "
vllm/lora/worker_manager.py
+27
-17
@@ -6,11 +6,6 @@ from typing import Any, Literal, Optional, Union
6
7
import torch
8
9
-from vllm.adapter_commons.utils import (add_adapter_worker,
10
- apply_adapters_worker,
11
- list_adapters_worker,
12
- set_active_adapters_worker)
13
-from vllm.adapter_commons.worker_manager import AbstractWorkerManager
9
from vllm.config.lora import LoRAConfig
10
from vllm.logger import init_logger
11
from vllm.lora.models import (LoRAModel, LoRAModelManager,
@@ -22,7 +17,7 @@ from vllm.lora.utils import get_adapter_absolute_path
17
logger = init_logger(__name__)
18
19
25
-class WorkerLoRAManager(AbstractWorkerManager):
20
+class WorkerLoRAManager:
21
"""WorkerLoRAManager that manages LoRA models on the worker side.
22
23
Every request, the requested LoRAs will be loaded (unless they are already
@@ -51,7 +46,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
46
self.vocab_size = vocab_size
47
self.lora_config = lora_config
48
self.max_position_embeddings = max_position_embeddings
54
- super().__init__(device)
49
+ self.device = device
50
# Lazily initialized by create_lora_manager.
51
self._adapter_manager: LoRAModelManager
52
@@ -164,19 +159,34 @@ class WorkerLoRAManager(AbstractWorkerManager):
159
160
def set_active_adapters(self, requests: set[Any],
161
mapping: Optional[Any]) -> None:
167
- set_active_adapters_worker(requests, mapping, self._apply_adapters,
168
- self._adapter_manager.set_adapter_mapping)
162
+ self._apply_adapters(requests)
163
+ if mapping is not None:
164
+ self._adapter_manager.set_adapter_mapping(mapping)
165
166
def _apply_adapters(self, adapter_requests: set[Any]) -> None:
171
- apply_adapters_worker(adapter_requests, self.list_adapters,
172
- self._adapter_manager.adapter_slots,
173
- self.remove_adapter, self.add_adapter)
167
+ existing_adapters = self.list_adapters()
168
+ models_map = {
169
+ adapter_request.adapter_id: adapter_request
170
+ for adapter_request in adapter_requests if adapter_request
171
+ }
172
+ if len(models_map) > self._adapter_manager.adapter_slots:
173
+ raise RuntimeError(
174
+ f"Number of requested models ({len(models_map)}) is greater "
175
+ "than the number of GPU model slots "
176
+ f"({self._adapter_manager.adapter_slots}).")
177
+ requested_ids = set(models_map)
178
+ for adapter_id in existing_adapters - requested_ids:
179
+ self.remove_adapter(adapter_id)
180
+ for adapter_id in requested_ids - existing_adapters:
181
+ self.add_adapter(models_map[adapter_id])
182
183
def add_adapter(self, adapter_request: Any) -> bool:
176
- return add_adapter_worker(adapter_request, self.list_adapters,
177
- self._load_adapter,
178
- self._adapter_manager.add_adapter,
179
- self._adapter_manager.activate_adapter)
184
+ if adapter_request.adapter_id in self.list_adapters():
185
+ return False
186
+ loaded_adapter = self._load_adapter(adapter_request)
187
+ loaded = self._adapter_manager.add_adapter(loaded_adapter)
188
+ self._adapter_manager.activate_adapter(loaded_adapter.id)
189
+ return loaded
190
191
def remove_adapter(self, adapter_id: int) -> bool:
192
return self._adapter_manager.remove_adapter(adapter_id)
@@ -185,7 +195,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
195
self._adapter_manager.remove_all_adapters()
196
197
def list_adapters(self) -> set[int]:
188
- return list_adapters_worker(self._adapter_manager.list_adapters)
198
+ return set(self._adapter_manager.list_adapters())
199
200
201
class LRUCacheWorkerLoRAManager(WorkerLoRAManager):