A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Enhance attention selector (#4751)
Woosuk Kwon committed
May 13, 2024 at 10:47 UTC
0fca3cdcf265cd375bca684d951702b6b7adf65a
49 files changed
+573
-220
tests/worker/test_model_runner.py
-1
@@ -307,7 +307,6 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
307
308
assert len(attn_metadata.slot_mapping) == len(input_tokens)
309
assert len(input_positions) == len(input_tokens)
310
- assert attn_metadata.kv_cache_dtype == "auto"
310
assert attn_metadata.num_prefills == prefill_batch_size
311
if enforce_eager:
312
assert attn_metadata.num_decode_tokens == decode_batch_size
vllm/attention/__init__.py
+2
-2
@@ -5,9 +5,9 @@ from vllm.attention.layer import Attention
5
from vllm.attention.selector import get_attn_backend
6
7
__all__ = [
8
+ "Attention",
9
"AttentionBackend",
10
"AttentionMetadata",
10
- "Attention",
11
- "get_attn_backend",
11
"AttentionMetadataPerStage",
12
+ "get_attn_backend",
13
]
vllm/attention/backends/abstract.py
+2
-3
@@ -94,8 +94,6 @@ class AttentionMetadata(Generic[T]):
94
# is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
95
# in block 0, and 1st slot in block 1, respectively.
96
slot_mapping: torch.Tensor
97
- # The kv cache's data type.
98
- kv_cache_dtype: str
97
98
def __post_init__(self):
99
if self.num_prefill_tokens > 0:
@@ -116,6 +114,7 @@ class AttentionImpl(ABC):
114
num_kv_heads: Optional[int] = None,
115
alibi_slopes: Optional[List[float]] = None,
116
sliding_window: Optional[int] = None,
117
+ kv_cache_dtype: str = "auto",
118
) -> None:
119
raise NotImplementedError
120
@@ -127,6 +126,6 @@ class AttentionImpl(ABC):
126
value: torch.Tensor,
127
kv_cache: torch.Tensor,
128
attn_metadata: AttentionMetadata,
130
- kv_scale: float,
129
+ kv_scale: float = 1.0,
130
) -> torch.Tensor:
131
raise NotImplementedError
vllm/attention/backends/flash_attn.py
+7
-6
@@ -140,16 +140,18 @@ class FlashAttentionImpl(AttentionImpl):
140
num_kv_heads: Optional[int] = None,
141
alibi_slopes: Optional[List[float]] = None,
142
sliding_window: Optional[int] = None,
143
+ kv_cache_dtype: str = "auto",
144
) -> None:
145
self.num_heads = num_heads
146
self.head_size = head_size
147
self.scale = float(scale)
148
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
148
- self.sliding_window = ((sliding_window, sliding_window)
149
- if sliding_window is not None else (-1, -1))
149
if alibi_slopes is not None:
150
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
151
self.alibi_slopes = alibi_slopes
152
+ self.sliding_window = ((sliding_window, sliding_window)
153
+ if sliding_window is not None else (-1, -1))
154
+ self.kv_cache_dtype = kv_cache_dtype
155
156
assert self.num_heads % self.num_kv_heads == 0
157
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -167,7 +169,7 @@ class FlashAttentionImpl(AttentionImpl):
169
value: torch.Tensor,
170
kv_cache: torch.Tensor,
171
attn_metadata: AttentionMetadata[FlashAttentionMetadata],
170
- kv_scale: float,
172
+ kv_scale: float = 1.0,
173
) -> torch.Tensor:
174
"""Forward pass with FlashAttention and PagedAttention.
175
@@ -196,8 +198,7 @@ class FlashAttentionImpl(AttentionImpl):
198
PagedAttention.write_to_paged_cache(key, value, key_cache,
199
value_cache,
200
attn_metadata.slot_mapping,
199
- attn_metadata.kv_cache_dtype,
200
- kv_scale)
201
+ self.kv_cache_dtype, kv_scale)
202
203
num_prefill_tokens = attn_metadata.num_prefill_tokens
204
num_decode_tokens = attn_metadata.num_decode_tokens
@@ -264,7 +265,7 @@ class FlashAttentionImpl(AttentionImpl):
265
decode_meta.block_tables,
266
decode_meta.seq_lens_tensor,
267
decode_meta.max_seq_len,
267
- attn_metadata.kv_cache_dtype,
268
+ self.kv_cache_dtype,
269
self.num_kv_heads,
270
self.scale,
271
self.alibi_slopes,
vllm/attention/backends/flashinfer.py
+23
-10
@@ -149,20 +149,33 @@ class FlashInferImpl(AttentionImpl):
149
num_kv_heads: Optional[int] = None,
150
alibi_slopes: Optional[List[float]] = None,
151
sliding_window: Optional[int] = None,
152
+ kv_cache_dtype: str = "auto",
153
) -> None:
153
- if sliding_window is not None:
154
- raise ValueError("Sliding window is not supported in FlashInfer.")
155
- self.sliding_window = (-1, -1)
156
- self.alibi_slopes = alibi_slopes
157
- self.scale = scale
154
self.num_heads = num_heads
155
self.head_size = head_size
156
+ self.scale = float(scale)
157
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
158
+ if alibi_slopes is not None:
159
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
160
+ self.alibi_slopes = alibi_slopes
161
+ if sliding_window is not None:
162
+ raise ValueError("Sliding window is not supported in FlashInfer.")
163
+ self.sliding_window = (-1, -1)
164
+ self.kv_cache_dtype = kv_cache_dtype
165
162
- def forward(self, query: torch.Tensor, key: torch.Tensor,
163
- value: torch.Tensor, kv_cache: Optional[torch.Tensor],
164
- attn_metadata: AttentionMetadata[FlashInferMetadata],
165
- kv_scale: float):
166
+ assert self.num_heads % self.num_kv_heads == 0
167
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
168
+
169
+ def forward(
170
+ self,
171
+ query: torch.Tensor,
172
+ key: torch.Tensor,
173
+ value: torch.Tensor,
174
+ kv_cache: Optional[torch.Tensor],
175
+ attn_metadata: AttentionMetadata[FlashInferMetadata],
176
+ kv_scale: float = 1.0,
177
+ ) -> torch.Tensor:
178
+ assert kv_scale == 1.0
179
num_tokens, hidden_size = query.shape
180
query = query.view(-1, self.num_heads, self.head_size)
181
key = key.view(-1, self.num_kv_heads, self.head_size)
@@ -183,7 +196,7 @@ class FlashInferImpl(AttentionImpl):
196
kv_cache[:, 0],
197
kv_cache[:, 1],
198
attn_metadata.slot_mapping.flatten(),
186
- attn_metadata.kv_cache_dtype,
199
+ self.kv_cache_dtype,
200
)
201
202
if prefill_meta := attn_metadata.prefill_metadata:
vllm/attention/backends/rocm_flash_attn.py
+9
-7
@@ -138,25 +138,27 @@ class ROCmFlashAttentionImpl(AttentionImpl):
138
num_kv_heads: Optional[int] = None,
139
alibi_slopes: Optional[List[float]] = None,
140
sliding_window: Optional[int] = None,
141
+ kv_cache_dtype: str = "auto",
142
) -> None:
143
self.num_heads = num_heads
144
self.head_size = head_size
145
self.scale = float(scale)
146
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
146
- self.sliding_window = ((sliding_window, sliding_window)
147
- if sliding_window is not None else (-1, -1))
147
if alibi_slopes is not None:
148
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
149
self.alibi_slopes = alibi_slopes
150
+ self.sliding_window = ((sliding_window, sliding_window)
151
+ if sliding_window is not None else (-1, -1))
152
+ self.kv_cache_dtype = kv_cache_dtype
153
154
assert self.num_heads % self.num_kv_heads == 0
155
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
156
155
- suppored_head_sizes = PagedAttention.get_supported_head_sizes()
156
- if head_size not in suppored_head_sizes:
157
+ supported_head_sizes = PagedAttention.get_supported_head_sizes()
158
+ if head_size not in supported_head_sizes:
159
raise ValueError(
160
f"Head size {head_size} is not supported by PagedAttention. "
159
- f"Supported head sizes are: {suppored_head_sizes}.")
161
+ f"Supported head sizes are: {supported_head_sizes}.")
162
163
self.use_naive_attn = False
164
# NOTE: Allow for switching between Triton and CK. Defaulting to triton.
@@ -229,7 +231,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
231
key_cache,
232
value_cache,
233
attn_metadata.slot_mapping,
232
- attn_metadata.kv_cache_dtype,
234
+ self.kv_cache_dtype,
235
kv_scale,
236
)
237
@@ -323,7 +325,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
325
decode_meta.block_tables,
326
decode_meta.seq_lens_tensor,
327
decode_meta.max_seq_len,
326
- attn_metadata.kv_cache_dtype,
328
+ self.kv_cache_dtype,
329
self.num_kv_heads,
330
self.scale,
331
self.alibi_slopes,
vllm/attention/backends/torch_sdpa.py
+17
-11
@@ -83,26 +83,32 @@ class TorchSDPABackendImpl(AttentionImpl):
83
num_kv_heads: Optional[int] = None,
84
alibi_slopes: Optional[List[float]] = None,
85
sliding_window: Optional[int] = None,
86
+ kv_cache_dtype: str = "auto",
87
) -> None:
88
self.num_heads = num_heads
89
self.head_size = head_size
90
self.scale = float(scale)
91
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
91
- self.sliding_window = sliding_window
92
if alibi_slopes is not None:
93
- assert len(alibi_slopes) == num_heads
93
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
94
self.alibi_slopes = alibi_slopes
96
- self.need_mask = (self.alibi_slopes is not None
97
- or self.sliding_window is not None)
95
+ self.sliding_window = sliding_window
96
+ self.kv_cache_dtype = kv_cache_dtype
97
98
assert self.num_heads % self.num_kv_heads == 0
99
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
101
- suppored_head_sizes = PagedAttention.get_supported_head_sizes()
102
- if head_size not in suppored_head_sizes:
100
+ self.need_mask = (self.alibi_slopes is not None
101
+ or self.sliding_window is not None)
102
+
103
+ supported_head_sizes = PagedAttention.get_supported_head_sizes()
104
+ if head_size not in supported_head_sizes:
105
raise ValueError(
106
f"Head size {head_size} is not supported by PagedAttention. "
105
- f"Supported head sizes are: {suppored_head_sizes}.")
107
+ f"Supported head sizes are: {supported_head_sizes}.")
108
+ if kv_cache_dtype != "auto":
109
+ raise NotImplementedError(
110
+ "Torch SDPA backend does not support FP8 KV cache. "
111
+ "Please use xFormers backend instead.")
112
113
def forward(
114
self,
@@ -111,7 +117,7 @@ class TorchSDPABackendImpl(AttentionImpl):
117
value: torch.Tensor,
118
kv_cache: Optional[torch.Tensor],
119
attn_metadata: TorchSDPAMetadata, # type: ignore
114
- kv_scale: float,
120
+ kv_scale: float = 1.0,
121
) -> torch.Tensor:
122
"""Forward pass with torch SDPA and PagedAttention.
123
@@ -124,6 +130,7 @@ class TorchSDPABackendImpl(AttentionImpl):
130
Returns:
131
shape = [num_tokens, num_heads * head_size]
132
"""
133
+ assert kv_scale == 1.0
134
num_tokens, hidden_size = query.shape
135
# Reshape the query, key, and value tensors.
136
query = query.view(-1, self.num_heads, self.head_size)
@@ -136,8 +143,7 @@ class TorchSDPABackendImpl(AttentionImpl):
143
PagedAttention.write_to_paged_cache(key, value, key_cache,
144
value_cache,
145
attn_metadata.slot_mapping,
139
- attn_metadata.kv_cache_dtype,
140
- kv_scale)
146
+ self.kv_cache_dtype, kv_scale)
147
148
if attn_metadata.is_prompt:
149
assert attn_metadata.seq_lens is not None
@@ -195,7 +201,7 @@ class TorchSDPABackendImpl(AttentionImpl):
201
attn_metadata.block_tables,
202
attn_metadata.seq_lens_tensor,
203
attn_metadata.max_seq_len,
198
- attn_metadata.kv_cache_dtype,
204
+ self.kv_cache_dtype,
205
self.num_kv_heads,
206
self.scale,
207
self.alibi_slopes,
vllm/attention/backends/xformers.py
+6
-6
@@ -149,15 +149,17 @@ class XFormersImpl(AttentionImpl):
149
num_kv_heads: Optional[int] = None,
150
alibi_slopes: Optional[List[float]] = None,
151
sliding_window: Optional[int] = None,
152
+ kv_cache_dtype: str = "auto",
153
) -> None:
154
self.num_heads = num_heads
155
self.head_size = head_size
156
self.scale = float(scale)
157
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
157
- self.sliding_window = sliding_window
158
if alibi_slopes is not None:
159
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
160
self.alibi_slopes = alibi_slopes
161
+ self.sliding_window = sliding_window
162
+ self.kv_cache_dtype = kv_cache_dtype
163
164
assert self.num_heads % self.num_kv_heads == 0
165
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -175,7 +177,7 @@ class XFormersImpl(AttentionImpl):
177
value: torch.Tensor,
178
kv_cache: Optional[torch.Tensor],
179
attn_metadata: AttentionMetadata[XFormersMetadata],
178
- kv_scale: float,
180
+ kv_scale: float = 1.0,
181
) -> torch.Tensor:
182
"""Forward pass with xFormers and PagedAttention.
183
@@ -188,7 +190,6 @@ class XFormersImpl(AttentionImpl):
190
Returns:
191
shape = [num_tokens, num_heads * head_size]
192
"""
191
- num_tokens, hidden_size = query.shape
193
query = query.view(-1, self.num_heads, self.head_size)
194
key = key.view(-1, self.num_kv_heads, self.head_size)
195
value = value.view(-1, self.num_kv_heads, self.head_size)
@@ -203,8 +204,7 @@ class XFormersImpl(AttentionImpl):
204
PagedAttention.write_to_paged_cache(key, value, key_cache,
205
value_cache,
206
attn_metadata.slot_mapping,
206
- attn_metadata.kv_cache_dtype,
207
- kv_scale)
207
+ self.kv_cache_dtype, kv_scale)
208
209
num_prefill_tokens = attn_metadata.num_prefill_tokens
210
num_decode_tokens = attn_metadata.num_decode_tokens
@@ -262,7 +262,7 @@ class XFormersImpl(AttentionImpl):
262
decode_meta.block_tables,
263
decode_meta.seq_lens_tensor,
264
decode_meta.max_seq_len,
265
- attn_metadata.kv_cache_dtype,
265
+ self.kv_cache_dtype,
266
self.num_kv_heads,
267
self.scale,
268
self.alibi_slopes,
vllm/attention/layer.py
+17
-2
@@ -7,6 +7,7 @@ import torch.nn as nn
7
from vllm.attention.backends.abstract import (AttentionMetadata,
8
AttentionMetadataPerStage)
9
from vllm.attention.selector import get_attn_backend
10
+from vllm.config import CacheConfig
11
12
13
class Attention(nn.Module):
@@ -29,10 +30,24 @@ class Attention(nn.Module):
30
num_kv_heads: Optional[int] = None,
31
alibi_slopes: Optional[List[float]] = None,
32
sliding_window: Optional[int] = None,
33
+ cache_config: Optional[CacheConfig] = None,
34
) -> None:
35
super().__init__()
34
- self.backend = get_attn_backend(torch.get_default_dtype())
35
- impl_cls = self.backend.get_impl_cls()
36
+ if cache_config is not None:
37
+ kv_cache_dtype = cache_config.cache_dtype
38
+ block_size = cache_config.block_size
39
+ else:
40
+ kv_cache_dtype = "auto"
41
+ block_size = 16
42
+ if num_kv_heads is None:
43
+ num_kv_heads = num_heads
44
+ # During model initialization, the default dtype is set as the model
45
+ # weight and activation dtype.
46
+ dtype = torch.get_default_dtype()
47
+ attn_backend = get_attn_backend(num_heads, head_size, num_kv_heads,
48
+ sliding_window, dtype, kv_cache_dtype,
49
+ block_size)
50
+ impl_cls = attn_backend.get_impl_cls()
51
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
52
alibi_slopes, sliding_window)
53
vllm/attention/selector.py
+23
-5
@@ -1,6 +1,6 @@
1
import enum
2
from functools import lru_cache
3
-from typing import Type
3
+from typing import Optional, Type
4
5
import torch
6
@@ -21,8 +21,18 @@ class _Backend(enum.Enum):
21
22
23
@lru_cache(maxsize=None)
24
-def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
25
- backend = _which_attn_to_use(dtype)
24
+def get_attn_backend(
25
+ num_heads: int,
26
+ head_size: int,
27
+ num_kv_heads: int,
28
+ sliding_window: Optional[int],
29
+ dtype: torch.dtype,
30
+ kv_cache_dtype: Optional[str],
31
+ block_size: int,
32
+) -> Type[AttentionBackend]:
33
+ backend = _which_attn_to_use(num_heads, head_size, num_kv_heads,
34
+ sliding_window, dtype, kv_cache_dtype,
35
+ block_size)
36
if backend == _Backend.FLASH_ATTN:
37
logger.info("Using FlashAttention-2 backend.")
38
from vllm.attention.backends.flash_attn import ( # noqa: F401
@@ -44,14 +54,22 @@ def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
54
return TorchSDPABackend
55
elif backend == _Backend.FLASHINFER:
56
logger.info("Using Flashinfer backend.")
47
- logger.warning("Eager mode is enforced for the Flashinfer backend. ")
57
+ logger.warning("Eager mode is enforced for the Flashinfer backend.")
58
from vllm.attention.backends.flashinfer import FlashInferBackend
59
return FlashInferBackend
60
else:
61
raise ValueError("Invalid attention backend.")
62
63
54
-def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
64
+def _which_attn_to_use(
65
+ num_heads: int,
66
+ head_size: int,
67
+ num_kv_heads: int,
68
+ sliding_window: Optional[int],
69
+ dtype: torch.dtype,
70
+ kv_cache_dtype: Optional[str],
71
+ block_size: int,
72
+) -> _Backend:
73
"""Returns which flash attention backend to use."""
74
if is_cpu():
75
return _Backend.TORCH_SDPA
vllm/model_executor/model_loader/__init__.py
+11
-8
@@ -2,26 +2,29 @@ from typing import Optional
2
3
from torch import nn
4
5
-from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
6
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
5
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
+ ModelConfig, ParallelConfig, SchedulerConfig,
7
+ VisionLanguageConfig)
8
from vllm.model_executor.model_loader.loader import (BaseModelLoader,
9
get_model_loader)
10
from vllm.model_executor.model_loader.utils import (
11
get_architecture_class_name, get_model_architecture)
12
13
13
-def get_model(
14
- *, model_config: ModelConfig, load_config: LoadConfig,
15
- device_config: DeviceConfig, parallel_config: ParallelConfig,
16
- scheduler_config: SchedulerConfig, lora_config: Optional[LoRAConfig],
17
- vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
14
+def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
15
+ device_config: DeviceConfig, parallel_config: ParallelConfig,
16
+ scheduler_config: SchedulerConfig,
17
+ lora_config: Optional[LoRAConfig],
18
+ vision_language_config: Optional[VisionLanguageConfig],
19
+ cache_config: CacheConfig) -> nn.Module:
20
loader = get_model_loader(load_config)
21
return loader.load_model(model_config=model_config,
22
device_config=device_config,
23
lora_config=lora_config,
24
vision_language_config=vision_language_config,
25
parallel_config=parallel_config,
24
- scheduler_config=scheduler_config)
26
+ scheduler_config=scheduler_config,
27
+ cache_config=cache_config)
28
29
30
__all__ = [
vllm/model_executor/model_loader/loader.py
+39
-22
@@ -9,9 +9,9 @@ import huggingface_hub
9
import torch
10
from torch import nn
11
12
-from vllm.config import (DeviceConfig, LoadConfig, LoadFormat, LoRAConfig,
13
- ModelConfig, ParallelConfig, SchedulerConfig,
14
- VisionLanguageConfig)
12
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
13
+ LoRAConfig, ModelConfig, ParallelConfig,
14
+ SchedulerConfig, VisionLanguageConfig)
15
from vllm.envs import VLLM_USE_MODELSCOPE
16
from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization.base_config import (
@@ -77,15 +77,16 @@ def _get_model_initialization_kwargs(
77
return extra_kwargs
78
79
80
-def _initialize_model(
81
- model_config: ModelConfig, load_config: LoadConfig,
82
- lora_config: Optional[LoRAConfig],
83
- vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
80
+def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
81
+ lora_config: Optional[LoRAConfig],
82
+ vision_language_config: Optional[VisionLanguageConfig],
83
+ cache_config: CacheConfig) -> nn.Module:
84
"""Initialize a model with the given configurations."""
85
model_class = get_model_architecture(model_config)[0]
86
quant_config = _get_quantization_config(model_config, load_config)
87
88
return model_class(config=model_config.hf_config,
89
+ cache_config=cache_config,
90
quant_config=quant_config,
91
**_get_model_initialization_kwargs(
92
model_class, lora_config, vision_language_config))
@@ -103,7 +104,8 @@ class BaseModelLoader(ABC):
104
lora_config: Optional[LoRAConfig],
105
vision_language_config: Optional[VisionLanguageConfig],
106
parallel_config: ParallelConfig,
106
- scheduler_config: SchedulerConfig) -> nn.Module:
107
+ scheduler_config: SchedulerConfig,
108
+ cache_config: CacheConfig) -> nn.Module:
109
"""Load a model with the given configurations."""
110
...
111
@@ -216,11 +218,13 @@ class DefaultModelLoader(BaseModelLoader):
218
lora_config: Optional[LoRAConfig],
219
vision_language_config: Optional[VisionLanguageConfig],
220
parallel_config: ParallelConfig,
219
- scheduler_config: SchedulerConfig) -> nn.Module:
221
+ scheduler_config: SchedulerConfig,
222
+ cache_config: CacheConfig) -> nn.Module:
223
with set_default_torch_dtype(model_config.dtype):
224
with torch.device(device_config.device):
225
model = _initialize_model(model_config, self.load_config,
223
- lora_config, vision_language_config)
226
+ lora_config, vision_language_config,
227
+ cache_config)
228
model.load_weights(
229
self._get_weights_iterator(model_config.model,
230
model_config.revision,
@@ -253,11 +257,13 @@ class DummyModelLoader(BaseModelLoader):
257
lora_config: Optional[LoRAConfig],
258
vision_language_config: Optional[VisionLanguageConfig],
259
parallel_config: ParallelConfig,
256
- scheduler_config: SchedulerConfig) -> nn.Module:
260
+ scheduler_config: SchedulerConfig,
261
+ cache_config: CacheConfig) -> nn.Module:
262
with set_default_torch_dtype(model_config.dtype):
263
with torch.device(device_config.device):
264
model = _initialize_model(model_config, self.load_config,
260
- lora_config, vision_language_config)
265
+ lora_config, vision_language_config,
266
+ cache_config)
267
# NOTE(woosuk): For accurate performance evaluation, we assign
268
# random values to the weights.
269
initialize_dummy_weights(model)
@@ -286,9 +292,12 @@ class TensorizerLoader(BaseModelLoader):
292
return tensorizer_weights_iterator(tensorizer_args)
293
294
def _load_model_unserialized(
289
- self, model_config: ModelConfig, device_config: DeviceConfig,
290
- lora_config: Optional[LoRAConfig],
291
- vision_language_config: Optional[VisionLanguageConfig]
295
+ self,
296
+ model_config: ModelConfig,
297
+ device_config: DeviceConfig,
298
+ lora_config: Optional[LoRAConfig],
299
+ vision_language_config: Optional[VisionLanguageConfig],
300
+ cache_config: CacheConfig,
301
) -> nn.Module:
302
"""Load an unserialized model with tensorizer.
303
@@ -299,15 +308,19 @@ class TensorizerLoader(BaseModelLoader):
308
with set_default_torch_dtype(model_config.dtype):
309
with torch.device(device_config.device):
310
model = _initialize_model(model_config, self.load_config,
302
- lora_config, vision_language_config)
311
+ lora_config, vision_language_config,
312
+ cache_config)
313
314
model.load_weights(self._get_weights_iterator())
315
return model.eval()
316
317
def _load_model_serialized(
308
- self, model_config: ModelConfig, device_config: DeviceConfig,
309
- lora_config: Optional[LoRAConfig],
310
- vision_language_config: Optional[VisionLanguageConfig]
318
+ self,
319
+ model_config: ModelConfig,
320
+ device_config: DeviceConfig,
321
+ lora_config: Optional[LoRAConfig],
322
+ vision_language_config: Optional[VisionLanguageConfig],
323
+ cache_config: CacheConfig,
324
) -> nn.Module:
325
"""Load a serialized model with tensorizer.
326
@@ -321,6 +334,7 @@ class TensorizerLoader(BaseModelLoader):
334
extra_kwargs = _get_model_initialization_kwargs(
335
model_class, lora_config, vision_language_config)
336
extra_kwargs["quant_config"] = quant_config
337
+ extra_kwargs["cache_config"] = cache_config
338
339
tensorizer_config = copy.copy(self.tensorizer_config)
340
tensorizer_config.model_class = model_class
@@ -335,16 +349,19 @@ class TensorizerLoader(BaseModelLoader):
349
lora_config: Optional[LoRAConfig],
350
vision_language_config: Optional[VisionLanguageConfig],
351
parallel_config: ParallelConfig,
338
- scheduler_config: SchedulerConfig) -> nn.Module:
352
+ scheduler_config: SchedulerConfig,
353
+ cache_config: CacheConfig) -> nn.Module:
354
self._verify_config(model_config, parallel_config)
355
356
if is_vllm_serialized_tensorizer(self.tensorizer_config):
357
return self._load_model_serialized(model_config, device_config,
358
lora_config,
344
- vision_language_config)
359
+ vision_language_config,
360
+ cache_config)
361
return self._load_model_unserialized(model_config, device_config,
362
lora_config,
347
- vision_language_config)
363
+ vision_language_config,
364
+ cache_config)
365
366
367
def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
vllm/model_executor/models/arctic.py
+13
-3
@@ -5,6 +5,7 @@ import torch
5
from torch import nn
6
7
from vllm.attention import Attention, AttentionMetadata
8
+from vllm.config import CacheConfig
9
from vllm.distributed import (get_tensor_model_parallel_rank,
10
get_tensor_model_parallel_world_size,
11
tensor_model_parallel_all_reduce)
@@ -215,6 +216,7 @@ class ArcticAttention(nn.Module):
216
self,
217
config: ArcticConfig,
218
layer_idx: Optional[int] = None,
219
+ cache_config: Optional[CacheConfig] = None,
220
quant_config: Optional[QuantizationConfig] = None,
221
):
222
super().__init__()
@@ -265,7 +267,8 @@ class ArcticAttention(nn.Module):
267
self.attn = Attention(self.num_heads,
268
self.head_dim,
269
self.scaling,
268
- num_kv_heads=self.num_kv_heads)
270
+ num_kv_heads=self.num_kv_heads,
271
+ cache_config=cache_config)
272
273
def forward(
274
self,
@@ -288,6 +291,7 @@ class ArcticDecoderLayer(nn.Module):
291
self,
292
config: ArcticConfig,
293
layer_idx: int,
294
+ cache_config: Optional[CacheConfig] = None,
295
quant_config: Optional[QuantizationConfig] = None,
296
) -> None:
297
super().__init__()
@@ -297,6 +301,7 @@ class ArcticDecoderLayer(nn.Module):
301
self.use_residual = config.use_residual and is_moe_layer
302
self.self_attn = ArcticAttention(config,
303
layer_idx,
304
+ cache_config,
305
quant_config=quant_config)
306
self.block_sparse_moe = ArcticMoE(
307
config,
@@ -356,6 +361,7 @@ class ArcticModel(nn.Module):
361
def __init__(
362
self,
363
config: ArcticConfig,
364
+ cache_config: Optional[CacheConfig] = None,
365
quant_config: Optional[QuantizationConfig] = None,
366
) -> None:
367
super().__init__()
@@ -366,7 +372,10 @@ class ArcticModel(nn.Module):
372
config.hidden_size,
373
org_num_embeddings=self.vocab_size)
374
self.layers = nn.ModuleList([
369
- ArcticDecoderLayer(config, layer_idx, quant_config=quant_config)
375
+ ArcticDecoderLayer(config,
376
+ layer_idx,
377
+ cache_config,
378
+ quant_config=quant_config)
379
for layer_idx in range(config.num_hidden_layers)
380
])
381
self._attn_implementation = config._attn_implementation
@@ -392,11 +401,12 @@ class ArcticForCausalLM(nn.Module):
401
402
def __init__(self,
403
config: ArcticConfig,
404
+ cache_config: Optional[CacheConfig] = None,
405
quant_config: Optional[QuantizationConfig] = None,
406
**kwargs) -> None:
407
super().__init__()
408
self.config = config
399
- self.model = ArcticModel(config, quant_config)
409
+ self.model = ArcticModel(config, cache_config, quant_config)
410
self.vocab_size = config.vocab_size
411
self.lm_head = ParallelLMHead(
412
self.vocab_size,
vllm/model_executor/models/baichuan.py
+22
-7
@@ -26,7 +26,7 @@ from torch import nn
26
from transformers import PretrainedConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
-from vllm.config import LoRAConfig
29
+from vllm.config import CacheConfig, LoRAConfig
30
from vllm.distributed import (get_tensor_model_parallel_rank,
31
get_tensor_model_parallel_world_size)
32
from vllm.model_executor.layers.activation import SiluAndMul
@@ -111,6 +111,7 @@ class BaiChuanAttention(nn.Module):
111
position_embedding: str,
112
rope_theta: float = 10000,
113
max_position_embeddings: int = 8192,
114
+ cache_config: Optional[CacheConfig] = None,
115
quant_config: Optional[QuantizationConfig] = None,
116
):
117
super().__init__()
@@ -162,7 +163,10 @@ class BaiChuanAttention(nn.Module):
163
base=self.rope_theta,
164
)
165
self.scaling = self.head_dim**-0.5
165
- self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
166
+ self.attn = Attention(self.num_heads,
167
+ self.head_dim,
168
+ self.scaling,
169
+ cache_config=cache_config)
170
171
def forward(
172
self,
@@ -185,6 +189,7 @@ class BaiChuanDecoderLayer(nn.Module):
189
def __init__(self,
190
config: PretrainedConfig,
191
position_embedding: str,
192
+ cache_config: Optional[CacheConfig] = None,
193
quant_config: Optional[QuantizationConfig] = None):
194
super().__init__()
195
self.hidden_size = config.hidden_size
@@ -197,6 +202,7 @@ class BaiChuanDecoderLayer(nn.Module):
202
position_embedding=position_embedding,
203
rope_theta=rope_theta,
204
max_position_embeddings=max_position_embeddings,
205
+ cache_config=cache_config,
206
quant_config=quant_config,
207
)
208
self.mlp = BaiChuanMLP(
@@ -244,6 +250,7 @@ class BaiChuanModel(nn.Module):
250
def __init__(self,
251
config: PretrainedConfig,
252
position_embedding: str,
253
+ cache_config: Optional[CacheConfig] = None,
254
quant_config: Optional[QuantizationConfig] = None):
255
super().__init__()
256
self.config = config
@@ -255,7 +262,8 @@ class BaiChuanModel(nn.Module):
262
config.hidden_size,
263
)
264
self.layers = nn.ModuleList([
258
- BaiChuanDecoderLayer(config, position_embedding, quant_config)
265
+ BaiChuanDecoderLayer(config, position_embedding, cache_config,
266
+ quant_config)
267
for _ in range(config.num_hidden_layers)
268
])
269
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -304,13 +312,15 @@ class BaiChuanBaseForCausalLM(nn.Module):
312
self,
313
config,
314
position_embedding: str,
315
+ cache_config: Optional[CacheConfig] = None,
316
quant_config: Optional[QuantizationConfig] = None,
317
lora_config: Optional[LoRAConfig] = None,
318
):
319
super().__init__()
320
self.config = config
321
self.quant_config = quant_config
313
- self.model = BaiChuanModel(config, position_embedding, quant_config)
322
+ self.model = BaiChuanModel(config, position_embedding, cache_config,
323
+ quant_config)
324
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
325
self.logits_processor = LogitsProcessor(config.vocab_size)
326
self.sampler = Sampler()
@@ -389,13 +399,16 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
399
def __init__(
400
self,
401
config,
402
+ cache_config: Optional[CacheConfig] = None,
403
quant_config: Optional[QuantizationConfig] = None,
404
lora_config: Optional[LoRAConfig] = None,
405
):
406
if config.hidden_size == 4096: # baichuan2 7b
396
- super().__init__(config, "ROPE", quant_config, lora_config)
407
+ super().__init__(config, "ROPE", cache_config, quant_config,
408
+ lora_config)
409
else: # baichuan 13b, baichuan2 13b
398
- super().__init__(config, "ALIBI", quant_config, lora_config)
410
+ super().__init__(config, "ALIBI", cache_config, quant_config,
411
+ lora_config)
412
413
414
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -404,7 +417,9 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
417
def __init__(
418
self,
419
config,
420
+ cache_config: Optional[CacheConfig] = None,
421
quant_config: Optional[QuantizationConfig] = None,
422
lora_config: Optional[LoRAConfig] = None,
423
):
410
- super().__init__(config, "ROPE", quant_config, lora_config)
424
+ super().__init__(config, "ROPE", cache_config, quant_config,
425
+ lora_config)
vllm/model_executor/models/bloom.py
+11
-4
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import BloomConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.config import CacheConfig
28
from vllm.distributed import (get_tensor_model_parallel_rank,
29
get_tensor_model_parallel_world_size)
30
from vllm.model_executor.layers.activation import get_act_fn
@@ -71,6 +72,7 @@ class BloomAttention(nn.Module):
72
def __init__(
73
self,
74
config: BloomConfig,
75
+ cache_config: Optional[CacheConfig] = None,
76
quant_config: Optional[QuantizationConfig] = None,
77
):
78
super().__init__()
@@ -108,7 +110,8 @@ class BloomAttention(nn.Module):
110
self.attn = Attention(self.num_heads,
111
self.head_dim,
112
scaling,
111
- alibi_slopes=alibi_slopes)
113
+ alibi_slopes=alibi_slopes,
114
+ cache_config=cache_config)
115
116
def forward(
117
self,
@@ -158,6 +161,7 @@ class BloomBlock(nn.Module):
161
def __init__(
162
self,
163
config: BloomConfig,
164
+ cache_config: Optional[CacheConfig] = None,
165
quant_config: Optional[QuantizationConfig] = None,
166
):
167
super().__init__()
@@ -165,7 +169,8 @@ class BloomBlock(nn.Module):
169
170
self.input_layernorm = nn.LayerNorm(hidden_size,
171
eps=config.layer_norm_epsilon)
168
- self.self_attention = BloomAttention(config, quant_config)
172
+ self.self_attention = BloomAttention(config, cache_config,
173
+ quant_config)
174
self.post_attention_layernorm = nn.LayerNorm(
175
hidden_size, eps=config.layer_norm_epsilon)
176
self.mlp = BloomMLP(config, quant_config)
@@ -214,6 +219,7 @@ class BloomModel(nn.Module):
219
def __init__(
220
self,
221
config: BloomConfig,
222
+ cache_config: Optional[CacheConfig] = None,
223
quant_config: Optional[QuantizationConfig] = None,
224
):
225
super().__init__()
@@ -229,7 +235,7 @@ class BloomModel(nn.Module):
235
236
# Transformer blocks
237
self.h = nn.ModuleList([
232
- BloomBlock(config, quant_config)
238
+ BloomBlock(config, cache_config, quant_config)
239
for _ in range(config.num_hidden_layers)
240
])
241
@@ -262,12 +268,13 @@ class BloomForCausalLM(nn.Module):
268
def __init__(
269
self,
270
config: BloomConfig,
271
+ cache_config: Optional[CacheConfig] = None,
272
quant_config: Optional[QuantizationConfig] = None,
273
):
274
super().__init__()
275
self.config = config
276
self.quant_config = quant_config
270
- self.transformer = BloomModel(config, quant_config)
277
+ self.transformer = BloomModel(config, cache_config, quant_config)
278
self.lm_head_weight = self.transformer.word_embeddings.weight
279
self.logits_processor = LogitsProcessor(config.vocab_size)
280
self.sampler = Sampler()
vllm/model_executor/models/chatglm.py
+14
-6
@@ -9,7 +9,7 @@ from torch import nn
9
from torch.nn import LayerNorm
10
11
from vllm.attention import Attention, AttentionMetadata
12
-from vllm.config import LoRAConfig
12
+from vllm.config import CacheConfig, LoRAConfig
13
from vllm.distributed import get_tensor_model_parallel_world_size
14
from vllm.model_executor.layers.activation import SiluAndMul
15
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -34,6 +34,7 @@ class GLMAttention(nn.Module):
34
def __init__(
35
self,
36
config,
37
+ cache_config: Optional[CacheConfig] = None,
38
quant_config: Optional[QuantizationConfig] = None,
39
):
40
super().__init__()
@@ -90,6 +91,7 @@ class GLMAttention(nn.Module):
91
self.head_dim,
92
self.scaling,
93
num_kv_heads=self.num_kv_heads,
94
+ cache_config=cache_config,
95
)
96
97
def forward(
@@ -167,6 +169,7 @@ class GLMBlock(nn.Module):
169
def __init__(
170
self,
171
config,
172
+ cache_config: Optional[CacheConfig] = None,
173
quant_config: Optional[QuantizationConfig] = None,
174
):
175
super().__init__()
@@ -181,7 +184,7 @@ class GLMBlock(nn.Module):
184
eps=config.layernorm_epsilon)
185
186
# Self attention.
184
- self.self_attention = GLMAttention(config, quant_config)
187
+ self.self_attention = GLMAttention(config, cache_config, quant_config)
188
self.hidden_dropout = config.hidden_dropout
189
190
# Layernorm on the attention output
@@ -237,6 +240,7 @@ class GLMTransformer(nn.Module):
240
def __init__(
241
self,
242
config,
243
+ cache_config: Optional[CacheConfig] = None,
244
quant_config: Optional[QuantizationConfig] = None,
245
):
246
super().__init__()
@@ -246,8 +250,10 @@ class GLMTransformer(nn.Module):
250
self.num_layers = config.num_layers
251
252
# Transformer layers.
249
- self.layers = nn.ModuleList(
250
- [GLMBlock(config, quant_config) for i in range(self.num_layers)])
253
+ self.layers = nn.ModuleList([
254
+ GLMBlock(config, cache_config, quant_config)
255
+ for i in range(self.num_layers)
256
+ ])
257
258
if self.post_layer_norm:
259
layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -282,6 +288,7 @@ class ChatGLMModel(nn.Module):
288
def __init__(
289
self,
290
config,
291
+ cache_config: Optional[CacheConfig] = None,
292
quant_config: Optional[QuantizationConfig] = None,
293
):
294
super().__init__()
@@ -292,7 +299,7 @@ class ChatGLMModel(nn.Module):
299
self.num_layers = config.num_layers
300
self.multi_query_group_num = config.multi_query_group_num
301
self.kv_channels = config.kv_channels
295
- self.encoder = GLMTransformer(config, quant_config)
302
+ self.encoder = GLMTransformer(config, cache_config, quant_config)
303
304
self.output_layer = ParallelLMHead(config.padded_vocab_size,
305
config.hidden_size)
@@ -334,13 +341,14 @@ class ChatGLMForCausalLM(nn.Module):
341
def __init__(
342
self,
343
config: ChatGLMConfig,
344
+ cache_config: Optional[CacheConfig] = None,
345
quant_config: Optional[QuantizationConfig] = None,
346
lora_config: Optional[LoRAConfig] = None,
347
):
348
super().__init__()
349
self.config: ChatGLMConfig = config
350
self.quant_config = quant_config
343
- self.transformer = ChatGLMModel(config, quant_config)
351
+ self.transformer = ChatGLMModel(config, cache_config, quant_config)
352
self.lm_head_weight = self.transformer.output_layer.weight
353
self.logits_processor = LogitsProcessor(config.padded_vocab_size)
354
self.sampler = Sampler()
vllm/model_executor/models/commandr.py
+11
-3
@@ -29,6 +29,7 @@ from torch.nn.parameter import Parameter
29
from transformers import CohereConfig
30
31
from vllm.attention import Attention, AttentionMetadata
32
+from vllm.config import CacheConfig
33
from vllm.distributed import (get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -124,6 +125,7 @@ class CohereAttention(nn.Module):
125
def __init__(
126
self,
127
config: CohereConfig,
128
+ cache_config: Optional[CacheConfig] = None,
129
quant_config: Optional[QuantizationConfig] = None,
130
):
131
super().__init__()
@@ -180,6 +182,7 @@ class CohereAttention(nn.Module):
182
self.head_dim,
183
self.scaling,
184
num_kv_heads=self.num_kv_heads,
185
+ cache_config=cache_config,
186
)
187
if self.use_qk_norm:
188
self.q_norm = LayerNorm(param_shape=(self.num_heads,
@@ -219,11 +222,14 @@ class CohereDecoderLayer(nn.Module):
222
223
def __init__(self,
224
config: CohereConfig,
225
+ cache_config: Optional[CacheConfig] = None,
226
quant_config: Optional[QuantizationConfig] = None):
227
super().__init__()
228
self.hidden_size = config.hidden_size
229
226
- self.self_attn = CohereAttention(config, quant_config=quant_config)
230
+ self.self_attn = CohereAttention(config,
231
+ cache_config,
232
+ quant_config=quant_config)
233
234
self.mlp = CohereMLP(config, quant_config=quant_config)
235
self.input_layernorm = LayerNorm(param_shape=(config.hidden_size),
@@ -258,6 +264,7 @@ class CohereModel(nn.Module):
264
def __init__(
265
self,
266
config: CohereConfig,
267
+ cache_config: Optional[CacheConfig] = None,
268
quant_config: Optional[QuantizationConfig] = None,
269
):
270
super().__init__()
@@ -266,7 +273,7 @@ class CohereModel(nn.Module):
273
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
274
config.hidden_size)
275
self.layers = nn.ModuleList([
269
- CohereDecoderLayer(config, quant_config=quant_config)
276
+ CohereDecoderLayer(config, cache_config, quant_config=quant_config)
277
for _ in range(config.num_hidden_layers)
278
])
279
self.norm = LayerNorm(param_shape=(config.hidden_size),
@@ -299,6 +306,7 @@ class CohereForCausalLM(nn.Module):
306
def __init__(
307
self,
308
config: CohereConfig,
309
+ cache_config: Optional[CacheConfig] = None,
310
quant_config: Optional[QuantizationConfig] = None,
311
) -> None:
312
super().__init__()
@@ -306,7 +314,7 @@ class CohereForCausalLM(nn.Module):
314
self.quant_config = quant_config
315
self.logits_processor = LogitsProcessor(config.vocab_size,
316
scale=config.logit_scale)
309
- self.model = CohereModel(config, quant_config)
317
+ self.model = CohereModel(config, cache_config, quant_config)
318
self.sampler = Sampler()
319
320
@torch.no_grad()
vllm/model_executor/models/dbrx.py
+13
-4
@@ -5,6 +5,7 @@ import torch
5
import torch.nn as nn
6
7
from vllm.attention import Attention, AttentionMetadata
8
+from vllm.config import CacheConfig
9
from vllm.distributed import (get_tensor_model_parallel_rank,
10
get_tensor_model_parallel_world_size,
11
tensor_model_parallel_all_reduce)
@@ -166,6 +167,7 @@ class DbrxAttention(nn.Module):
167
def __init__(
168
self,
169
config: DbrxConfig,
170
+ cache_config: Optional[CacheConfig] = None,
171
quant_config: Optional[QuantizationConfig] = None,
172
):
173
super().__init__()
@@ -221,6 +223,7 @@ class DbrxAttention(nn.Module):
223
self.head_dim,
224
self.scaling,
225
num_kv_heads=self.num_kv_heads,
226
+ cache_config=cache_config,
227
)
228
229
def forward(
@@ -279,10 +282,12 @@ class DbrxBlock(nn.Module):
282
def __init__(
283
self,
284
config: DbrxConfig,
285
+ cache_config: Optional[CacheConfig] = None,
286
quant_config: Optional[QuantizationConfig] = None,
287
):
288
super().__init__()
285
- self.norm_attn_norm = DbrxFusedNormAttention(config, quant_config)
289
+ self.norm_attn_norm = DbrxFusedNormAttention(config, cache_config,
290
+ quant_config)
291
self.ffn = DbrxExperts(config, quant_config)
292
293
def forward(
@@ -308,6 +313,7 @@ class DbrxModel(nn.Module):
313
def __init__(
314
self,
315
config: DbrxConfig,
316
+ cache_config: Optional[CacheConfig] = None,
317
quant_config: Optional[QuantizationConfig] = None,
318
):
319
super().__init__()
@@ -315,8 +321,10 @@ class DbrxModel(nn.Module):
321
config.vocab_size,
322
config.d_model,
323
)
318
- self.blocks = nn.ModuleList(
319
- [DbrxBlock(config, quant_config) for _ in range(config.n_layers)])
324
+ self.blocks = nn.ModuleList([
325
+ DbrxBlock(config, cache_config, quant_config)
326
+ for _ in range(config.n_layers)
327
+ ])
328
self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
329
for module in self.modules():
330
if hasattr(module, "bias") and isinstance(module.bias,
@@ -349,13 +357,14 @@ class DbrxForCausalLM(nn.Module):
357
def __init__(
358
self,
359
config: DbrxConfig,
360
+ cache_config: Optional[CacheConfig] = None,
361
quant_config: Optional[QuantizationConfig] = None,
362
):
363
super().__init__()
364
self.config = config
365
self.quant_config = quant_config
366
self.unpadded_vocab_size = config.vocab_size
358
- self.transformer = DbrxModel(config, quant_config)
367
+ self.transformer = DbrxModel(config, cache_config, quant_config)
368
self.lm_head = ParallelLMHead(
369
config.vocab_size,
370
config.d_model,
vllm/model_executor/models/decilm.py
+3
-1
@@ -28,7 +28,7 @@ from typing import Iterable, Optional, Tuple
28
import torch
29
from transformers import PretrainedConfig
30
31
-from vllm.config import LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig
32
from vllm.model_executor.layers.quantization.base_config import (
33
QuantizationConfig)
34
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -56,12 +56,14 @@ class DeciLMForCausalLM(LlamaForCausalLM):
56
def __init__(
57
self,
58
config: Optional[PretrainedConfig] = None,
59
+ cache_config: Optional[CacheConfig] = None,
60
quant_config: Optional[QuantizationConfig] = None,
61
lora_config: Optional[LoRAConfig] = None,
62
) -> None:
63
config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
64
delattr(config, "num_key_value_heads_per_layer")
65
super().__init__(config=config,
66
+ cache_config=cache_config,
67
quant_config=quant_config,
68
lora_config=lora_config)
69
vllm/model_executor/models/deepseek.py
+13
-3
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig
32
from vllm.distributed import (get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size,
34
tensor_model_parallel_all_reduce)
@@ -178,6 +179,7 @@ class DeepseekAttention(nn.Module):
179
rope_theta: float = 10000,
180
rope_scaling: Optional[Dict[str, Any]] = None,
181
max_position_embeddings: int = 8192,
182
+ cache_config: Optional[CacheConfig] = None,
183
quant_config: Optional[QuantizationConfig] = None,
184
) -> None:
185
super().__init__()
@@ -229,7 +231,8 @@ class DeepseekAttention(nn.Module):
231
self.attn = Attention(self.num_heads,
232
self.head_dim,
233
self.scaling,
232
- num_kv_heads=self.num_kv_heads)
234
+ num_kv_heads=self.num_kv_heads,
235
+ cache_config=cache_config)
236
237
def forward(
238
self,
@@ -252,6 +255,7 @@ class DeepseekDecoderLayer(nn.Module):
255
self,
256
config: PretrainedConfig,
257
layer_idx: int,
258
+ cache_config: Optional[CacheConfig] = None,
259
quant_config: Optional[QuantizationConfig] = None,
260
) -> None:
261
super().__init__()
@@ -267,6 +271,7 @@ class DeepseekDecoderLayer(nn.Module):
271
rope_theta=rope_theta,
272
rope_scaling=rope_scaling,
273
max_position_embeddings=max_position_embeddings,
274
+ cache_config=cache_config,
275
quant_config=quant_config,
276
)
277
if (config.n_routed_experts is not None
@@ -321,6 +326,7 @@ class DeepseekModel(nn.Module):
326
def __init__(
327
self,
328
config: PretrainedConfig,
329
+ cache_config: Optional[CacheConfig] = None,
330
quant_config: Optional[QuantizationConfig] = None,
331
) -> None:
332
super().__init__()
@@ -332,7 +338,10 @@ class DeepseekModel(nn.Module):
338
config.hidden_size,
339
)
340
self.layers = nn.ModuleList([
335
- DeepseekDecoderLayer(config, layer_idx, quant_config=quant_config)
341
+ DeepseekDecoderLayer(config,
342
+ layer_idx,
343
+ cache_config,
344
+ quant_config=quant_config)
345
for layer_idx in range(config.num_hidden_layers)
346
])
347
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -360,12 +369,13 @@ class DeepseekForCausalLM(nn.Module):
369
def __init__(
370
self,
371
config: PretrainedConfig,
372
+ cache_config: Optional[CacheConfig] = None,
373
quant_config: Optional[QuantizationConfig] = None,
374
) -> None:
375
super().__init__()
376
self.config = config
377
self.quant_config = quant_config
368
- self.model = DeepseekModel(config, quant_config)
378
+ self.model = DeepseekModel(config, cache_config, quant_config)
379
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
380
self.logits_processor = LogitsProcessor(config.vocab_size)
381
self.sampler = Sampler()
vllm/model_executor/models/falcon.py
+11
-4
@@ -27,6 +27,7 @@ from torch.nn import LayerNorm
27
from transformers import FalconConfig as HF_FalconConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
+from vllm.config import CacheConfig
31
from vllm.distributed import (get_tensor_model_parallel_rank,
32
get_tensor_model_parallel_world_size,
33
tensor_model_parallel_all_reduce)
@@ -77,6 +78,7 @@ class FalconAttention(nn.Module):
78
def __init__(
79
self,
80
config: FalconConfig,
81
+ cache_config: Optional[CacheConfig] = None,
82
quant_config: Optional[QuantizationConfig] = None,
83
):
84
super().__init__()
@@ -168,7 +170,8 @@ class FalconAttention(nn.Module):
170
self.attn = Attention(self.num_heads,
171
self.head_dim,
172
scale=self.inv_norm_factor,
171
- num_kv_heads=self.num_kv_heads)
173
+ num_kv_heads=self.num_kv_heads,
174
+ cache_config=cache_config)
175
176
def forward(
177
self,
@@ -229,12 +232,14 @@ class FalconDecoderLayer(nn.Module):
232
def __init__(
233
self,
234
config: FalconConfig,
235
+ cache_config: Optional[CacheConfig] = None,
236
quant_config: Optional[QuantizationConfig] = None,
237
):
238
super().__init__()
239
hidden_size = config.hidden_size
240
self.num_heads = config.num_attention_heads
237
- self.self_attention = FalconAttention(config, quant_config)
241
+ self.self_attention = FalconAttention(config, cache_config,
242
+ quant_config)
243
self.mlp = FalconMLP(config, quant_config)
244
self.config = config
245
@@ -311,6 +316,7 @@ class FalconModel(nn.Module):
316
def __init__(
317
self,
318
config: FalconConfig,
319
+ cache_config: Optional[CacheConfig] = None,
320
quant_config: Optional[QuantizationConfig] = None,
321
):
322
super().__init__()
@@ -327,7 +333,7 @@ class FalconModel(nn.Module):
333
334
# Transformer blocks
335
self.h = nn.ModuleList([
330
- FalconDecoderLayer(config, quant_config)
336
+ FalconDecoderLayer(config, cache_config, quant_config)
337
for _ in range(config.num_hidden_layers)
338
])
339
@@ -359,12 +365,13 @@ class FalconForCausalLM(nn.Module):
365
def __init__(
366
self,
367
config: FalconConfig,
368
+ cache_config: Optional[CacheConfig] = None,
369
quant_config: Optional[QuantizationConfig] = None,
370
):
371
super().__init__()
372
self.config = config
373
self.quant_config = quant_config
367
- self.transformer = FalconModel(config, quant_config)
374
+ self.transformer = FalconModel(config, cache_config, quant_config)
375
self.lm_head_weight = self.transformer.word_embeddings.weight
376
self.logits_processor = LogitsProcessor(config.vocab_size)
377
self.sampler = Sampler()
vllm/model_executor/models/gemma.py
+10
-4
@@ -22,7 +22,7 @@ from torch import nn
22
from transformers import GemmaConfig
23
24
from vllm.attention import Attention, AttentionMetadata
25
-from vllm.config import LoRAConfig
25
+from vllm.config import CacheConfig, LoRAConfig
26
from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.activation import GeluAndMul
@@ -107,6 +107,7 @@ class GemmaAttention(nn.Module):
107
head_dim: int,
108
max_position_embeddings: int = 8192,
109
rope_theta: float = 10000,
110
+ cache_config: Optional[CacheConfig] = None,
111
quant_config: Optional[QuantizationConfig] = None) -> None:
112
super().__init__()
113
self.hidden_size = hidden_size
@@ -155,7 +156,8 @@ class GemmaAttention(nn.Module):
156
self.attn = Attention(self.num_heads,
157
self.head_dim,
158
self.scaling,
158
- num_kv_heads=self.num_kv_heads)
159
+ num_kv_heads=self.num_kv_heads,
160
+ cache_config=cache_config)
161
162
def forward(
163
self,
@@ -177,6 +179,7 @@ class GemmaDecoderLayer(nn.Module):
179
def __init__(
180
self,
181
config: GemmaConfig,
182
+ cache_config: Optional[CacheConfig] = None,
183
quant_config: Optional[QuantizationConfig] = None,
184
) -> None:
185
super().__init__()
@@ -188,6 +191,7 @@ class GemmaDecoderLayer(nn.Module):
191
head_dim=config.head_dim,
192
max_position_embeddings=config.max_position_embeddings,
193
rope_theta=config.rope_theta,
194
+ cache_config=cache_config,
195
quant_config=quant_config,
196
)
197
self.mlp = GemmaMLP(
@@ -236,6 +240,7 @@ class GemmaModel(nn.Module):
240
def __init__(
241
self,
242
config: GemmaConfig,
243
+ cache_config: Optional[CacheConfig] = None,
244
quant_config: Optional[QuantizationConfig] = None,
245
) -> None:
246
super().__init__()
@@ -246,7 +251,7 @@ class GemmaModel(nn.Module):
251
config.hidden_size,
252
)
253
self.layers = nn.ModuleList([
249
- GemmaDecoderLayer(config, quant_config)
254
+ GemmaDecoderLayer(config, cache_config, quant_config)
255
for _ in range(config.num_hidden_layers)
256
])
257
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -309,6 +314,7 @@ class GemmaForCausalLM(nn.Module):
314
def __init__(
315
self,
316
config: GemmaConfig,
317
+ cache_config: Optional[CacheConfig] = None,
318
quant_config: Optional[QuantizationConfig] = None,
319
lora_config: Optional[LoRAConfig] = None,
320
) -> None:
@@ -316,7 +322,7 @@ class GemmaForCausalLM(nn.Module):
322
super().__init__()
323
self.config = config
324
self.quant_config = quant_config
319
- self.model = GemmaModel(config, quant_config)
325
+ self.model = GemmaModel(config, cache_config, quant_config)
326
self.logits_processor = LogitsProcessor(config.vocab_size)
327
self.sampler = Sampler()
328
vllm/model_executor/models/gpt2.py
+12
-4
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import GPT2Config
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.config import CacheConfig
28
from vllm.distributed import get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -45,6 +46,7 @@ class GPT2Attention(nn.Module):
46
def __init__(
47
self,
48
config: GPT2Config,
49
+ cache_config: Optional[CacheConfig] = None,
50
quant_config: Optional[QuantizationConfig] = None,
51
):
52
super().__init__()
@@ -70,7 +72,10 @@ class GPT2Attention(nn.Module):
72
bias=True,
73
quant_config=quant_config,
74
)
73
- self.attn = Attention(self.num_heads, self.head_dim, scale=self.scale)
75
+ self.attn = Attention(self.num_heads,
76
+ self.head_dim,
77
+ scale=self.scale,
78
+ cache_config=cache_config)
79
80
def forward(
81
self,
@@ -122,6 +127,7 @@ class GPT2Block(nn.Module):
127
def __init__(
128
self,
129
config: GPT2Config,
130
+ cache_config: Optional[CacheConfig] = None,
131
quant_config: Optional[QuantizationConfig] = None,
132
):
133
super().__init__()
@@ -130,7 +136,7 @@ class GPT2Block(nn.Module):
136
hidden_size)
137
138
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
133
- self.attn = GPT2Attention(config, quant_config)
139
+ self.attn = GPT2Attention(config, cache_config, quant_config)
140
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
141
self.mlp = GPT2MLP(inner_dim, config, quant_config)
142
@@ -163,6 +169,7 @@ class GPT2Model(nn.Module):
169
def __init__(
170
self,
171
config: GPT2Config,
172
+ cache_config: Optional[CacheConfig] = None,
173
quant_config: Optional[QuantizationConfig] = None,
174
):
175
super().__init__()
@@ -174,7 +181,7 @@ class GPT2Model(nn.Module):
181
self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
182
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
183
self.h = nn.ModuleList([
177
- GPT2Block(config, quant_config)
184
+ GPT2Block(config, cache_config, quant_config)
185
for _ in range(config.num_hidden_layers)
186
])
187
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -203,12 +210,13 @@ class GPT2LMHeadModel(nn.Module):
210
def __init__(
211
self,
212
config: GPT2Config,
213
+ cache_config: Optional[CacheConfig] = None,
214
quant_config: Optional[QuantizationConfig] = None,
215
):
216
super().__init__()
217
self.config = config
218
self.quant_config = quant_config
211
- self.transformer = GPT2Model(config, quant_config)
219
+ self.transformer = GPT2Model(config, cache_config, quant_config)
220
self.lm_head_weight = self.transformer.wte.weight
221
self.logits_processor = LogitsProcessor(config.vocab_size)
222
self.sampler = Sampler()
vllm/model_executor/models/gpt_bigcode.py
+10
-4
@@ -25,6 +25,7 @@ from torch import nn
25
from transformers import GPTBigCodeConfig
26
27
from vllm.attention import Attention, AttentionMetadata
28
+from vllm.config import CacheConfig
29
from vllm.distributed import get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -46,6 +47,7 @@ class GPTBigCodeAttention(nn.Module):
47
def __init__(
48
self,
49
config: GPTBigCodeConfig,
50
+ cache_config: Optional[CacheConfig] = None,
51
quant_config: Optional[QuantizationConfig] = None,
52
):
53
super().__init__()
@@ -85,7 +87,8 @@ class GPTBigCodeAttention(nn.Module):
87
self.attn = Attention(self.num_heads,
88
self.head_dim,
89
scale=self.scale,
88
- num_kv_heads=self.num_kv_heads)
90
+ num_kv_heads=self.num_kv_heads,
91
+ cache_config=cache_config)
92
93
def forward(
94
self,
@@ -143,6 +146,7 @@ class GPTBigCodeBlock(nn.Module):
146
def __init__(
147
self,
148
config: GPTBigCodeConfig,
149
+ cache_config: Optional[CacheConfig] = None,
150
quant_config: Optional[QuantizationConfig] = None,
151
):
152
super().__init__()
@@ -151,7 +155,7 @@ class GPTBigCodeBlock(nn.Module):
155
hidden_size)
156
157
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
154
- self.attn = GPTBigCodeAttention(config, quant_config)
158
+ self.attn = GPTBigCodeAttention(config, cache_config, quant_config)
159
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
160
self.mlp = GPTBigMLP(inner_dim, config, quant_config)
161
@@ -184,6 +188,7 @@ class GPTBigCodeModel(nn.Module):
188
def __init__(
189
self,
190
config: GPTBigCodeConfig,
191
+ cache_config: Optional[CacheConfig] = None,
192
quant_config: Optional[QuantizationConfig] = None,
193
):
194
super().__init__()
@@ -195,7 +200,7 @@ class GPTBigCodeModel(nn.Module):
200
self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
201
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
202
self.h = nn.ModuleList([
198
- GPTBigCodeBlock(config, quant_config)
203
+ GPTBigCodeBlock(config, cache_config, quant_config)
204
for _ in range(config.num_hidden_layers)
205
])
206
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -224,12 +229,13 @@ class GPTBigCodeForCausalLM(nn.Module):
229
def __init__(
230
self,
231
config: GPTBigCodeConfig,
232
+ cache_config: Optional[CacheConfig] = None,
233
quant_config: Optional[QuantizationConfig] = None,
234
):
235
super().__init__()
236
self.config = config
237
self.quant_config = quant_config
232
- self.transformer = GPTBigCodeModel(config, quant_config)
238
+ self.transformer = GPTBigCodeModel(config, cache_config, quant_config)
239
self.lm_head_weight = self.transformer.wte.weight
240
self.logits_processor = LogitsProcessor(config.vocab_size)
241
self.sampler = Sampler()
vllm/model_executor/models/gpt_j.py
+15
-5
@@ -23,6 +23,7 @@ from torch import nn
23
from transformers import GPTJConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
+from vllm.config import CacheConfig
27
from vllm.distributed import get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -45,6 +46,7 @@ class GPTJAttention(nn.Module):
46
def __init__(
47
self,
48
config: GPTJConfig,
49
+ cache_config: Optional[CacheConfig] = None,
50
quant_config: Optional[QuantizationConfig] = None,
51
):
52
super().__init__()
@@ -83,7 +85,10 @@ class GPTJAttention(nn.Module):
85
base=rope_theta,
86
is_neox_style=False,
87
)
86
- self.attn = Attention(self.num_heads, self.head_size, scaling)
88
+ self.attn = Attention(self.num_heads,
89
+ self.head_size,
90
+ scaling,
91
+ cache_config=cache_config)
92
93
def forward(
94
self,
@@ -135,13 +140,14 @@ class GPTJBlock(nn.Module):
140
def __init__(
141
self,
142
config: GPTJConfig,
143
+ cache_config: Optional[CacheConfig] = None,
144
quant_config: Optional[QuantizationConfig] = None,
145
):
146
super().__init__()
147
inner_dim = (4 * config.n_embd
148
if config.n_inner is None else config.n_inner)
149
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
144
- self.attn = GPTJAttention(config, quant_config)
150
+ self.attn = GPTJAttention(config, cache_config, quant_config)
151
self.mlp = GPTJMLP(inner_dim, config, quant_config)
152
153
def forward(
@@ -169,6 +175,7 @@ class GPTJModel(nn.Module):
175
def __init__(
176
self,
177
config: GPTJConfig,
178
+ cache_config: Optional[CacheConfig] = None,
179
quant_config: Optional[QuantizationConfig] = None,
180
):
181
super().__init__()
@@ -178,8 +185,10 @@ class GPTJModel(nn.Module):
185
config.vocab_size,
186
self.embed_dim,
187
)
181
- self.h = nn.ModuleList(
182
- [GPTJBlock(config, quant_config) for _ in range(config.n_layer)])
188
+ self.h = nn.ModuleList([
189
+ GPTJBlock(config, cache_config, quant_config)
190
+ for _ in range(config.n_layer)
191
+ ])
192
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
193
194
def forward(
@@ -207,13 +216,14 @@ class GPTJForCausalLM(nn.Module):
216
def __init__(
217
self,
218
config: GPTJConfig,
219
+ cache_config: Optional[CacheConfig] = None,
220
quant_config: Optional[QuantizationConfig] = None,
221
):
222
super().__init__()
223
self.config = config
224
self.quant_config = quant_config
225
assert not config.tie_word_embeddings
216
- self.transformer = GPTJModel(config, quant_config)
226
+ self.transformer = GPTJModel(config, cache_config, quant_config)
227
self.lm_head = ParallelLMHead(
228
config.vocab_size,
229
config.n_embd,
vllm/model_executor/models/gpt_neox.py
+12
-4
@@ -23,6 +23,7 @@ from torch import nn
23
from transformers import GPTNeoXConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
+from vllm.config import CacheConfig
27
from vllm.distributed import get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -45,6 +46,7 @@ class GPTNeoXAttention(nn.Module):
46
def __init__(
47
self,
48
config: GPTNeoXConfig,
49
+ cache_config: Optional[CacheConfig] = None,
50
quant_config: Optional[QuantizationConfig] = None,
51
):
52
super().__init__()
@@ -84,7 +86,10 @@ class GPTNeoXAttention(nn.Module):
86
max_position=max_position_embeddings,
87
base=rope_theta,
88
)
87
- self.attn = Attention(self.num_heads, self.head_size, scaling)
89
+ self.attn = Attention(self.num_heads,
90
+ self.head_size,
91
+ scaling,
92
+ cache_config=cache_config)
93
94
def forward(
95
self,
@@ -134,6 +139,7 @@ class GPTNeoXLayer(nn.Module):
139
def __init__(
140
self,
141
config: GPTNeoXConfig,
142
+ cache_config: Optional[CacheConfig] = None,
143
quant_config: Optional[QuantizationConfig] = None,
144
):
145
super().__init__()
@@ -142,7 +148,7 @@ class GPTNeoXLayer(nn.Module):
148
eps=config.layer_norm_eps)
149
self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
150
eps=config.layer_norm_eps)
145
- self.attention = GPTNeoXAttention(config, quant_config)
151
+ self.attention = GPTNeoXAttention(config, cache_config, quant_config)
152
self.mlp = GPTNeoXMLP(config, quant_config)
153
154
def forward(
@@ -182,6 +188,7 @@ class GPTNeoXModel(nn.Module):
188
def __init__(
189
self,
190
config: GPTNeoXConfig,
191
+ cache_config: Optional[CacheConfig] = None,
192
quant_config: Optional[QuantizationConfig] = None,
193
):
194
super().__init__()
@@ -192,7 +199,7 @@ class GPTNeoXModel(nn.Module):
199
config.hidden_size,
200
)
201
self.layers = nn.ModuleList([
195
- GPTNeoXLayer(config, quant_config)
202
+ GPTNeoXLayer(config, cache_config, quant_config)
203
for _ in range(config.num_hidden_layers)
204
])
205
self.final_layer_norm = nn.LayerNorm(config.hidden_size,
@@ -223,12 +230,13 @@ class GPTNeoXForCausalLM(nn.Module):
230
def __init__(
231
self,
232
config,
233
+ cache_config: Optional[CacheConfig] = None,
234
quant_config: Optional[QuantizationConfig] = None,
235
):
236
super().__init__()
237
self.config = config
238
self.quant_config = quant_config
231
- self.gpt_neox = GPTNeoXModel(config, quant_config)
239
+ self.gpt_neox = GPTNeoXModel(config, cache_config, quant_config)
240
self.embed_out = ParallelLMHead(
241
config.vocab_size,
242
config.hidden_size,
vllm/model_executor/models/internlm2.py
+10
-3
@@ -6,6 +6,7 @@ from torch import nn
6
from transformers import PretrainedConfig
7
8
from vllm.attention import Attention, AttentionMetadata
9
+from vllm.config import CacheConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.model_executor.layers.activation import SiluAndMul
12
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -64,6 +65,7 @@ class InternLM2Attention(nn.Module):
65
rope_theta: float = 10000,
66
rope_scaling: Optional[Dict[str, Any]] = None,
67
max_position_embeddings: int = 8192,
68
+ cache_config: Optional[CacheConfig] = None,
69
quant_config: Optional[QuantizationConfig] = None,
70
) -> None:
71
super().__init__()
@@ -114,7 +116,8 @@ class InternLM2Attention(nn.Module):
116
self.attn = Attention(self.num_heads,
117
self.head_dim,
118
self.scaling,
117
- num_kv_heads=self.num_kv_heads)
119
+ num_kv_heads=self.num_kv_heads,
120
+ cache_config=cache_config)
121
122
def forward(
123
self,
@@ -136,6 +139,7 @@ class InternLMDecoderLayer(nn.Module):
139
def __init__(
140
self,
141
config: PretrainedConfig,
142
+ cache_config: Optional[CacheConfig] = None,
143
quant_config: Optional[QuantizationConfig] = None,
144
) -> None:
145
super().__init__()
@@ -151,6 +155,7 @@ class InternLMDecoderLayer(nn.Module):
155
rope_theta=rope_theta,
156
rope_scaling=rope_scaling,
157
max_position_embeddings=max_position_embeddings,
158
+ cache_config=cache_config,
159
quant_config=quant_config,
160
)
161
self.feed_forward = InternLM2MLP(
@@ -196,6 +201,7 @@ class InternLM2Model(nn.Module):
201
def __init__(
202
self,
203
config: PretrainedConfig,
204
+ cache_config: Optional[CacheConfig] = None,
205
quant_config: Optional[QuantizationConfig] = None,
206
) -> None:
207
super().__init__()
@@ -207,7 +213,7 @@ class InternLM2Model(nn.Module):
213
config.hidden_size,
214
)
215
self.layers = nn.ModuleList([
210
- InternLMDecoderLayer(config, quant_config)
216
+ InternLMDecoderLayer(config, cache_config, quant_config)
217
for _ in range(config.num_hidden_layers)
218
])
219
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -239,12 +245,13 @@ class InternLM2ForCausalLM(nn.Module):
245
def __init__(
246
self,
247
config: PretrainedConfig,
248
+ cache_config: Optional[CacheConfig] = None,
249
quant_config: Optional[QuantizationConfig] = None,
250
) -> None:
251
super().__init__()
252
self.config = config
253
self.quant_config = quant_config
247
- self.model = InternLM2Model(config, quant_config)
254
+ self.model = InternLM2Model(config, cache_config, quant_config)
255
self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
256
self.logits_processor = LogitsProcessor(config.vocab_size)
257
self.sampler = Sampler()
vllm/model_executor/models/jais.py
+9
-3
@@ -26,6 +26,7 @@ import torch
26
from torch import nn
27
28
from vllm.attention import Attention, AttentionMetadata
29
+from vllm.config import CacheConfig
30
from vllm.distributed import (get_tensor_model_parallel_rank,
31
get_tensor_model_parallel_world_size)
32
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -69,6 +70,7 @@ class JAISAttention(nn.Module):
70
def __init__(
71
self,
72
config: JAISConfig,
73
+ cache_config: Optional[CacheConfig] = None,
74
quant_config: Optional[QuantizationConfig] = None,
75
):
76
super().__init__()
@@ -108,6 +110,7 @@ class JAISAttention(nn.Module):
110
self.head_dim,
111
scale=self.scale,
112
alibi_slopes=alibi_slopes,
113
+ cache_config=cache_config,
114
)
115
116
def forward(
@@ -170,6 +173,7 @@ class JAISBlock(nn.Module):
173
def __init__(
174
self,
175
config: JAISConfig,
176
+ cache_config: Optional[CacheConfig] = None,
177
quant_config: Optional[QuantizationConfig] = None,
178
):
179
super().__init__()
@@ -178,7 +182,7 @@ class JAISBlock(nn.Module):
182
hidden_size)
183
184
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
181
- self.attn = JAISAttention(config, quant_config)
185
+ self.attn = JAISAttention(config, cache_config, quant_config)
186
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
187
self.mlp = JAISMLP(inner_dim, config, quant_config)
188
@@ -211,6 +215,7 @@ class JAISModel(nn.Module):
215
def __init__(
216
self,
217
config: JAISConfig,
218
+ cache_config: Optional[CacheConfig] = None,
219
quant_config: Optional[QuantizationConfig] = None,
220
):
221
super().__init__()
@@ -228,7 +233,7 @@ class JAISModel(nn.Module):
233
else:
234
self.embeddings_scale = config.mup_embeddings_scale
235
self.h = nn.ModuleList([
231
- JAISBlock(config, quant_config)
236
+ JAISBlock(config, cache_config, quant_config)
237
for _ in range(config.num_hidden_layers)
238
])
239
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -262,12 +267,13 @@ class JAISLMHeadModel(nn.Module):
267
def __init__(
268
self,
269
config: JAISConfig,
270
+ cache_config: Optional[CacheConfig] = None,
271
quant_config: Optional[QuantizationConfig] = None,
272
):
273
super().__init__()
274
self.config = config
275
self.quant_config = quant_config
270
- self.transformer = JAISModel(config, quant_config)
276
+ self.transformer = JAISModel(config, cache_config, quant_config)
277
self.lm_head_weight = self.transformer.wte.weight
278
if hasattr(config, "width_scale"):
279
self.output_logits_scale = config.width_scale
vllm/model_executor/models/llama.py
+13
-4
@@ -28,7 +28,7 @@ from torch import nn
28
from transformers import LlamaConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
-from vllm.config import LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig
32
from vllm.distributed import (get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
@@ -94,6 +94,7 @@ class LlamaAttention(nn.Module):
94
quant_config: Optional[QuantizationConfig] = None,
95
bias: bool = False,
96
sliding_window: Optional[int] = None,
97
+ cache_config: Optional[CacheConfig] = None,
98
) -> None:
99
super().__init__()
100
self.hidden_size = hidden_size
@@ -153,7 +154,8 @@ class LlamaAttention(nn.Module):
154
self.head_dim,
155
self.scaling,
156
num_kv_heads=self.num_kv_heads,
156
- sliding_window=sliding_window)
157
+ sliding_window=sliding_window,
158
+ cache_config=cache_config)
159
160
def forward(
161
self,
@@ -176,6 +178,7 @@ class LlamaDecoderLayer(nn.Module):
178
def __init__(
179
self,
180
config: LlamaConfig,
181
+ cache_config: Optional[CacheConfig] = None,
182
quant_config: Optional[QuantizationConfig] = None,
183
) -> None:
184
super().__init__()
@@ -204,6 +207,7 @@ class LlamaDecoderLayer(nn.Module):
207
quant_config=quant_config,
208
bias=attention_bias,
209
sliding_window=sliding_window,
210
+ cache_config=cache_config,
211
)
212
self.mlp = LlamaMLP(
213
hidden_size=self.hidden_size,
@@ -251,6 +255,7 @@ class LlamaModel(nn.Module):
255
def __init__(
256
self,
257
config: LlamaConfig,
258
+ cache_config: Optional[CacheConfig] = None,
259
quant_config: Optional[QuantizationConfig] = None,
260
lora_config: Optional[LoRAConfig] = None,
261
) -> None:
@@ -267,7 +272,7 @@ class LlamaModel(nn.Module):
272
org_num_embeddings=config.vocab_size,
273
)
274
self.layers = nn.ModuleList([
270
- LlamaDecoderLayer(config, quant_config)
275
+ LlamaDecoderLayer(config, cache_config, quant_config)
276
for _ in range(config.num_hidden_layers)
277
])
278
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -332,12 +337,16 @@ class LlamaForCausalLM(nn.Module):
337
def __init__(
338
self,
339
config: LlamaConfig,
340
+ cache_config: Optional[CacheConfig] = None,
341
quant_config: Optional[QuantizationConfig] = None,
342
lora_config: Optional[LoRAConfig] = None,
343
) -> None:
344
super().__init__()
345
self.config = config
340
- self.model = LlamaModel(config, quant_config, lora_config=lora_config)
346
+ self.model = LlamaModel(config,
347
+ cache_config,
348
+ quant_config,
349
+ lora_config=lora_config)
350
self.unpadded_vocab_size = config.vocab_size
351
if lora_config:
352
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/llava.py
+4
-2
@@ -7,7 +7,7 @@ from torch import nn
7
from transformers import CLIPVisionModel, LlavaConfig
8
9
from vllm.attention import AttentionMetadata
10
-from vllm.config import VisionLanguageConfig
10
+from vllm.config import CacheConfig, VisionLanguageConfig
11
from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
from vllm.model_executor.layers.quantization.base_config import (
@@ -62,6 +62,7 @@ class LlavaForConditionalGeneration(nn.Module):
62
def __init__(self,
63
config: "LlavaConfig",
64
vision_language_config: VisionLanguageConfig,
65
+ cache_config: Optional[CacheConfig] = None,
66
quant_config: Optional["QuantizationConfig"] = None) -> None:
67
super().__init__()
68
self.config = config
@@ -85,7 +86,8 @@ class LlavaForConditionalGeneration(nn.Module):
86
projector_hidden_act=config.projector_hidden_act)
87
88
self.quant_config = quant_config
88
- self.language_model = LlamaModel(config.text_config, quant_config)
89
+ self.language_model = LlamaModel(config.text_config, cache_config,
90
+ quant_config)
91
self.unpadded_vocab_size = config.text_config.vocab_size
92
self.lm_head = ParallelLMHead(
93
self.unpadded_vocab_size,
vllm/model_executor/models/minicpm.py
+10
-3
@@ -28,7 +28,7 @@ import torch
28
from torch import nn
29
30
from vllm.attention import Attention, AttentionMetadata
31
-from vllm.config import LoRAConfig
31
+from vllm.config import CacheConfig, LoRAConfig
32
from vllm.distributed import (get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size,
34
tensor_model_parallel_all_reduce)
@@ -181,6 +181,7 @@ class MiniCPMAttention(nn.Module):
181
rope_theta: float = 10000,
182
rope_scaling: Optional[Dict[str, Any]] = None,
183
max_position_embeddings: int = 8192,
184
+ cache_config: Optional[CacheConfig] = None,
185
quant_config: Optional[QuantizationConfig] = None,
186
) -> None:
187
super().__init__()
@@ -234,7 +235,8 @@ class MiniCPMAttention(nn.Module):
235
self.attn = Attention(self.num_heads,
236
self.head_dim,
237
self.scaling,
237
- num_kv_heads=self.num_kv_heads)
238
+ num_kv_heads=self.num_kv_heads,
239
+ cache_config=cache_config)
240
241
def forward(
242
self,
@@ -259,6 +261,7 @@ class MiniCPMDecoderLayer(nn.Module):
261
def __init__(
262
self,
263
config,
264
+ cache_config: Optional[CacheConfig] = None,
265
quant_config: Optional[QuantizationConfig] = None,
266
) -> None:
267
super().__init__()
@@ -275,6 +278,7 @@ class MiniCPMDecoderLayer(nn.Module):
278
rope_theta=rope_theta,
279
rope_scaling=rope_scaling,
280
max_position_embeddings=max_position_embeddings,
281
+ cache_config=cache_config,
282
quant_config=quant_config,
283
)
284
self.num_experts = getattr(self.config, "num_experts", 0)
@@ -330,6 +334,7 @@ class MiniCPMModel(nn.Module):
334
def __init__(
335
self,
336
config,
337
+ cache_config: Optional[CacheConfig] = None,
338
quant_config: Optional[QuantizationConfig] = None,
339
lora_config: Optional[LoRAConfig] = None,
340
) -> None:
@@ -346,7 +351,7 @@ class MiniCPMModel(nn.Module):
351
org_num_embeddings=config.vocab_size,
352
)
353
self.layers = nn.ModuleList([
349
- MiniCPMDecoderLayer(config, quant_config)
354
+ MiniCPMDecoderLayer(config, cache_config, quant_config)
355
for _ in range(config.num_hidden_layers)
356
])
357
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -413,6 +418,7 @@ class MiniCPMForCausalLM(nn.Module):
418
def __init__(
419
self,
420
config,
421
+ cache_config: Optional[CacheConfig] = None,
422
quant_config: Optional[QuantizationConfig] = None,
423
lora_config: Optional[LoRAConfig] = None,
424
) -> None:
@@ -421,6 +427,7 @@ class MiniCPMForCausalLM(nn.Module):
427
self.num_experts = getattr(self.config, "num_experts", 0)
428
self.quant_config = quant_config
429
self.model = MiniCPMModel(config,
430
+ cache_config,
431
quant_config,
432
lora_config=lora_config)
433
unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/mixtral.py
+11
-2
@@ -29,7 +29,7 @@ from transformers import MixtralConfig
29
30
from vllm import _custom_ops as ops
31
from vllm.attention import Attention, AttentionMetadata
32
-from vllm.config import LoRAConfig
32
+from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import (get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size,
35
tensor_model_parallel_all_reduce)
@@ -252,6 +252,7 @@ class MixtralAttention(nn.Module):
252
num_kv_heads: int,
253
max_position: int = 4096 * 32,
254
rope_theta: float = 10000,
255
+ cache_config: Optional[CacheConfig] = None,
256
quant_config: Optional[QuantizationConfig] = None,
257
sliding_window: Optional[int] = None) -> None:
258
super().__init__()
@@ -313,6 +314,7 @@ class MixtralAttention(nn.Module):
314
self.scaling,
315
num_kv_heads=self.num_kv_heads,
316
sliding_window=self.sliding_window,
317
+ cache_config=cache_config,
318
)
319
320
def forward(
@@ -335,6 +337,7 @@ class MixtralDecoderLayer(nn.Module):
337
def __init__(
338
self,
339
config: MixtralConfig,
340
+ cache_config: Optional[CacheConfig] = None,
341
quant_config: Optional[QuantizationConfig] = None,
342
) -> None:
343
super().__init__()
@@ -348,6 +351,7 @@ class MixtralDecoderLayer(nn.Module):
351
num_kv_heads=config.num_key_value_heads,
352
rope_theta=rope_theta,
353
sliding_window=config.sliding_window,
354
+ cache_config=cache_config,
355
quant_config=quant_config)
356
self.block_sparse_moe = MixtralMoE(
357
num_experts=config.num_local_experts,
@@ -394,6 +398,7 @@ class MixtralModel(nn.Module):
398
def __init__(
399
self,
400
config: MixtralConfig,
401
+ cache_config: Optional[CacheConfig] = None,
402
quant_config: Optional[QuantizationConfig] = None,
403
lora_config: Optional[LoRAConfig] = None,
404
) -> None:
@@ -410,7 +415,9 @@ class MixtralModel(nn.Module):
415
org_num_embeddings=config.vocab_size,
416
)
417
self.layers = nn.ModuleList([
413
- MixtralDecoderLayer(config, quant_config=quant_config)
418
+ MixtralDecoderLayer(config,
419
+ cache_config,
420
+ quant_config=quant_config)
421
for _ in range(config.num_hidden_layers)
422
])
423
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -460,12 +467,14 @@ class MixtralForCausalLM(nn.Module):
467
def __init__(
468
self,
469
config: MixtralConfig,
470
+ cache_config: Optional[CacheConfig] = None,
471
quant_config: Optional[QuantizationConfig] = None,
472
lora_config: Optional[LoRAConfig] = None,
473
) -> None:
474
super().__init__()
475
self.config = config
476
self.model = MixtralModel(config,
477
+ cache_config,
478
quant_config,
479
lora_config=lora_config)
480
self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/mixtral_quant.py
+21
-10
@@ -30,6 +30,7 @@ from torch import nn
30
from transformers import MixtralConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
+from vllm.config import CacheConfig
34
from vllm.distributed import (get_tensor_model_parallel_rank,
35
get_tensor_model_parallel_world_size,
36
tensor_model_parallel_all_reduce)
@@ -157,14 +158,17 @@ class MixtralMoE(nn.Module):
158
159
class MixtralAttention(nn.Module):
160
160
- def __init__(self,
161
- hidden_size: int,
162
- num_heads: int,
163
- num_kv_heads: int,
164
- max_position: int = 4096 * 32,
165
- rope_theta: float = 10000,
166
- quant_config: Optional[QuantizationConfig] = None,
167
- sliding_window: Optional[int] = None) -> None:
161
+ def __init__(
162
+ self,
163
+ hidden_size: int,
164
+ num_heads: int,
165
+ num_kv_heads: int,
166
+ max_position: int = 4096 * 32,
167
+ rope_theta: float = 10000,
168
+ quant_config: Optional[QuantizationConfig] = None,
169
+ sliding_window: Optional[int] = None,
170
+ cache_config: Optional[CacheConfig] = None,
171
+ ) -> None:
172
super().__init__()
173
self.hidden_size = hidden_size
174
tp_size = get_tensor_model_parallel_world_size()
@@ -215,6 +219,7 @@ class MixtralAttention(nn.Module):
219
self.scaling,
220
num_kv_heads=self.num_kv_heads,
221
sliding_window=self.sliding_window,
222
+ cache_config=cache_config,
223
)
224
225
def forward(
@@ -237,6 +242,7 @@ class MixtralDecoderLayer(nn.Module):
242
def __init__(
243
self,
244
config: MixtralConfig,
245
+ cache_config: Optional[CacheConfig] = None,
246
quant_config: Optional[QuantizationConfig] = None,
247
) -> None:
248
super().__init__()
@@ -250,6 +256,7 @@ class MixtralDecoderLayer(nn.Module):
256
num_kv_heads=config.num_key_value_heads,
257
rope_theta=rope_theta,
258
sliding_window=config.sliding_window,
259
+ cache_config=cache_config,
260
quant_config=quant_config)
261
self.block_sparse_moe = MixtralMoE(config=config,
262
quant_config=quant_config)
@@ -292,6 +299,7 @@ class MixtralModel(nn.Module):
299
def __init__(
300
self,
301
config: MixtralConfig,
302
+ cache_config: Optional[CacheConfig] = None,
303
quant_config: Optional[QuantizationConfig] = None,
304
) -> None:
305
super().__init__()
@@ -303,7 +311,9 @@ class MixtralModel(nn.Module):
311
config.hidden_size,
312
)
313
self.layers = nn.ModuleList([
306
- MixtralDecoderLayer(config, quant_config=quant_config)
314
+ MixtralDecoderLayer(config,
315
+ cache_config,
316
+ quant_config=quant_config)
317
for _ in range(config.num_hidden_layers)
318
])
319
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -332,12 +342,13 @@ class MixtralForCausalLM(nn.Module):
342
def __init__(
343
self,
344
config: MixtralConfig,
345
+ cache_config: Optional[CacheConfig] = None,
346
quant_config: Optional[QuantizationConfig] = None,
347
) -> None:
348
super().__init__()
349
self.config = config
350
self.quant_config = quant_config
340
- self.model = MixtralModel(config, quant_config)
351
+ self.model = MixtralModel(config, cache_config, quant_config)
352
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
353
self.logits_processor = LogitsProcessor(config.vocab_size)
354
self.sampler = Sampler()
vllm/model_executor/models/mpt.py
+13
-5
@@ -7,6 +7,7 @@ import torch
7
import torch.nn as nn
8
9
from vllm.attention import Attention, AttentionMetadata
10
+from vllm.config import CacheConfig
11
from vllm.distributed import (get_tensor_model_parallel_rank,
12
get_tensor_model_parallel_world_size)
13
from vllm.model_executor.layers.activation import get_act_fn
@@ -43,6 +44,7 @@ class MPTAttention(nn.Module):
44
def __init__(
45
self,
46
config: MPTConfig,
47
+ cache_config: Optional[CacheConfig] = None,
48
quant_config: Optional[QuantizationConfig] = None,
49
):
50
super().__init__()
@@ -107,7 +109,8 @@ class MPTAttention(nn.Module):
109
self.head_dim,
110
scaling,
111
alibi_slopes=alibi_slopes,
110
- num_kv_heads=self.num_kv_heads)
112
+ num_kv_heads=self.num_kv_heads,
113
+ cache_config=cache_config)
114
115
def forward(
116
self,
@@ -166,12 +169,13 @@ class MPTBlock(nn.Module):
169
def __init__(
170
self,
171
config: MPTConfig,
172
+ cache_config: Optional[CacheConfig] = None,
173
quant_config: Optional[QuantizationConfig] = None,
174
):
175
super().__init__()
176
hidden_size = config.d_model
177
self.norm_1 = nn.LayerNorm(hidden_size)
174
- self.attn = MPTAttention(config, quant_config)
178
+ self.attn = MPTAttention(config, cache_config, quant_config)
179
self.norm_2 = nn.LayerNorm(hidden_size)
180
self.ffn = MPTMLP(config, quant_config)
181
@@ -201,6 +205,7 @@ class MPTModel(nn.Module):
205
def __init__(
206
self,
207
config: MPTConfig,
208
+ cache_config: Optional[CacheConfig] = None,
209
quant_config: Optional[QuantizationConfig] = None,
210
):
211
super().__init__()
@@ -211,8 +216,10 @@ class MPTModel(nn.Module):
216
config.vocab_size,
217
config.d_model,
218
)
214
- self.blocks = nn.ModuleList(
215
- [MPTBlock(config, quant_config) for _ in range(config.n_layers)])
219
+ self.blocks = nn.ModuleList([
220
+ MPTBlock(config, cache_config, quant_config)
221
+ for _ in range(config.n_layers)
222
+ ])
223
self.norm_f = nn.LayerNorm(config.d_model)
224
if config.no_bias:
225
for module in self.modules():
@@ -246,6 +253,7 @@ class MPTForCausalLM(nn.Module):
253
def __init__(
254
self,
255
config: MPTConfig,
256
+ cache_config: Optional[CacheConfig] = None,
257
quant_config: Optional[QuantizationConfig] = None,
258
):
259
super().__init__()
@@ -253,7 +261,7 @@ class MPTForCausalLM(nn.Module):
261
assert config.tie_word_embeddings
262
self.quant_config = quant_config
263
256
- self.transformer = MPTModel(config, quant_config)
264
+ self.transformer = MPTModel(config, cache_config, quant_config)
265
self.lm_head_weight = self.transformer.wte.weight
266
self.logits_processor = LogitsProcessor(config.vocab_size)
267
self.sampler = Sampler()
vllm/model_executor/models/olmo.py
+10
-4
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers import OlmoConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.config import CacheConfig
32
from vllm.distributed import get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -55,6 +56,7 @@ class OlmoAttention(nn.Module):
56
def __init__(
57
self,
58
config: OlmoConfig,
59
+ cache_config: Optional[CacheConfig] = None,
60
quant_config: Optional[QuantizationConfig] = None,
61
):
62
super().__init__()
@@ -93,7 +95,8 @@ class OlmoAttention(nn.Module):
95
self.scaling = self.head_dim**-0.5
96
self.attn = Attention(self.num_heads,
97
self.head_dim,
96
- scale=self.scaling)
98
+ scale=self.scaling,
99
+ cache_config=cache_config)
100
101
# Attention output projection.
102
self.o_proj = RowParallelLinear(
@@ -175,10 +178,11 @@ class OlmoDecoderLayer(nn.Module):
178
179
def __init__(self,
180
config: OlmoConfig,
181
+ cache_config: Optional[CacheConfig] = None,
182
quant_config: Optional[QuantizationConfig] = None):
183
super().__init__()
184
# Attention block.
181
- self.self_attn = OlmoAttention(config, quant_config)
185
+ self.self_attn = OlmoAttention(config, cache_config, quant_config)
186
187
# MLP block.
188
self.mlp = OlmoMLP(config, quant_config)
@@ -217,6 +221,7 @@ class OlmoModel(nn.Module):
221
222
def __init__(self,
223
config: OlmoConfig,
224
+ cache_config: Optional[CacheConfig] = None,
225
quant_config: Optional[QuantizationConfig] = None):
226
super().__init__()
227
self.config = config
@@ -224,7 +229,7 @@ class OlmoModel(nn.Module):
229
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
230
config.hidden_size)
231
self.layers = nn.ModuleList([
227
- OlmoDecoderLayer(config, quant_config)
232
+ OlmoDecoderLayer(config, cache_config, quant_config)
233
for layer_idx in range(config.num_hidden_layers)
234
])
235
self.norm = nn.LayerNorm(config.hidden_size,
@@ -271,10 +276,11 @@ class OlmoForCausalLM(nn.Module):
276
277
def __init__(self,
278
config: OlmoConfig,
279
+ cache_config: Optional[CacheConfig] = None,
280
quant_config: Optional[QuantizationConfig] = None):
281
super().__init__()
282
self.config = config
277
- self.model = OlmoModel(config, quant_config)
283
+ self.model = OlmoModel(config, cache_config, quant_config)
284
if config.tie_word_embeddings:
285
self.lm_head_weight = self.model.embed_tokens.weight
286
else:
vllm/model_executor/models/opt.py
+12
-4
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import OPTConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.config import CacheConfig
28
from vllm.distributed import get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -61,6 +62,7 @@ class OPTAttention(nn.Module):
62
embed_dim: int,
63
num_heads: int,
64
bias: bool = True,
65
+ cache_config: Optional[CacheConfig] = None,
66
quant_config: Optional[QuantizationConfig] = None,
67
) -> None:
68
super().__init__()
@@ -88,7 +90,8 @@ class OPTAttention(nn.Module):
90
)
91
self.attn = Attention(self.num_heads,
92
self.head_dim,
91
- scale=self.scaling)
93
+ scale=self.scaling,
94
+ cache_config=cache_config)
95
96
def forward(
97
self,
@@ -108,6 +111,7 @@ class OPTDecoderLayer(nn.Module):
111
def __init__(
112
self,
113
config: OPTConfig,
114
+ cache_config: Optional[CacheConfig] = None,
115
quant_config: Optional[QuantizationConfig] = None,
116
):
117
super().__init__()
@@ -117,6 +121,7 @@ class OPTDecoderLayer(nn.Module):
121
embed_dim=self.embed_dim,
122
num_heads=config.num_attention_heads,
123
bias=config.enable_bias,
124
+ cache_config=cache_config,
125
quant_config=quant_config,
126
)
127
self.do_layer_norm_before = config.do_layer_norm_before
@@ -181,6 +186,7 @@ class OPTDecoder(nn.Module):
186
def __init__(
187
self,
188
config: OPTConfig,
189
+ cache_config: Optional[CacheConfig] = None,
190
quant_config: Optional[QuantizationConfig] = None,
191
):
192
super().__init__()
@@ -226,7 +232,7 @@ class OPTDecoder(nn.Module):
232
self.final_layer_norm = None
233
234
self.layers = nn.ModuleList([
229
- OPTDecoderLayer(config, quant_config)
235
+ OPTDecoderLayer(config, cache_config, quant_config)
236
for _ in range(config.num_hidden_layers)
237
])
238
@@ -259,10 +265,11 @@ class OPTModel(nn.Module):
265
def __init__(
266
self,
267
config: OPTConfig,
268
+ cache_config: Optional[CacheConfig] = None,
269
quant_config: Optional[QuantizationConfig] = None,
270
):
271
super().__init__()
265
- self.decoder = OPTDecoder(config, quant_config)
272
+ self.decoder = OPTDecoder(config, cache_config, quant_config)
273
274
def forward(
275
self,
@@ -279,12 +286,13 @@ class OPTForCausalLM(nn.Module):
286
def __init__(
287
self,
288
config,
289
+ cache_config: Optional[CacheConfig] = None,
290
quant_config: Optional[QuantizationConfig] = None,
291
):
292
super().__init__()
293
self.config = config
294
self.quant_config = quant_config
287
- self.model = OPTModel(config, quant_config)
295
+ self.model = OPTModel(config, cache_config, quant_config)
296
self.lm_head_weight = self.model.decoder.embed_tokens.weight
297
self.logits_processor = LogitsProcessor(config.vocab_size)
298
self.sampler = Sampler()
vllm/model_executor/models/orion.py
+10
-3
@@ -11,6 +11,7 @@ from torch import nn
11
from transformers import PretrainedConfig
12
13
from vllm.attention import Attention, AttentionMetadata
14
+from vllm.config import CacheConfig
15
from vllm.distributed import get_tensor_model_parallel_world_size
16
from vllm.model_executor.layers.activation import SiluAndMul
17
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -68,6 +69,7 @@ class OrionAttention(nn.Module):
69
rope_theta: float = 10000,
70
rope_scaling: Optional[Dict[str, Any]] = None,
71
max_position_embeddings: int = 8192,
72
+ cache_config: Optional[CacheConfig] = None,
73
quant_config: Optional[QuantizationConfig] = None,
74
) -> None:
75
super().__init__()
@@ -118,7 +120,8 @@ class OrionAttention(nn.Module):
120
self.attn = Attention(self.num_heads,
121
self.head_dim,
122
self.scaling,
121
- num_kv_heads=self.num_kv_heads)
123
+ num_kv_heads=self.num_kv_heads,
124
+ cache_config=cache_config)
125
126
def forward(
127
self,
@@ -140,6 +143,7 @@ class OrionDecoderLayer(nn.Module):
143
def __init__(
144
self,
145
config: PretrainedConfig,
146
+ cache_config: Optional[CacheConfig] = None,
147
quant_config: Optional[QuantizationConfig] = None,
148
) -> None:
149
super().__init__()
@@ -155,6 +159,7 @@ class OrionDecoderLayer(nn.Module):
159
rope_theta=rope_theta,
160
rope_scaling=rope_scaling,
161
max_position_embeddings=max_position_embeddings,
162
+ cache_config=cache_config,
163
quant_config=quant_config,
164
)
165
self.mlp = OrionMLP(
@@ -202,6 +207,7 @@ class OrionModel(nn.Module):
207
def __init__(
208
self,
209
config: PretrainedConfig,
210
+ cache_config: Optional[CacheConfig] = None,
211
quant_config: Optional[QuantizationConfig] = None,
212
) -> None:
213
super().__init__()
@@ -213,7 +219,7 @@ class OrionModel(nn.Module):
219
config.hidden_size,
220
)
221
self.layers = nn.ModuleList([
216
- OrionDecoderLayer(config, quant_config)
222
+ OrionDecoderLayer(config, cache_config, quant_config)
223
for _ in range(config.num_hidden_layers)
224
])
225
self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -245,12 +251,13 @@ class OrionForCausalLM(nn.Module):
251
def __init__(
252
self,
253
config: PretrainedConfig,
254
+ cache_config: Optional[CacheConfig] = None,
255
quant_config: Optional[QuantizationConfig] = None,
256
) -> None:
257
super().__init__()
258
self.config = config
259
self.quant_config = quant_config
253
- self.model = OrionModel(config, quant_config)
260
+ self.model = OrionModel(config, cache_config, quant_config)
261
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
262
self.logits_processor = LogitsProcessor(config.vocab_size)
263
self.sampler = Sampler()
vllm/model_executor/models/phi.py
+12
-4
@@ -42,6 +42,7 @@ from torch import nn
42
from transformers import PretrainedConfig
43
44
from vllm.attention import Attention, AttentionMetadata
45
+from vllm.config import CacheConfig
46
from vllm.distributed import get_tensor_model_parallel_world_size
47
from vllm.model_executor.layers.activation import get_act_fn
48
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -63,6 +64,7 @@ class PhiAttention(nn.Module):
64
65
def __init__(self,
66
config: PretrainedConfig,
67
+ cache_config: Optional[CacheConfig] = None,
68
quant_config: Optional[QuantizationConfig] = None):
69
super().__init__()
70
self.total_num_heads = config.num_attention_heads
@@ -105,7 +107,10 @@ class PhiAttention(nn.Module):
107
max_position=max_position_embeddings,
108
base=rope_theta,
109
)
108
- self.attn = Attention(self.num_heads, self.head_size, scaling)
110
+ self.attn = Attention(self.num_heads,
111
+ self.head_size,
112
+ scaling,
113
+ cache_config=cache_config)
114
115
def forward(
116
self,
@@ -155,11 +160,12 @@ class PhiLayer(nn.Module):
160
161
def __init__(self,
162
config: PretrainedConfig,
163
+ cache_config: Optional[CacheConfig] = None,
164
quant_config: Optional[QuantizationConfig] = None):
165
super().__init__()
166
self.input_layernorm = nn.LayerNorm(config.hidden_size,
167
eps=config.layer_norm_eps)
162
- self.self_attn = PhiAttention(config, quant_config)
168
+ self.self_attn = PhiAttention(config, cache_config, quant_config)
169
self.mlp = PhiMLP(config, quant_config)
170
171
def forward(
@@ -186,6 +192,7 @@ class PhiModel(nn.Module):
192
193
def __init__(self,
194
config: PretrainedConfig,
195
+ cache_config: Optional[CacheConfig] = None,
196
quant_config: Optional[QuantizationConfig] = None):
197
super().__init__()
198
self.config = config
@@ -193,7 +200,7 @@ class PhiModel(nn.Module):
200
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
201
config.hidden_size)
202
self.layers = nn.ModuleList([
196
- PhiLayer(config, quant_config)
203
+ PhiLayer(config, cache_config, quant_config)
204
for _ in range(config.num_hidden_layers)
205
])
206
self.final_layernorm = nn.LayerNorm(config.hidden_size,
@@ -225,12 +232,13 @@ class PhiForCausalLM(nn.Module):
232
233
def __init__(self,
234
config: PretrainedConfig,
235
+ cache_config: Optional[CacheConfig] = None,
236
quant_config: Optional[QuantizationConfig] = None):
237
super().__init__()
238
self.config = config
239
self.quant_config = quant_config
240
233
- self.model = PhiModel(config, quant_config)
241
+ self.model = PhiModel(config, cache_config, quant_config)
242
243
self.lm_head = ParallelLMHead(config.vocab_size,
244
config.hidden_size,
vllm/model_executor/models/qwen.py
+12
-3
@@ -11,6 +11,7 @@ from torch import nn
11
from transformers import PretrainedConfig
12
13
from vllm.attention import Attention, AttentionMetadata
14
+from vllm.config import CacheConfig
15
from vllm.distributed import get_tensor_model_parallel_world_size
16
from vllm.model_executor.layers.activation import SiluAndMul
17
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -68,6 +69,7 @@ class QWenAttention(nn.Module):
69
max_position_embeddings: int,
70
rope_theta: float = 10000,
71
rope_scaling: Optional[Dict[str, Any]] = None,
72
+ cache_config: Optional[CacheConfig] = None,
73
quant_config: Optional[QuantizationConfig] = None,
74
):
75
super().__init__()
@@ -101,7 +103,10 @@ class QWenAttention(nn.Module):
103
base=rope_theta,
104
rope_scaling=rope_scaling,
105
)
104
- self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
106
+ self.attn = Attention(self.num_heads,
107
+ self.head_dim,
108
+ self.scaling,
109
+ cache_config=cache_config)
110
111
def forward(
112
self,
@@ -123,6 +128,7 @@ class QWenBlock(nn.Module):
128
def __init__(
129
self,
130
config: PretrainedConfig,
131
+ cache_config: Optional[CacheConfig] = None,
132
quant_config: Optional[QuantizationConfig] = None,
133
):
134
super().__init__()
@@ -135,6 +141,7 @@ class QWenBlock(nn.Module):
141
config.max_position_embeddings,
142
rope_theta=rope_theta,
143
rope_scaling=rope_scaling,
144
+ cache_config=cache_config,
145
quant_config=quant_config)
146
147
self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -175,6 +182,7 @@ class QWenModel(nn.Module):
182
def __init__(
183
self,
184
config: PretrainedConfig,
185
+ cache_config: Optional[CacheConfig] = None,
186
quant_config: Optional[QuantizationConfig] = None,
187
):
188
super().__init__()
@@ -186,7 +194,7 @@ class QWenModel(nn.Module):
194
config.hidden_size,
195
)
196
self.h = nn.ModuleList([
189
- QWenBlock(config, quant_config)
197
+ QWenBlock(config, cache_config, quant_config)
198
for _ in range(config.num_hidden_layers)
199
])
200
self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -218,12 +226,13 @@ class QWenLMHeadModel(nn.Module):
226
def __init__(
227
self,
228
config: PretrainedConfig,
229
+ cache_config: Optional[CacheConfig] = None,
230
quant_config: Optional[QuantizationConfig] = None,
231
):
232
super().__init__()
233
self.config = config
234
self.quant_config = quant_config
226
- self.transformer = QWenModel(config, quant_config)
235
+ self.transformer = QWenModel(config, cache_config, quant_config)
236
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
237
self.logits_processor = LogitsProcessor(config.vocab_size)
238
self.sampler = Sampler()
vllm/model_executor/models/qwen2.py
+10
-4
@@ -29,7 +29,7 @@ from torch import nn
29
from transformers import Qwen2Config
30
31
from vllm.attention import Attention, AttentionMetadata
32
-from vllm.config import LoRAConfig
32
+from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -87,6 +87,7 @@ class Qwen2Attention(nn.Module):
87
max_position: int = 4096 * 32,
88
rope_theta: float = 10000,
89
use_sliding_window: bool = False,
90
+ cache_config: Optional[CacheConfig] = None,
91
quant_config: Optional[QuantizationConfig] = None,
92
sliding_window: Optional[int] = None) -> None:
93
super().__init__()
@@ -137,7 +138,8 @@ class Qwen2Attention(nn.Module):
138
self.head_dim,
139
self.scaling,
140
num_kv_heads=self.num_kv_heads,
140
- sliding_window=self.sliding_window)
141
+ sliding_window=self.sliding_window,
142
+ cache_config=cache_config)
143
144
def forward(
145
self,
@@ -160,6 +162,7 @@ class Qwen2DecoderLayer(nn.Module):
162
self,
163
config: Qwen2Config,
164
layer_idx: int,
165
+ cache_config: Optional[CacheConfig] = None,
166
quant_config: Optional[QuantizationConfig] = None,
167
) -> None:
168
super().__init__()
@@ -175,6 +178,7 @@ class Qwen2DecoderLayer(nn.Module):
178
num_kv_heads=config.num_key_value_heads,
179
rope_theta=rope_theta,
180
use_sliding_window=use_sliding_window,
181
+ cache_config=cache_config,
182
quant_config=quant_config,
183
sliding_window=config.sliding_window)
184
self.mlp = Qwen2MLP(
@@ -222,6 +226,7 @@ class Qwen2Model(nn.Module):
226
def __init__(
227
self,
228
config: Qwen2Config,
229
+ cache_config: Optional[CacheConfig] = None,
230
quant_config: Optional[QuantizationConfig] = None,
231
) -> None:
232
super().__init__()
@@ -234,7 +239,7 @@ class Qwen2Model(nn.Module):
239
config.hidden_size,
240
)
241
self.layers = nn.ModuleList([
237
- Qwen2DecoderLayer(config, layer_idx, quant_config)
242
+ Qwen2DecoderLayer(config, layer_idx, cache_config, quant_config)
243
for layer_idx in range(config.num_hidden_layers)
244
])
245
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -287,6 +292,7 @@ class Qwen2ForCausalLM(nn.Module):
292
def __init__(
293
self,
294
config: Qwen2Config,
295
+ cache_config: Optional[CacheConfig] = None,
296
quant_config: Optional[QuantizationConfig] = None,
297
lora_config: Optional[LoRAConfig] = None,
298
) -> None:
@@ -294,7 +300,7 @@ class Qwen2ForCausalLM(nn.Module):
300
super().__init__()
301
self.config = config
302
self.quant_config = quant_config
297
- self.model = Qwen2Model(config, quant_config)
303
+ self.model = Qwen2Model(config, cache_config, quant_config)
304
305
if config.tie_word_embeddings:
306
self.lm_head_weight = self.model.embed_tokens.weight
vllm/model_executor/models/qwen2_moe.py
+13
-3
@@ -30,6 +30,7 @@ from torch import nn
30
from transformers import PretrainedConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
+from vllm.config import CacheConfig
34
from vllm.distributed import (get_tensor_model_parallel_rank,
35
get_tensor_model_parallel_world_size,
36
tensor_model_parallel_all_reduce)
@@ -187,6 +188,7 @@ class Qwen2MoeAttention(nn.Module):
188
rope_theta: float = 10000,
189
rope_scaling: Optional[Dict[str, Any]] = None,
190
max_position_embeddings: int = 8192,
191
+ cache_config: Optional[CacheConfig] = None,
192
quant_config: Optional[QuantizationConfig] = None,
193
) -> None:
194
super().__init__()
@@ -238,7 +240,8 @@ class Qwen2MoeAttention(nn.Module):
240
self.attn = Attention(self.num_heads,
241
self.head_dim,
242
self.scaling,
241
- num_kv_heads=self.num_kv_heads)
243
+ num_kv_heads=self.num_kv_heads,
244
+ cache_config=cache_config)
245
246
def forward(
247
self,
@@ -261,6 +264,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
264
self,
265
config: PretrainedConfig,
266
layer_idx: int,
267
+ cache_config: Optional[CacheConfig] = None,
268
quant_config: Optional[QuantizationConfig] = None,
269
) -> None:
270
super().__init__()
@@ -276,6 +280,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
280
rope_theta=rope_theta,
281
rope_scaling=rope_scaling,
282
max_position_embeddings=max_position_embeddings,
283
+ cache_config=cache_config,
284
quant_config=quant_config,
285
)
286
if (config.num_experts is not None
@@ -328,6 +333,7 @@ class Qwen2MoeModel(nn.Module):
333
def __init__(
334
self,
335
config: PretrainedConfig,
336
+ cache_config: Optional[CacheConfig] = None,
337
quant_config: Optional[QuantizationConfig] = None,
338
) -> None:
339
super().__init__()
@@ -339,7 +345,10 @@ class Qwen2MoeModel(nn.Module):
345
config.hidden_size,
346
)
347
self.layers = nn.ModuleList([
342
- Qwen2MoeDecoderLayer(config, layer_idx, quant_config=quant_config)
348
+ Qwen2MoeDecoderLayer(config,
349
+ layer_idx,
350
+ cache_config,
351
+ quant_config=quant_config)
352
for layer_idx in range(config.num_hidden_layers)
353
])
354
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -369,12 +378,13 @@ class Qwen2MoeForCausalLM(nn.Module):
378
def __init__(
379
self,
380
config: PretrainedConfig,
381
+ cache_config: Optional[CacheConfig] = None,
382
quant_config: Optional[QuantizationConfig] = None,
383
) -> None:
384
super().__init__()
385
self.config = config
386
self.quant_config = quant_config
377
- self.model = Qwen2MoeModel(config, quant_config)
387
+ self.model = Qwen2MoeModel(config, cache_config, quant_config)
388
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
389
self.logits_processor = LogitsProcessor(config.vocab_size)
390
self.sampler = Sampler()
vllm/model_executor/models/stablelm.py
+10
-4
@@ -26,6 +26,7 @@ from torch import nn
26
from transformers import PretrainedConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
+from vllm.config import CacheConfig
30
from vllm.distributed import get_tensor_model_parallel_world_size
31
from vllm.model_executor.layers.activation import SiluAndMul
32
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -72,6 +73,7 @@ class StablelmAttention(nn.Module):
73
74
def __init__(self,
75
config: PretrainedConfig,
76
+ cache_config: Optional[CacheConfig] = None,
77
quant_config: Optional[QuantizationConfig] = None) -> None:
78
super().__init__()
79
self.config = config
@@ -124,7 +126,8 @@ class StablelmAttention(nn.Module):
126
self.attn = Attention(self.num_heads,
127
self.head_dim,
128
self.scaling,
127
- num_kv_heads=self.num_key_value_heads)
129
+ num_kv_heads=self.num_key_value_heads,
130
+ cache_config=cache_config)
131
132
def forward(
133
self,
@@ -146,10 +149,11 @@ class StablelmDecoderLayer(nn.Module):
149
def __init__(
150
self,
151
config: PretrainedConfig,
152
+ cache_config: Optional[CacheConfig] = None,
153
quant_config: Optional[QuantizationConfig] = None,
154
) -> None:
155
super().__init__()
152
- self.self_attn = StablelmAttention(config)
156
+ self.self_attn = StablelmAttention(config, cache_config, quant_config)
157
self.mlp = StablelmMLP(config, quant_config)
158
norm_eps = getattr(config, "norm_eps",
159
getattr(config, "layer_norm_eps", 1e-05))
@@ -188,6 +192,7 @@ class StableLMEpochModel(nn.Module):
192
193
def __init__(self,
194
config: PretrainedConfig,
195
+ cache_config: Optional[CacheConfig] = None,
196
quant_config: Optional[QuantizationConfig] = None) -> None:
197
super().__init__()
198
self.embed_tokens = VocabParallelEmbedding(
@@ -195,7 +200,7 @@ class StableLMEpochModel(nn.Module):
200
config.hidden_size,
201
)
202
self.layers = nn.ModuleList([
198
- StablelmDecoderLayer(config, quant_config)
203
+ StablelmDecoderLayer(config, cache_config, quant_config)
204
for _ in range(config.num_hidden_layers)
205
])
206
norm_eps = getattr(config, "norm_eps",
@@ -227,12 +232,13 @@ class StablelmForCausalLM(nn.Module):
232
def __init__(
233
self,
234
config: PretrainedConfig,
235
+ cache_config: Optional[CacheConfig] = None,
236
quant_config: Optional[QuantizationConfig] = None,
237
) -> None:
238
super().__init__()
239
self.config = config
240
self.quant_config = quant_config
235
- self.model = StableLMEpochModel(config, quant_config)
241
+ self.model = StableLMEpochModel(config, cache_config, quant_config)
242
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
243
self.logits_processor = LogitsProcessor(config.vocab_size)
244
self.sampler = Sampler()
vllm/model_executor/models/starcoder2.py
+15
-3
@@ -25,6 +25,7 @@ from torch import nn
25
from transformers import Starcoder2Config
26
27
from vllm.attention import Attention, AttentionMetadata
28
+from vllm.config import CacheConfig
29
from vllm.distributed import get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -46,6 +47,7 @@ class Starcoder2Attention(nn.Module):
47
48
def __init__(self,
49
config: Starcoder2Config,
50
+ cache_config: Optional[CacheConfig] = None,
51
quant_config: Optional[QuantizationConfig] = None):
52
super().__init__()
53
self.config = config
@@ -101,6 +103,7 @@ class Starcoder2Attention(nn.Module):
103
self.scaling,
104
num_kv_heads=self.num_kv_heads,
105
sliding_window=self.sliding_window,
106
+ cache_config=cache_config,
107
)
108
109
def forward(
@@ -150,10 +153,13 @@ class Starcoder2DecoderLayer(nn.Module):
153
154
def __init__(self,
155
config: Starcoder2Config,
156
+ cache_config: Optional[CacheConfig] = None,
157
quant_config: Optional[QuantizationConfig] = None):
158
super().__init__()
159
self.hidden_size = config.hidden_size
156
- self.self_attn = Starcoder2Attention(config, quant_config=quant_config)
160
+ self.self_attn = Starcoder2Attention(config,
161
+ cache_config,
162
+ quant_config=quant_config)
163
self.mlp = Starcoder2MLP(config, quant_config=quant_config)
164
self.input_layernorm = nn.LayerNorm(config.hidden_size,
165
eps=config.norm_epsilon)
@@ -191,6 +197,7 @@ class Starcoder2Model(nn.Module):
197
198
def __init__(self,
199
config: Starcoder2Config,
200
+ cache_config: Optional[CacheConfig] = None,
201
quant_config: Optional[QuantizationConfig] = None):
202
super().__init__()
203
self.config = config
@@ -201,7 +208,9 @@ class Starcoder2Model(nn.Module):
208
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
209
config.hidden_size)
210
self.layers = nn.ModuleList([
204
- Starcoder2DecoderLayer(config, quant_config=quant_config)
211
+ Starcoder2DecoderLayer(config,
212
+ cache_config,
213
+ quant_config=quant_config)
214
for _ in range(config.num_hidden_layers)
215
])
216
self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
@@ -226,10 +235,13 @@ class Starcoder2ForCausalLM(nn.Module):
235
236
def __init__(self,
237
config: Starcoder2Config,
238
+ cache_config: Optional[CacheConfig] = None,
239
quant_config: Optional[QuantizationConfig] = None):
240
super().__init__()
241
self.config = config
232
- self.model = Starcoder2Model(config, quant_config=quant_config)
242
+ self.model = Starcoder2Model(config,
243
+ cache_config,
244
+ quant_config=quant_config)
245
self.vocab_size = config.vocab_size
246
self.unpadded_vocab_size = config.vocab_size
247
if config.tie_word_embeddings:
vllm/model_executor/models/xverse.py
+10
-4
@@ -27,7 +27,7 @@ from torch import nn
27
from transformers import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
-from vllm.config import LoRAConfig
30
+from vllm.config import CacheConfig, LoRAConfig
31
from vllm.distributed import get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import SiluAndMul
33
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -89,6 +89,7 @@ class XverseAttention(nn.Module):
89
quant_config: Optional[QuantizationConfig] = None,
90
bias: bool = False,
91
sliding_window: Optional[int] = None,
92
+ cache_config: Optional[CacheConfig] = None,
93
) -> None:
94
super().__init__()
95
self.hidden_size = hidden_size
@@ -133,7 +134,8 @@ class XverseAttention(nn.Module):
134
self.head_dim,
135
self.scaling,
136
num_kv_heads=self.num_kv_heads,
136
- sliding_window=sliding_window)
137
+ sliding_window=sliding_window,
138
+ cache_config=cache_config)
139
140
def forward(
141
self,
@@ -155,6 +157,7 @@ class XverseDecoderLayer(nn.Module):
157
def __init__(
158
self,
159
config: PretrainedConfig,
160
+ cache_config: Optional[CacheConfig] = None,
161
quant_config: Optional[QuantizationConfig] = None,
162
) -> None:
163
super().__init__()
@@ -175,6 +178,7 @@ class XverseDecoderLayer(nn.Module):
178
quant_config=quant_config,
179
bias=getattr(config, "bias", False),
180
sliding_window=sliding_window,
181
+ cache_config=cache_config,
182
)
183
self.mlp = XverseMLP(
184
hidden_size=self.hidden_size,
@@ -221,6 +225,7 @@ class XverseModel(nn.Module):
225
def __init__(
226
self,
227
config: PretrainedConfig,
228
+ cache_config: Optional[CacheConfig] = None,
229
quant_config: Optional[QuantizationConfig] = None,
230
lora_config: Optional[LoRAConfig] = None,
231
) -> None:
@@ -237,7 +242,7 @@ class XverseModel(nn.Module):
242
org_num_embeddings=config.vocab_size,
243
)
244
self.layers = nn.ModuleList([
240
- XverseDecoderLayer(config, quant_config)
245
+ XverseDecoderLayer(config, cache_config, quant_config)
246
for _ in range(config.num_hidden_layers)
247
])
248
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -295,13 +300,14 @@ class XverseForCausalLM(nn.Module):
300
def __init__(
301
self,
302
config: PretrainedConfig,
303
+ cache_config: Optional[CacheConfig] = None,
304
quant_config: Optional[QuantizationConfig] = None,
305
lora_config=None,
306
) -> None:
307
super().__init__()
308
self.config = config
309
self.quant_config = quant_config
304
- self.model = XverseModel(config, quant_config)
310
+ self.model = XverseModel(config, cache_config, quant_config)
311
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
312
self.logits_processor = LogitsProcessor(config.vocab_size)
313
self.sampler = Sampler()
vllm/worker/cache_engine.py
+11
-3
@@ -31,7 +31,7 @@ class CacheEngine:
31
32
self.head_size = model_config.get_head_size()
33
self.num_layers = model_config.get_num_layers(parallel_config)
34
- self.num_heads = model_config.get_num_kv_heads(parallel_config)
34
+ self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
35
36
self.block_size = cache_config.block_size
37
self.num_gpu_blocks = cache_config.num_gpu_blocks
@@ -43,7 +43,15 @@ class CacheEngine:
43
self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
44
45
# Get attention backend.
46
- self.attn_backend = get_attn_backend(model_config.dtype)
46
+ self.attn_backend = get_attn_backend(
47
+ model_config.get_num_attention_heads(parallel_config),
48
+ self.head_size,
49
+ self.num_kv_heads,
50
+ model_config.get_sliding_window(),
51
+ model_config.dtype,
52
+ cache_config.cache_dtype,
53
+ self.block_size,
54
+ )
55
56
# Initialize the cache.
57
self.gpu_cache = self._allocate_kv_cache(self.num_gpu_blocks, "cuda")
@@ -56,7 +64,7 @@ class CacheEngine:
64
) -> List[torch.Tensor]:
65
"""Allocates KV cache on the specified device."""
66
kv_cache_shape = self.attn_backend.get_kv_cache_shape(
59
- num_blocks, self.block_size, self.num_heads, self.head_size)
67
+ num_blocks, self.block_size, self.num_kv_heads, self.head_size)
68
pin_memory = is_pin_memory_available() if device == "cpu" else False
69
kv_cache: List[torch.Tensor] = []
70
for _ in range(self.num_layers):
vllm/worker/cpu_model_runner.py
+11
-4
@@ -53,7 +53,15 @@ class CPUModelRunner:
53
self.kv_cache_dtype = kv_cache_dtype
54
self.sliding_window = model_config.get_sliding_window()
55
self.block_size = cache_config.block_size
56
- self.attn_backend = get_attn_backend(self.model_config.dtype)
56
+ self.attn_backend = get_attn_backend(
57
+ self.model_config.get_num_attention_heads(self.parallel_config),
58
+ self.model_config.get_head_size(),
59
+ self.model_config.get_num_kv_heads(self.parallel_config),
60
+ self.model_config.get_sliding_window(),
61
+ self.model_config.dtype,
62
+ self.kv_cache_dtype,
63
+ self.block_size,
64
+ )
65
66
# Lazy initialization.
67
self.model: nn.Module # Set after init_Model
@@ -66,7 +74,8 @@ class CPUModelRunner:
74
vision_language_config=self.vision_language_config,
75
lora_config=self.lora_config,
76
parallel_config=self.parallel_config,
69
- scheduler_config=self.scheduler_config)
77
+ scheduler_config=self.scheduler_config,
78
+ cache_config=self.cache_config)
79
80
def _prepare_prompt(
81
self,
@@ -158,7 +167,6 @@ class CPUModelRunner:
167
decode_metadata=None,
168
block_tables=torch.tensor([]),
169
slot_mapping=slot_mapping,
161
- kv_cache_dtype=self.kv_cache_dtype,
170
)
171
return (input_tokens, input_positions, attn_metadata, seq_lens,
172
multi_modal_input)
@@ -242,7 +250,6 @@ class CPUModelRunner:
250
prefill_metadata=None,
251
decode_metadata=None,
252
block_tables=block_tables,
245
- kv_cache_dtype=self.kv_cache_dtype,
253
)
254
return (
255
input_tokens,
vllm/worker/cpu_worker.py
+9
-1
@@ -53,7 +53,15 @@ class CPUCacheEngine:
53
self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
54
55
# Get attention backend.
56
- self.attn_backend = get_attn_backend(model_config.dtype)
56
+ self.attn_backend = get_attn_backend(
57
+ self.model_config.get_num_attention_heads(self.parallel_config),
58
+ self.model_config.get_head_size(),
59
+ self.model_config.get_num_kv_heads(self.parallel_config),
60
+ self.model_config.get_sliding_window(),
61
+ self.model_config.dtype,
62
+ cache_config.cache_dtype,
63
+ self.block_size,
64
+ )
65
66
# Initialize the cache.
67
self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks)
vllm/worker/embedding_model_runner.py
-1
@@ -235,7 +235,6 @@ class EmbeddingModelRunner(ModelRunner):
235
num_decode_tokens=num_decode_tokens,
236
prefill_metadata=prefill_attn_metadata,
237
decode_metadata=decode_attn_metadata,
238
- kv_cache_dtype=self.kv_cache_dtype,
238
)
239
240
return (input_tokens, input_positions, attn_metadata, pooling_metadata,
vllm/worker/model_runner.py
+11
-4
@@ -141,10 +141,18 @@ class ModelRunner:
141
self.graph_block_tables = np.zeros(
142
(max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
143
dtype=np.int32)
144
- self.attn_backend = get_attn_backend(self.model_config.dtype)
144
+ self.attn_backend = get_attn_backend(
145
+ self.model_config.get_num_attention_heads(self.parallel_config),
146
+ self.model_config.get_head_size(),
147
+ self.model_config.get_num_kv_heads(self.parallel_config),
148
+ self.model_config.get_sliding_window(),
149
+ self.model_config.dtype,
150
+ self.kv_cache_dtype,
151
+ self.block_size,
152
+ )
153
154
# Lazy initialization
147
- self.model: torch.nn.Module # Set after load_model
155
+ self.model: nn.Module # Set after load_model
156
# Set if the backend is flashinfer.
157
self.flashinfer_workspace_buffer: torch.Tensor
158
# Set after load_model.
@@ -160,6 +168,7 @@ class ModelRunner:
168
vision_language_config=self.vision_language_config,
169
parallel_config=self.parallel_config,
170
scheduler_config=self.scheduler_config,
171
+ cache_config=self.cache_config,
172
)
173
174
self.model_memory_usage = m.consumed_memory
@@ -753,7 +762,6 @@ class ModelRunner:
762
num_decode_tokens=num_decode_tokens,
763
prefill_metadata=prefill_attn_metadata,
764
decode_metadata=decode_attn_metadata,
756
- kv_cache_dtype=self.kv_cache_dtype,
765
)
766
767
return (input_tokens, input_positions, attn_metadata,
@@ -965,7 +973,6 @@ class ModelRunner:
973
slot_mapping=slot_mapping[:batch_size],
974
prefill_metadata=None,
975
decode_metadata=decode_metadata,
968
- kv_cache_dtype=self.kv_cache_dtype,
976
)
977
978
if self.lora_config: