@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0fca3cdcf2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Enhance attention selector (#4751)

Woosuk Kwon committed May 13, 2024 at 10:47 UTC 0fca3cdcf265cd375bca684d951702b6b7adf65a
49 files changed +573 -220
tests/worker/test_model_runner.py
-1
@@ -307,7 +307,6 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
307
308 assert len(attn_metadata.slot_mapping) == len(input_tokens)
309 assert len(input_positions) == len(input_tokens)
310 - assert attn_metadata.kv_cache_dtype == "auto"
310 assert attn_metadata.num_prefills == prefill_batch_size
311 if enforce_eager:
312 assert attn_metadata.num_decode_tokens == decode_batch_size
vllm/attention/__init__.py
+2 -2
@@ -5,9 +5,9 @@ from vllm.attention.layer import Attention
5 from vllm.attention.selector import get_attn_backend
6
7 __all__ = [
8 + "Attention",
9 "AttentionBackend",
10 "AttentionMetadata",
10 - "Attention",
11 - "get_attn_backend",
11 "AttentionMetadataPerStage",
12 + "get_attn_backend",
13 ]
vllm/attention/backends/abstract.py
+2 -3
@@ -94,8 +94,6 @@ class AttentionMetadata(Generic[T]):
94 # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
95 # in block 0, and 1st slot in block 1, respectively.
96 slot_mapping: torch.Tensor
97 - # The kv cache's data type.
98 - kv_cache_dtype: str
97
98 def __post_init__(self):
99 if self.num_prefill_tokens > 0:
@@ -116,6 +114,7 @@ class AttentionImpl(ABC):
114 num_kv_heads: Optional[int] = None,
115 alibi_slopes: Optional[List[float]] = None,
116 sliding_window: Optional[int] = None,
117 + kv_cache_dtype: str = "auto",
118 ) -> None:
119 raise NotImplementedError
120
@@ -127,6 +126,6 @@ class AttentionImpl(ABC):
126 value: torch.Tensor,
127 kv_cache: torch.Tensor,
128 attn_metadata: AttentionMetadata,
130 - kv_scale: float,
129 + kv_scale: float = 1.0,
130 ) -> torch.Tensor:
131 raise NotImplementedError
vllm/attention/backends/flash_attn.py
+7 -6
@@ -140,16 +140,18 @@ class FlashAttentionImpl(AttentionImpl):
140 num_kv_heads: Optional[int] = None,
141 alibi_slopes: Optional[List[float]] = None,
142 sliding_window: Optional[int] = None,
143 + kv_cache_dtype: str = "auto",
144 ) -> None:
145 self.num_heads = num_heads
146 self.head_size = head_size
147 self.scale = float(scale)
148 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
148 - self.sliding_window = ((sliding_window, sliding_window)
149 - if sliding_window is not None else (-1, -1))
149 if alibi_slopes is not None:
150 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
151 self.alibi_slopes = alibi_slopes
152 + self.sliding_window = ((sliding_window, sliding_window)
153 + if sliding_window is not None else (-1, -1))
154 + self.kv_cache_dtype = kv_cache_dtype
155
156 assert self.num_heads % self.num_kv_heads == 0
157 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -167,7 +169,7 @@ class FlashAttentionImpl(AttentionImpl):
169 value: torch.Tensor,
170 kv_cache: torch.Tensor,
171 attn_metadata: AttentionMetadata[FlashAttentionMetadata],
170 - kv_scale: float,
172 + kv_scale: float = 1.0,
173 ) -> torch.Tensor:
174 """Forward pass with FlashAttention and PagedAttention.
175
@@ -196,8 +198,7 @@ class FlashAttentionImpl(AttentionImpl):
198 PagedAttention.write_to_paged_cache(key, value, key_cache,
199 value_cache,
200 attn_metadata.slot_mapping,
199 - attn_metadata.kv_cache_dtype,
200 - kv_scale)
201 + self.kv_cache_dtype, kv_scale)
202
203 num_prefill_tokens = attn_metadata.num_prefill_tokens
204 num_decode_tokens = attn_metadata.num_decode_tokens
@@ -264,7 +265,7 @@ class FlashAttentionImpl(AttentionImpl):
265 decode_meta.block_tables,
266 decode_meta.seq_lens_tensor,
267 decode_meta.max_seq_len,
267 - attn_metadata.kv_cache_dtype,
268 + self.kv_cache_dtype,
269 self.num_kv_heads,
270 self.scale,
271 self.alibi_slopes,
vllm/attention/backends/flashinfer.py
+23 -10
@@ -149,20 +149,33 @@ class FlashInferImpl(AttentionImpl):
149 num_kv_heads: Optional[int] = None,
150 alibi_slopes: Optional[List[float]] = None,
151 sliding_window: Optional[int] = None,
152 + kv_cache_dtype: str = "auto",
153 ) -> None:
153 - if sliding_window is not None:
154 - raise ValueError("Sliding window is not supported in FlashInfer.")
155 - self.sliding_window = (-1, -1)
156 - self.alibi_slopes = alibi_slopes
157 - self.scale = scale
154 self.num_heads = num_heads
155 self.head_size = head_size
156 + self.scale = float(scale)
157 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
158 + if alibi_slopes is not None:
159 + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
160 + self.alibi_slopes = alibi_slopes
161 + if sliding_window is not None:
162 + raise ValueError("Sliding window is not supported in FlashInfer.")
163 + self.sliding_window = (-1, -1)
164 + self.kv_cache_dtype = kv_cache_dtype
165
162 - def forward(self, query: torch.Tensor, key: torch.Tensor,
163 - value: torch.Tensor, kv_cache: Optional[torch.Tensor],
164 - attn_metadata: AttentionMetadata[FlashInferMetadata],
165 - kv_scale: float):
166 + assert self.num_heads % self.num_kv_heads == 0
167 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
168 +
169 + def forward(
170 + self,
171 + query: torch.Tensor,
172 + key: torch.Tensor,
173 + value: torch.Tensor,
174 + kv_cache: Optional[torch.Tensor],
175 + attn_metadata: AttentionMetadata[FlashInferMetadata],
176 + kv_scale: float = 1.0,
177 + ) -> torch.Tensor:
178 + assert kv_scale == 1.0
179 num_tokens, hidden_size = query.shape
180 query = query.view(-1, self.num_heads, self.head_size)
181 key = key.view(-1, self.num_kv_heads, self.head_size)
@@ -183,7 +196,7 @@ class FlashInferImpl(AttentionImpl):
196 kv_cache[:, 0],
197 kv_cache[:, 1],
198 attn_metadata.slot_mapping.flatten(),
186 - attn_metadata.kv_cache_dtype,
199 + self.kv_cache_dtype,
200 )
201
202 if prefill_meta := attn_metadata.prefill_metadata:
vllm/attention/backends/rocm_flash_attn.py
+9 -7
@@ -138,25 +138,27 @@ class ROCmFlashAttentionImpl(AttentionImpl):
138 num_kv_heads: Optional[int] = None,
139 alibi_slopes: Optional[List[float]] = None,
140 sliding_window: Optional[int] = None,
141 + kv_cache_dtype: str = "auto",
142 ) -> None:
143 self.num_heads = num_heads
144 self.head_size = head_size
145 self.scale = float(scale)
146 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
146 - self.sliding_window = ((sliding_window, sliding_window)
147 - if sliding_window is not None else (-1, -1))
147 if alibi_slopes is not None:
148 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
149 self.alibi_slopes = alibi_slopes
150 + self.sliding_window = ((sliding_window, sliding_window)
151 + if sliding_window is not None else (-1, -1))
152 + self.kv_cache_dtype = kv_cache_dtype
153
154 assert self.num_heads % self.num_kv_heads == 0
155 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
156
155 - suppored_head_sizes = PagedAttention.get_supported_head_sizes()
156 - if head_size not in suppored_head_sizes:
157 + supported_head_sizes = PagedAttention.get_supported_head_sizes()
158 + if head_size not in supported_head_sizes:
159 raise ValueError(
160 f"Head size {head_size} is not supported by PagedAttention. "
159 - f"Supported head sizes are: {suppored_head_sizes}.")
161 + f"Supported head sizes are: {supported_head_sizes}.")
162
163 self.use_naive_attn = False
164 # NOTE: Allow for switching between Triton and CK. Defaulting to triton.
@@ -229,7 +231,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
231 key_cache,
232 value_cache,
233 attn_metadata.slot_mapping,
232 - attn_metadata.kv_cache_dtype,
234 + self.kv_cache_dtype,
235 kv_scale,
236 )
237
@@ -323,7 +325,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
325 decode_meta.block_tables,
326 decode_meta.seq_lens_tensor,
327 decode_meta.max_seq_len,
326 - attn_metadata.kv_cache_dtype,
328 + self.kv_cache_dtype,
329 self.num_kv_heads,
330 self.scale,
331 self.alibi_slopes,
vllm/attention/backends/torch_sdpa.py
+17 -11
@@ -83,26 +83,32 @@ class TorchSDPABackendImpl(AttentionImpl):
83 num_kv_heads: Optional[int] = None,
84 alibi_slopes: Optional[List[float]] = None,
85 sliding_window: Optional[int] = None,
86 + kv_cache_dtype: str = "auto",
87 ) -> None:
88 self.num_heads = num_heads
89 self.head_size = head_size
90 self.scale = float(scale)
91 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
91 - self.sliding_window = sliding_window
92 if alibi_slopes is not None:
93 - assert len(alibi_slopes) == num_heads
93 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
94 self.alibi_slopes = alibi_slopes
96 - self.need_mask = (self.alibi_slopes is not None
97 - or self.sliding_window is not None)
95 + self.sliding_window = sliding_window
96 + self.kv_cache_dtype = kv_cache_dtype
97
98 assert self.num_heads % self.num_kv_heads == 0
99 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
101 - suppored_head_sizes = PagedAttention.get_supported_head_sizes()
102 - if head_size not in suppored_head_sizes:
100 + self.need_mask = (self.alibi_slopes is not None
101 + or self.sliding_window is not None)
102 +
103 + supported_head_sizes = PagedAttention.get_supported_head_sizes()
104 + if head_size not in supported_head_sizes:
105 raise ValueError(
106 f"Head size {head_size} is not supported by PagedAttention. "
105 - f"Supported head sizes are: {suppored_head_sizes}.")
107 + f"Supported head sizes are: {supported_head_sizes}.")
108 + if kv_cache_dtype != "auto":
109 + raise NotImplementedError(
110 + "Torch SDPA backend does not support FP8 KV cache. "
111 + "Please use xFormers backend instead.")
112
113 def forward(
114 self,
@@ -111,7 +117,7 @@ class TorchSDPABackendImpl(AttentionImpl):
117 value: torch.Tensor,
118 kv_cache: Optional[torch.Tensor],
119 attn_metadata: TorchSDPAMetadata, # type: ignore
114 - kv_scale: float,
120 + kv_scale: float = 1.0,
121 ) -> torch.Tensor:
122 """Forward pass with torch SDPA and PagedAttention.
123
@@ -124,6 +130,7 @@ class TorchSDPABackendImpl(AttentionImpl):
130 Returns:
131 shape = [num_tokens, num_heads * head_size]
132 """
133 + assert kv_scale == 1.0
134 num_tokens, hidden_size = query.shape
135 # Reshape the query, key, and value tensors.
136 query = query.view(-1, self.num_heads, self.head_size)
@@ -136,8 +143,7 @@ class TorchSDPABackendImpl(AttentionImpl):
143 PagedAttention.write_to_paged_cache(key, value, key_cache,
144 value_cache,
145 attn_metadata.slot_mapping,
139 - attn_metadata.kv_cache_dtype,
140 - kv_scale)
146 + self.kv_cache_dtype, kv_scale)
147
148 if attn_metadata.is_prompt:
149 assert attn_metadata.seq_lens is not None
@@ -195,7 +201,7 @@ class TorchSDPABackendImpl(AttentionImpl):
201 attn_metadata.block_tables,
202 attn_metadata.seq_lens_tensor,
203 attn_metadata.max_seq_len,
198 - attn_metadata.kv_cache_dtype,
204 + self.kv_cache_dtype,
205 self.num_kv_heads,
206 self.scale,
207 self.alibi_slopes,
vllm/attention/backends/xformers.py
+6 -6
@@ -149,15 +149,17 @@ class XFormersImpl(AttentionImpl):
149 num_kv_heads: Optional[int] = None,
150 alibi_slopes: Optional[List[float]] = None,
151 sliding_window: Optional[int] = None,
152 + kv_cache_dtype: str = "auto",
153 ) -> None:
154 self.num_heads = num_heads
155 self.head_size = head_size
156 self.scale = float(scale)
157 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
157 - self.sliding_window = sliding_window
158 if alibi_slopes is not None:
159 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
160 self.alibi_slopes = alibi_slopes
161 + self.sliding_window = sliding_window
162 + self.kv_cache_dtype = kv_cache_dtype
163
164 assert self.num_heads % self.num_kv_heads == 0
165 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -175,7 +177,7 @@ class XFormersImpl(AttentionImpl):
177 value: torch.Tensor,
178 kv_cache: Optional[torch.Tensor],
179 attn_metadata: AttentionMetadata[XFormersMetadata],
178 - kv_scale: float,
180 + kv_scale: float = 1.0,
181 ) -> torch.Tensor:
182 """Forward pass with xFormers and PagedAttention.
183
@@ -188,7 +190,6 @@ class XFormersImpl(AttentionImpl):
190 Returns:
191 shape = [num_tokens, num_heads * head_size]
192 """
191 - num_tokens, hidden_size = query.shape
193 query = query.view(-1, self.num_heads, self.head_size)
194 key = key.view(-1, self.num_kv_heads, self.head_size)
195 value = value.view(-1, self.num_kv_heads, self.head_size)
@@ -203,8 +204,7 @@ class XFormersImpl(AttentionImpl):
204 PagedAttention.write_to_paged_cache(key, value, key_cache,
205 value_cache,
206 attn_metadata.slot_mapping,
206 - attn_metadata.kv_cache_dtype,
207 - kv_scale)
207 + self.kv_cache_dtype, kv_scale)
208
209 num_prefill_tokens = attn_metadata.num_prefill_tokens
210 num_decode_tokens = attn_metadata.num_decode_tokens
@@ -262,7 +262,7 @@ class XFormersImpl(AttentionImpl):
262 decode_meta.block_tables,
263 decode_meta.seq_lens_tensor,
264 decode_meta.max_seq_len,
265 - attn_metadata.kv_cache_dtype,
265 + self.kv_cache_dtype,
266 self.num_kv_heads,
267 self.scale,
268 self.alibi_slopes,
vllm/attention/layer.py
+17 -2
@@ -7,6 +7,7 @@ import torch.nn as nn
7 from vllm.attention.backends.abstract import (AttentionMetadata,
8 AttentionMetadataPerStage)
9 from vllm.attention.selector import get_attn_backend
10 +from vllm.config import CacheConfig
11
12
13 class Attention(nn.Module):
@@ -29,10 +30,24 @@ class Attention(nn.Module):
30 num_kv_heads: Optional[int] = None,
31 alibi_slopes: Optional[List[float]] = None,
32 sliding_window: Optional[int] = None,
33 + cache_config: Optional[CacheConfig] = None,
34 ) -> None:
35 super().__init__()
34 - self.backend = get_attn_backend(torch.get_default_dtype())
35 - impl_cls = self.backend.get_impl_cls()
36 + if cache_config is not None:
37 + kv_cache_dtype = cache_config.cache_dtype
38 + block_size = cache_config.block_size
39 + else:
40 + kv_cache_dtype = "auto"
41 + block_size = 16
42 + if num_kv_heads is None:
43 + num_kv_heads = num_heads
44 + # During model initialization, the default dtype is set as the model
45 + # weight and activation dtype.
46 + dtype = torch.get_default_dtype()
47 + attn_backend = get_attn_backend(num_heads, head_size, num_kv_heads,
48 + sliding_window, dtype, kv_cache_dtype,
49 + block_size)
50 + impl_cls = attn_backend.get_impl_cls()
51 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
52 alibi_slopes, sliding_window)
53
vllm/attention/selector.py
+23 -5
@@ -1,6 +1,6 @@
1 import enum
2 from functools import lru_cache
3 -from typing import Type
3 +from typing import Optional, Type
4
5 import torch
6
@@ -21,8 +21,18 @@ class _Backend(enum.Enum):
21
22
23 @lru_cache(maxsize=None)
24 -def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
25 - backend = _which_attn_to_use(dtype)
24 +def get_attn_backend(
25 + num_heads: int,
26 + head_size: int,
27 + num_kv_heads: int,
28 + sliding_window: Optional[int],
29 + dtype: torch.dtype,
30 + kv_cache_dtype: Optional[str],
31 + block_size: int,
32 +) -> Type[AttentionBackend]:
33 + backend = _which_attn_to_use(num_heads, head_size, num_kv_heads,
34 + sliding_window, dtype, kv_cache_dtype,
35 + block_size)
36 if backend == _Backend.FLASH_ATTN:
37 logger.info("Using FlashAttention-2 backend.")
38 from vllm.attention.backends.flash_attn import ( # noqa: F401
@@ -44,14 +54,22 @@ def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
54 return TorchSDPABackend
55 elif backend == _Backend.FLASHINFER:
56 logger.info("Using Flashinfer backend.")
47 - logger.warning("Eager mode is enforced for the Flashinfer backend. ")
57 + logger.warning("Eager mode is enforced for the Flashinfer backend.")
58 from vllm.attention.backends.flashinfer import FlashInferBackend
59 return FlashInferBackend
60 else:
61 raise ValueError("Invalid attention backend.")
62
63
54 -def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
64 +def _which_attn_to_use(
65 + num_heads: int,
66 + head_size: int,
67 + num_kv_heads: int,
68 + sliding_window: Optional[int],
69 + dtype: torch.dtype,
70 + kv_cache_dtype: Optional[str],
71 + block_size: int,
72 +) -> _Backend:
73 """Returns which flash attention backend to use."""
74 if is_cpu():
75 return _Backend.TORCH_SDPA
vllm/model_executor/model_loader/__init__.py
+11 -8
@@ -2,26 +2,29 @@ from typing import Optional
2
3 from torch import nn
4
5 -from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
6 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
5 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 + ModelConfig, ParallelConfig, SchedulerConfig,
7 + VisionLanguageConfig)
8 from vllm.model_executor.model_loader.loader import (BaseModelLoader,
9 get_model_loader)
10 from vllm.model_executor.model_loader.utils import (
11 get_architecture_class_name, get_model_architecture)
12
13
13 -def get_model(
14 - *, model_config: ModelConfig, load_config: LoadConfig,
15 - device_config: DeviceConfig, parallel_config: ParallelConfig,
16 - scheduler_config: SchedulerConfig, lora_config: Optional[LoRAConfig],
17 - vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
14 +def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
15 + device_config: DeviceConfig, parallel_config: ParallelConfig,
16 + scheduler_config: SchedulerConfig,
17 + lora_config: Optional[LoRAConfig],
18 + vision_language_config: Optional[VisionLanguageConfig],
19 + cache_config: CacheConfig) -> nn.Module:
20 loader = get_model_loader(load_config)
21 return loader.load_model(model_config=model_config,
22 device_config=device_config,
23 lora_config=lora_config,
24 vision_language_config=vision_language_config,
25 parallel_config=parallel_config,
24 - scheduler_config=scheduler_config)
26 + scheduler_config=scheduler_config,
27 + cache_config=cache_config)
28
29
30 __all__ = [
vllm/model_executor/model_loader/loader.py
+39 -22
@@ -9,9 +9,9 @@ import huggingface_hub
9 import torch
10 from torch import nn
11
12 -from vllm.config import (DeviceConfig, LoadConfig, LoadFormat, LoRAConfig,
13 - ModelConfig, ParallelConfig, SchedulerConfig,
14 - VisionLanguageConfig)
12 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
13 + LoRAConfig, ModelConfig, ParallelConfig,
14 + SchedulerConfig, VisionLanguageConfig)
15 from vllm.envs import VLLM_USE_MODELSCOPE
16 from vllm.logger import init_logger
17 from vllm.model_executor.layers.quantization.base_config import (
@@ -77,15 +77,16 @@ def _get_model_initialization_kwargs(
77 return extra_kwargs
78
79
80 -def _initialize_model(
81 - model_config: ModelConfig, load_config: LoadConfig,
82 - lora_config: Optional[LoRAConfig],
83 - vision_language_config: Optional[VisionLanguageConfig]) -> nn.Module:
80 +def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
81 + lora_config: Optional[LoRAConfig],
82 + vision_language_config: Optional[VisionLanguageConfig],
83 + cache_config: CacheConfig) -> nn.Module:
84 """Initialize a model with the given configurations."""
85 model_class = get_model_architecture(model_config)[0]
86 quant_config = _get_quantization_config(model_config, load_config)
87
88 return model_class(config=model_config.hf_config,
89 + cache_config=cache_config,
90 quant_config=quant_config,
91 **_get_model_initialization_kwargs(
92 model_class, lora_config, vision_language_config))
@@ -103,7 +104,8 @@ class BaseModelLoader(ABC):
104 lora_config: Optional[LoRAConfig],
105 vision_language_config: Optional[VisionLanguageConfig],
106 parallel_config: ParallelConfig,
106 - scheduler_config: SchedulerConfig) -> nn.Module:
107 + scheduler_config: SchedulerConfig,
108 + cache_config: CacheConfig) -> nn.Module:
109 """Load a model with the given configurations."""
110 ...
111
@@ -216,11 +218,13 @@ class DefaultModelLoader(BaseModelLoader):
218 lora_config: Optional[LoRAConfig],
219 vision_language_config: Optional[VisionLanguageConfig],
220 parallel_config: ParallelConfig,
219 - scheduler_config: SchedulerConfig) -> nn.Module:
221 + scheduler_config: SchedulerConfig,
222 + cache_config: CacheConfig) -> nn.Module:
223 with set_default_torch_dtype(model_config.dtype):
224 with torch.device(device_config.device):
225 model = _initialize_model(model_config, self.load_config,
223 - lora_config, vision_language_config)
226 + lora_config, vision_language_config,
227 + cache_config)
228 model.load_weights(
229 self._get_weights_iterator(model_config.model,
230 model_config.revision,
@@ -253,11 +257,13 @@ class DummyModelLoader(BaseModelLoader):
257 lora_config: Optional[LoRAConfig],
258 vision_language_config: Optional[VisionLanguageConfig],
259 parallel_config: ParallelConfig,
256 - scheduler_config: SchedulerConfig) -> nn.Module:
260 + scheduler_config: SchedulerConfig,
261 + cache_config: CacheConfig) -> nn.Module:
262 with set_default_torch_dtype(model_config.dtype):
263 with torch.device(device_config.device):
264 model = _initialize_model(model_config, self.load_config,
260 - lora_config, vision_language_config)
265 + lora_config, vision_language_config,
266 + cache_config)
267 # NOTE(woosuk): For accurate performance evaluation, we assign
268 # random values to the weights.
269 initialize_dummy_weights(model)
@@ -286,9 +292,12 @@ class TensorizerLoader(BaseModelLoader):
292 return tensorizer_weights_iterator(tensorizer_args)
293
294 def _load_model_unserialized(
289 - self, model_config: ModelConfig, device_config: DeviceConfig,
290 - lora_config: Optional[LoRAConfig],
291 - vision_language_config: Optional[VisionLanguageConfig]
295 + self,
296 + model_config: ModelConfig,
297 + device_config: DeviceConfig,
298 + lora_config: Optional[LoRAConfig],
299 + vision_language_config: Optional[VisionLanguageConfig],
300 + cache_config: CacheConfig,
301 ) -> nn.Module:
302 """Load an unserialized model with tensorizer.
303
@@ -299,15 +308,19 @@ class TensorizerLoader(BaseModelLoader):
308 with set_default_torch_dtype(model_config.dtype):
309 with torch.device(device_config.device):
310 model = _initialize_model(model_config, self.load_config,
302 - lora_config, vision_language_config)
311 + lora_config, vision_language_config,
312 + cache_config)
313
314 model.load_weights(self._get_weights_iterator())
315 return model.eval()
316
317 def _load_model_serialized(
308 - self, model_config: ModelConfig, device_config: DeviceConfig,
309 - lora_config: Optional[LoRAConfig],
310 - vision_language_config: Optional[VisionLanguageConfig]
318 + self,
319 + model_config: ModelConfig,
320 + device_config: DeviceConfig,
321 + lora_config: Optional[LoRAConfig],
322 + vision_language_config: Optional[VisionLanguageConfig],
323 + cache_config: CacheConfig,
324 ) -> nn.Module:
325 """Load a serialized model with tensorizer.
326
@@ -321,6 +334,7 @@ class TensorizerLoader(BaseModelLoader):
334 extra_kwargs = _get_model_initialization_kwargs(
335 model_class, lora_config, vision_language_config)
336 extra_kwargs["quant_config"] = quant_config
337 + extra_kwargs["cache_config"] = cache_config
338
339 tensorizer_config = copy.copy(self.tensorizer_config)
340 tensorizer_config.model_class = model_class
@@ -335,16 +349,19 @@ class TensorizerLoader(BaseModelLoader):
349 lora_config: Optional[LoRAConfig],
350 vision_language_config: Optional[VisionLanguageConfig],
351 parallel_config: ParallelConfig,
338 - scheduler_config: SchedulerConfig) -> nn.Module:
352 + scheduler_config: SchedulerConfig,
353 + cache_config: CacheConfig) -> nn.Module:
354 self._verify_config(model_config, parallel_config)
355
356 if is_vllm_serialized_tensorizer(self.tensorizer_config):
357 return self._load_model_serialized(model_config, device_config,
358 lora_config,
344 - vision_language_config)
359 + vision_language_config,
360 + cache_config)
361 return self._load_model_unserialized(model_config, device_config,
362 lora_config,
347 - vision_language_config)
363 + vision_language_config,
364 + cache_config)
365
366
367 def get_model_loader(load_config: LoadConfig) -> BaseModelLoader:
vllm/model_executor/models/arctic.py
+13 -3
@@ -5,6 +5,7 @@ import torch
5 from torch import nn
6
7 from vllm.attention import Attention, AttentionMetadata
8 +from vllm.config import CacheConfig
9 from vllm.distributed import (get_tensor_model_parallel_rank,
10 get_tensor_model_parallel_world_size,
11 tensor_model_parallel_all_reduce)
@@ -215,6 +216,7 @@ class ArcticAttention(nn.Module):
216 self,
217 config: ArcticConfig,
218 layer_idx: Optional[int] = None,
219 + cache_config: Optional[CacheConfig] = None,
220 quant_config: Optional[QuantizationConfig] = None,
221 ):
222 super().__init__()
@@ -265,7 +267,8 @@ class ArcticAttention(nn.Module):
267 self.attn = Attention(self.num_heads,
268 self.head_dim,
269 self.scaling,
268 - num_kv_heads=self.num_kv_heads)
270 + num_kv_heads=self.num_kv_heads,
271 + cache_config=cache_config)
272
273 def forward(
274 self,
@@ -288,6 +291,7 @@ class ArcticDecoderLayer(nn.Module):
291 self,
292 config: ArcticConfig,
293 layer_idx: int,
294 + cache_config: Optional[CacheConfig] = None,
295 quant_config: Optional[QuantizationConfig] = None,
296 ) -> None:
297 super().__init__()
@@ -297,6 +301,7 @@ class ArcticDecoderLayer(nn.Module):
301 self.use_residual = config.use_residual and is_moe_layer
302 self.self_attn = ArcticAttention(config,
303 layer_idx,
304 + cache_config,
305 quant_config=quant_config)
306 self.block_sparse_moe = ArcticMoE(
307 config,
@@ -356,6 +361,7 @@ class ArcticModel(nn.Module):
361 def __init__(
362 self,
363 config: ArcticConfig,
364 + cache_config: Optional[CacheConfig] = None,
365 quant_config: Optional[QuantizationConfig] = None,
366 ) -> None:
367 super().__init__()
@@ -366,7 +372,10 @@ class ArcticModel(nn.Module):
372 config.hidden_size,
373 org_num_embeddings=self.vocab_size)
374 self.layers = nn.ModuleList([
369 - ArcticDecoderLayer(config, layer_idx, quant_config=quant_config)
375 + ArcticDecoderLayer(config,
376 + layer_idx,
377 + cache_config,
378 + quant_config=quant_config)
379 for layer_idx in range(config.num_hidden_layers)
380 ])
381 self._attn_implementation = config._attn_implementation
@@ -392,11 +401,12 @@ class ArcticForCausalLM(nn.Module):
401
402 def __init__(self,
403 config: ArcticConfig,
404 + cache_config: Optional[CacheConfig] = None,
405 quant_config: Optional[QuantizationConfig] = None,
406 **kwargs) -> None:
407 super().__init__()
408 self.config = config
399 - self.model = ArcticModel(config, quant_config)
409 + self.model = ArcticModel(config, cache_config, quant_config)
410 self.vocab_size = config.vocab_size
411 self.lm_head = ParallelLMHead(
412 self.vocab_size,
vllm/model_executor/models/baichuan.py
+22 -7
@@ -26,7 +26,7 @@ from torch import nn
26 from transformers import PretrainedConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 -from vllm.config import LoRAConfig
29 +from vllm.config import CacheConfig, LoRAConfig
30 from vllm.distributed import (get_tensor_model_parallel_rank,
31 get_tensor_model_parallel_world_size)
32 from vllm.model_executor.layers.activation import SiluAndMul
@@ -111,6 +111,7 @@ class BaiChuanAttention(nn.Module):
111 position_embedding: str,
112 rope_theta: float = 10000,
113 max_position_embeddings: int = 8192,
114 + cache_config: Optional[CacheConfig] = None,
115 quant_config: Optional[QuantizationConfig] = None,
116 ):
117 super().__init__()
@@ -162,7 +163,10 @@ class BaiChuanAttention(nn.Module):
163 base=self.rope_theta,
164 )
165 self.scaling = self.head_dim**-0.5
165 - self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
166 + self.attn = Attention(self.num_heads,
167 + self.head_dim,
168 + self.scaling,
169 + cache_config=cache_config)
170
171 def forward(
172 self,
@@ -185,6 +189,7 @@ class BaiChuanDecoderLayer(nn.Module):
189 def __init__(self,
190 config: PretrainedConfig,
191 position_embedding: str,
192 + cache_config: Optional[CacheConfig] = None,
193 quant_config: Optional[QuantizationConfig] = None):
194 super().__init__()
195 self.hidden_size = config.hidden_size
@@ -197,6 +202,7 @@ class BaiChuanDecoderLayer(nn.Module):
202 position_embedding=position_embedding,
203 rope_theta=rope_theta,
204 max_position_embeddings=max_position_embeddings,
205 + cache_config=cache_config,
206 quant_config=quant_config,
207 )
208 self.mlp = BaiChuanMLP(
@@ -244,6 +250,7 @@ class BaiChuanModel(nn.Module):
250 def __init__(self,
251 config: PretrainedConfig,
252 position_embedding: str,
253 + cache_config: Optional[CacheConfig] = None,
254 quant_config: Optional[QuantizationConfig] = None):
255 super().__init__()
256 self.config = config
@@ -255,7 +262,8 @@ class BaiChuanModel(nn.Module):
262 config.hidden_size,
263 )
264 self.layers = nn.ModuleList([
258 - BaiChuanDecoderLayer(config, position_embedding, quant_config)
265 + BaiChuanDecoderLayer(config, position_embedding, cache_config,
266 + quant_config)
267 for _ in range(config.num_hidden_layers)
268 ])
269 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -304,13 +312,15 @@ class BaiChuanBaseForCausalLM(nn.Module):
312 self,
313 config,
314 position_embedding: str,
315 + cache_config: Optional[CacheConfig] = None,
316 quant_config: Optional[QuantizationConfig] = None,
317 lora_config: Optional[LoRAConfig] = None,
318 ):
319 super().__init__()
320 self.config = config
321 self.quant_config = quant_config
313 - self.model = BaiChuanModel(config, position_embedding, quant_config)
322 + self.model = BaiChuanModel(config, position_embedding, cache_config,
323 + quant_config)
324 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
325 self.logits_processor = LogitsProcessor(config.vocab_size)
326 self.sampler = Sampler()
@@ -389,13 +399,16 @@ class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
399 def __init__(
400 self,
401 config,
402 + cache_config: Optional[CacheConfig] = None,
403 quant_config: Optional[QuantizationConfig] = None,
404 lora_config: Optional[LoRAConfig] = None,
405 ):
406 if config.hidden_size == 4096: # baichuan2 7b
396 - super().__init__(config, "ROPE", quant_config, lora_config)
407 + super().__init__(config, "ROPE", cache_config, quant_config,
408 + lora_config)
409 else: # baichuan 13b, baichuan2 13b
398 - super().__init__(config, "ALIBI", quant_config, lora_config)
410 + super().__init__(config, "ALIBI", cache_config, quant_config,
411 + lora_config)
412
413
414 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
@@ -404,7 +417,9 @@ class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
417 def __init__(
418 self,
419 config,
420 + cache_config: Optional[CacheConfig] = None,
421 quant_config: Optional[QuantizationConfig] = None,
422 lora_config: Optional[LoRAConfig] = None,
423 ):
410 - super().__init__(config, "ROPE", quant_config, lora_config)
424 + super().__init__(config, "ROPE", cache_config, quant_config,
425 + lora_config)
vllm/model_executor/models/bloom.py
+11 -4
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import BloomConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.config import CacheConfig
28 from vllm.distributed import (get_tensor_model_parallel_rank,
29 get_tensor_model_parallel_world_size)
30 from vllm.model_executor.layers.activation import get_act_fn
@@ -71,6 +72,7 @@ class BloomAttention(nn.Module):
72 def __init__(
73 self,
74 config: BloomConfig,
75 + cache_config: Optional[CacheConfig] = None,
76 quant_config: Optional[QuantizationConfig] = None,
77 ):
78 super().__init__()
@@ -108,7 +110,8 @@ class BloomAttention(nn.Module):
110 self.attn = Attention(self.num_heads,
111 self.head_dim,
112 scaling,
111 - alibi_slopes=alibi_slopes)
113 + alibi_slopes=alibi_slopes,
114 + cache_config=cache_config)
115
116 def forward(
117 self,
@@ -158,6 +161,7 @@ class BloomBlock(nn.Module):
161 def __init__(
162 self,
163 config: BloomConfig,
164 + cache_config: Optional[CacheConfig] = None,
165 quant_config: Optional[QuantizationConfig] = None,
166 ):
167 super().__init__()
@@ -165,7 +169,8 @@ class BloomBlock(nn.Module):
169
170 self.input_layernorm = nn.LayerNorm(hidden_size,
171 eps=config.layer_norm_epsilon)
168 - self.self_attention = BloomAttention(config, quant_config)
172 + self.self_attention = BloomAttention(config, cache_config,
173 + quant_config)
174 self.post_attention_layernorm = nn.LayerNorm(
175 hidden_size, eps=config.layer_norm_epsilon)
176 self.mlp = BloomMLP(config, quant_config)
@@ -214,6 +219,7 @@ class BloomModel(nn.Module):
219 def __init__(
220 self,
221 config: BloomConfig,
222 + cache_config: Optional[CacheConfig] = None,
223 quant_config: Optional[QuantizationConfig] = None,
224 ):
225 super().__init__()
@@ -229,7 +235,7 @@ class BloomModel(nn.Module):
235
236 # Transformer blocks
237 self.h = nn.ModuleList([
232 - BloomBlock(config, quant_config)
238 + BloomBlock(config, cache_config, quant_config)
239 for _ in range(config.num_hidden_layers)
240 ])
241
@@ -262,12 +268,13 @@ class BloomForCausalLM(nn.Module):
268 def __init__(
269 self,
270 config: BloomConfig,
271 + cache_config: Optional[CacheConfig] = None,
272 quant_config: Optional[QuantizationConfig] = None,
273 ):
274 super().__init__()
275 self.config = config
276 self.quant_config = quant_config
270 - self.transformer = BloomModel(config, quant_config)
277 + self.transformer = BloomModel(config, cache_config, quant_config)
278 self.lm_head_weight = self.transformer.word_embeddings.weight
279 self.logits_processor = LogitsProcessor(config.vocab_size)
280 self.sampler = Sampler()
vllm/model_executor/models/chatglm.py
+14 -6
@@ -9,7 +9,7 @@ from torch import nn
9 from torch.nn import LayerNorm
10
11 from vllm.attention import Attention, AttentionMetadata
12 -from vllm.config import LoRAConfig
12 +from vllm.config import CacheConfig, LoRAConfig
13 from vllm.distributed import get_tensor_model_parallel_world_size
14 from vllm.model_executor.layers.activation import SiluAndMul
15 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -34,6 +34,7 @@ class GLMAttention(nn.Module):
34 def __init__(
35 self,
36 config,
37 + cache_config: Optional[CacheConfig] = None,
38 quant_config: Optional[QuantizationConfig] = None,
39 ):
40 super().__init__()
@@ -90,6 +91,7 @@ class GLMAttention(nn.Module):
91 self.head_dim,
92 self.scaling,
93 num_kv_heads=self.num_kv_heads,
94 + cache_config=cache_config,
95 )
96
97 def forward(
@@ -167,6 +169,7 @@ class GLMBlock(nn.Module):
169 def __init__(
170 self,
171 config,
172 + cache_config: Optional[CacheConfig] = None,
173 quant_config: Optional[QuantizationConfig] = None,
174 ):
175 super().__init__()
@@ -181,7 +184,7 @@ class GLMBlock(nn.Module):
184 eps=config.layernorm_epsilon)
185
186 # Self attention.
184 - self.self_attention = GLMAttention(config, quant_config)
187 + self.self_attention = GLMAttention(config, cache_config, quant_config)
188 self.hidden_dropout = config.hidden_dropout
189
190 # Layernorm on the attention output
@@ -237,6 +240,7 @@ class GLMTransformer(nn.Module):
240 def __init__(
241 self,
242 config,
243 + cache_config: Optional[CacheConfig] = None,
244 quant_config: Optional[QuantizationConfig] = None,
245 ):
246 super().__init__()
@@ -246,8 +250,10 @@ class GLMTransformer(nn.Module):
250 self.num_layers = config.num_layers
251
252 # Transformer layers.
249 - self.layers = nn.ModuleList(
250 - [GLMBlock(config, quant_config) for i in range(self.num_layers)])
253 + self.layers = nn.ModuleList([
254 + GLMBlock(config, cache_config, quant_config)
255 + for i in range(self.num_layers)
256 + ])
257
258 if self.post_layer_norm:
259 layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -282,6 +288,7 @@ class ChatGLMModel(nn.Module):
288 def __init__(
289 self,
290 config,
291 + cache_config: Optional[CacheConfig] = None,
292 quant_config: Optional[QuantizationConfig] = None,
293 ):
294 super().__init__()
@@ -292,7 +299,7 @@ class ChatGLMModel(nn.Module):
299 self.num_layers = config.num_layers
300 self.multi_query_group_num = config.multi_query_group_num
301 self.kv_channels = config.kv_channels
295 - self.encoder = GLMTransformer(config, quant_config)
302 + self.encoder = GLMTransformer(config, cache_config, quant_config)
303
304 self.output_layer = ParallelLMHead(config.padded_vocab_size,
305 config.hidden_size)
@@ -334,13 +341,14 @@ class ChatGLMForCausalLM(nn.Module):
341 def __init__(
342 self,
343 config: ChatGLMConfig,
344 + cache_config: Optional[CacheConfig] = None,
345 quant_config: Optional[QuantizationConfig] = None,
346 lora_config: Optional[LoRAConfig] = None,
347 ):
348 super().__init__()
349 self.config: ChatGLMConfig = config
350 self.quant_config = quant_config
343 - self.transformer = ChatGLMModel(config, quant_config)
351 + self.transformer = ChatGLMModel(config, cache_config, quant_config)
352 self.lm_head_weight = self.transformer.output_layer.weight
353 self.logits_processor = LogitsProcessor(config.padded_vocab_size)
354 self.sampler = Sampler()
vllm/model_executor/models/commandr.py
+11 -3
@@ -29,6 +29,7 @@ from torch.nn.parameter import Parameter
29 from transformers import CohereConfig
30
31 from vllm.attention import Attention, AttentionMetadata
32 +from vllm.config import CacheConfig
33 from vllm.distributed import (get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -124,6 +125,7 @@ class CohereAttention(nn.Module):
125 def __init__(
126 self,
127 config: CohereConfig,
128 + cache_config: Optional[CacheConfig] = None,
129 quant_config: Optional[QuantizationConfig] = None,
130 ):
131 super().__init__()
@@ -180,6 +182,7 @@ class CohereAttention(nn.Module):
182 self.head_dim,
183 self.scaling,
184 num_kv_heads=self.num_kv_heads,
185 + cache_config=cache_config,
186 )
187 if self.use_qk_norm:
188 self.q_norm = LayerNorm(param_shape=(self.num_heads,
@@ -219,11 +222,14 @@ class CohereDecoderLayer(nn.Module):
222
223 def __init__(self,
224 config: CohereConfig,
225 + cache_config: Optional[CacheConfig] = None,
226 quant_config: Optional[QuantizationConfig] = None):
227 super().__init__()
228 self.hidden_size = config.hidden_size
229
226 - self.self_attn = CohereAttention(config, quant_config=quant_config)
230 + self.self_attn = CohereAttention(config,
231 + cache_config,
232 + quant_config=quant_config)
233
234 self.mlp = CohereMLP(config, quant_config=quant_config)
235 self.input_layernorm = LayerNorm(param_shape=(config.hidden_size),
@@ -258,6 +264,7 @@ class CohereModel(nn.Module):
264 def __init__(
265 self,
266 config: CohereConfig,
267 + cache_config: Optional[CacheConfig] = None,
268 quant_config: Optional[QuantizationConfig] = None,
269 ):
270 super().__init__()
@@ -266,7 +273,7 @@ class CohereModel(nn.Module):
273 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
274 config.hidden_size)
275 self.layers = nn.ModuleList([
269 - CohereDecoderLayer(config, quant_config=quant_config)
276 + CohereDecoderLayer(config, cache_config, quant_config=quant_config)
277 for _ in range(config.num_hidden_layers)
278 ])
279 self.norm = LayerNorm(param_shape=(config.hidden_size),
@@ -299,6 +306,7 @@ class CohereForCausalLM(nn.Module):
306 def __init__(
307 self,
308 config: CohereConfig,
309 + cache_config: Optional[CacheConfig] = None,
310 quant_config: Optional[QuantizationConfig] = None,
311 ) -> None:
312 super().__init__()
@@ -306,7 +314,7 @@ class CohereForCausalLM(nn.Module):
314 self.quant_config = quant_config
315 self.logits_processor = LogitsProcessor(config.vocab_size,
316 scale=config.logit_scale)
309 - self.model = CohereModel(config, quant_config)
317 + self.model = CohereModel(config, cache_config, quant_config)
318 self.sampler = Sampler()
319
320 @torch.no_grad()
vllm/model_executor/models/dbrx.py
+13 -4
@@ -5,6 +5,7 @@ import torch
5 import torch.nn as nn
6
7 from vllm.attention import Attention, AttentionMetadata
8 +from vllm.config import CacheConfig
9 from vllm.distributed import (get_tensor_model_parallel_rank,
10 get_tensor_model_parallel_world_size,
11 tensor_model_parallel_all_reduce)
@@ -166,6 +167,7 @@ class DbrxAttention(nn.Module):
167 def __init__(
168 self,
169 config: DbrxConfig,
170 + cache_config: Optional[CacheConfig] = None,
171 quant_config: Optional[QuantizationConfig] = None,
172 ):
173 super().__init__()
@@ -221,6 +223,7 @@ class DbrxAttention(nn.Module):
223 self.head_dim,
224 self.scaling,
225 num_kv_heads=self.num_kv_heads,
226 + cache_config=cache_config,
227 )
228
229 def forward(
@@ -279,10 +282,12 @@ class DbrxBlock(nn.Module):
282 def __init__(
283 self,
284 config: DbrxConfig,
285 + cache_config: Optional[CacheConfig] = None,
286 quant_config: Optional[QuantizationConfig] = None,
287 ):
288 super().__init__()
285 - self.norm_attn_norm = DbrxFusedNormAttention(config, quant_config)
289 + self.norm_attn_norm = DbrxFusedNormAttention(config, cache_config,
290 + quant_config)
291 self.ffn = DbrxExperts(config, quant_config)
292
293 def forward(
@@ -308,6 +313,7 @@ class DbrxModel(nn.Module):
313 def __init__(
314 self,
315 config: DbrxConfig,
316 + cache_config: Optional[CacheConfig] = None,
317 quant_config: Optional[QuantizationConfig] = None,
318 ):
319 super().__init__()
@@ -315,8 +321,10 @@ class DbrxModel(nn.Module):
321 config.vocab_size,
322 config.d_model,
323 )
318 - self.blocks = nn.ModuleList(
319 - [DbrxBlock(config, quant_config) for _ in range(config.n_layers)])
324 + self.blocks = nn.ModuleList([
325 + DbrxBlock(config, cache_config, quant_config)
326 + for _ in range(config.n_layers)
327 + ])
328 self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
329 for module in self.modules():
330 if hasattr(module, "bias") and isinstance(module.bias,
@@ -349,13 +357,14 @@ class DbrxForCausalLM(nn.Module):
357 def __init__(
358 self,
359 config: DbrxConfig,
360 + cache_config: Optional[CacheConfig] = None,
361 quant_config: Optional[QuantizationConfig] = None,
362 ):
363 super().__init__()
364 self.config = config
365 self.quant_config = quant_config
366 self.unpadded_vocab_size = config.vocab_size
358 - self.transformer = DbrxModel(config, quant_config)
367 + self.transformer = DbrxModel(config, cache_config, quant_config)
368 self.lm_head = ParallelLMHead(
369 config.vocab_size,
370 config.d_model,
vllm/model_executor/models/decilm.py
+3 -1
@@ -28,7 +28,7 @@ from typing import Iterable, Optional, Tuple
28 import torch
29 from transformers import PretrainedConfig
30
31 -from vllm.config import LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig
32 from vllm.model_executor.layers.quantization.base_config import (
33 QuantizationConfig)
34 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -56,12 +56,14 @@ class DeciLMForCausalLM(LlamaForCausalLM):
56 def __init__(
57 self,
58 config: Optional[PretrainedConfig] = None,
59 + cache_config: Optional[CacheConfig] = None,
60 quant_config: Optional[QuantizationConfig] = None,
61 lora_config: Optional[LoRAConfig] = None,
62 ) -> None:
63 config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
64 delattr(config, "num_key_value_heads_per_layer")
65 super().__init__(config=config,
66 + cache_config=cache_config,
67 quant_config=quant_config,
68 lora_config=lora_config)
69
vllm/model_executor/models/deepseek.py
+13 -3
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import CacheConfig
32 from vllm.distributed import (get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size,
34 tensor_model_parallel_all_reduce)
@@ -178,6 +179,7 @@ class DeepseekAttention(nn.Module):
179 rope_theta: float = 10000,
180 rope_scaling: Optional[Dict[str, Any]] = None,
181 max_position_embeddings: int = 8192,
182 + cache_config: Optional[CacheConfig] = None,
183 quant_config: Optional[QuantizationConfig] = None,
184 ) -> None:
185 super().__init__()
@@ -229,7 +231,8 @@ class DeepseekAttention(nn.Module):
231 self.attn = Attention(self.num_heads,
232 self.head_dim,
233 self.scaling,
232 - num_kv_heads=self.num_kv_heads)
234 + num_kv_heads=self.num_kv_heads,
235 + cache_config=cache_config)
236
237 def forward(
238 self,
@@ -252,6 +255,7 @@ class DeepseekDecoderLayer(nn.Module):
255 self,
256 config: PretrainedConfig,
257 layer_idx: int,
258 + cache_config: Optional[CacheConfig] = None,
259 quant_config: Optional[QuantizationConfig] = None,
260 ) -> None:
261 super().__init__()
@@ -267,6 +271,7 @@ class DeepseekDecoderLayer(nn.Module):
271 rope_theta=rope_theta,
272 rope_scaling=rope_scaling,
273 max_position_embeddings=max_position_embeddings,
274 + cache_config=cache_config,
275 quant_config=quant_config,
276 )
277 if (config.n_routed_experts is not None
@@ -321,6 +326,7 @@ class DeepseekModel(nn.Module):
326 def __init__(
327 self,
328 config: PretrainedConfig,
329 + cache_config: Optional[CacheConfig] = None,
330 quant_config: Optional[QuantizationConfig] = None,
331 ) -> None:
332 super().__init__()
@@ -332,7 +338,10 @@ class DeepseekModel(nn.Module):
338 config.hidden_size,
339 )
340 self.layers = nn.ModuleList([
335 - DeepseekDecoderLayer(config, layer_idx, quant_config=quant_config)
341 + DeepseekDecoderLayer(config,
342 + layer_idx,
343 + cache_config,
344 + quant_config=quant_config)
345 for layer_idx in range(config.num_hidden_layers)
346 ])
347 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -360,12 +369,13 @@ class DeepseekForCausalLM(nn.Module):
369 def __init__(
370 self,
371 config: PretrainedConfig,
372 + cache_config: Optional[CacheConfig] = None,
373 quant_config: Optional[QuantizationConfig] = None,
374 ) -> None:
375 super().__init__()
376 self.config = config
377 self.quant_config = quant_config
368 - self.model = DeepseekModel(config, quant_config)
378 + self.model = DeepseekModel(config, cache_config, quant_config)
379 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
380 self.logits_processor = LogitsProcessor(config.vocab_size)
381 self.sampler = Sampler()
vllm/model_executor/models/falcon.py
+11 -4
@@ -27,6 +27,7 @@ from torch.nn import LayerNorm
27 from transformers import FalconConfig as HF_FalconConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 +from vllm.config import CacheConfig
31 from vllm.distributed import (get_tensor_model_parallel_rank,
32 get_tensor_model_parallel_world_size,
33 tensor_model_parallel_all_reduce)
@@ -77,6 +78,7 @@ class FalconAttention(nn.Module):
78 def __init__(
79 self,
80 config: FalconConfig,
81 + cache_config: Optional[CacheConfig] = None,
82 quant_config: Optional[QuantizationConfig] = None,
83 ):
84 super().__init__()
@@ -168,7 +170,8 @@ class FalconAttention(nn.Module):
170 self.attn = Attention(self.num_heads,
171 self.head_dim,
172 scale=self.inv_norm_factor,
171 - num_kv_heads=self.num_kv_heads)
173 + num_kv_heads=self.num_kv_heads,
174 + cache_config=cache_config)
175
176 def forward(
177 self,
@@ -229,12 +232,14 @@ class FalconDecoderLayer(nn.Module):
232 def __init__(
233 self,
234 config: FalconConfig,
235 + cache_config: Optional[CacheConfig] = None,
236 quant_config: Optional[QuantizationConfig] = None,
237 ):
238 super().__init__()
239 hidden_size = config.hidden_size
240 self.num_heads = config.num_attention_heads
237 - self.self_attention = FalconAttention(config, quant_config)
241 + self.self_attention = FalconAttention(config, cache_config,
242 + quant_config)
243 self.mlp = FalconMLP(config, quant_config)
244 self.config = config
245
@@ -311,6 +316,7 @@ class FalconModel(nn.Module):
316 def __init__(
317 self,
318 config: FalconConfig,
319 + cache_config: Optional[CacheConfig] = None,
320 quant_config: Optional[QuantizationConfig] = None,
321 ):
322 super().__init__()
@@ -327,7 +333,7 @@ class FalconModel(nn.Module):
333
334 # Transformer blocks
335 self.h = nn.ModuleList([
330 - FalconDecoderLayer(config, quant_config)
336 + FalconDecoderLayer(config, cache_config, quant_config)
337 for _ in range(config.num_hidden_layers)
338 ])
339
@@ -359,12 +365,13 @@ class FalconForCausalLM(nn.Module):
365 def __init__(
366 self,
367 config: FalconConfig,
368 + cache_config: Optional[CacheConfig] = None,
369 quant_config: Optional[QuantizationConfig] = None,
370 ):
371 super().__init__()
372 self.config = config
373 self.quant_config = quant_config
367 - self.transformer = FalconModel(config, quant_config)
374 + self.transformer = FalconModel(config, cache_config, quant_config)
375 self.lm_head_weight = self.transformer.word_embeddings.weight
376 self.logits_processor = LogitsProcessor(config.vocab_size)
377 self.sampler = Sampler()
vllm/model_executor/models/gemma.py
+10 -4
@@ -22,7 +22,7 @@ from torch import nn
22 from transformers import GemmaConfig
23
24 from vllm.attention import Attention, AttentionMetadata
25 -from vllm.config import LoRAConfig
25 +from vllm.config import CacheConfig, LoRAConfig
26 from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.activation import GeluAndMul
@@ -107,6 +107,7 @@ class GemmaAttention(nn.Module):
107 head_dim: int,
108 max_position_embeddings: int = 8192,
109 rope_theta: float = 10000,
110 + cache_config: Optional[CacheConfig] = None,
111 quant_config: Optional[QuantizationConfig] = None) -> None:
112 super().__init__()
113 self.hidden_size = hidden_size
@@ -155,7 +156,8 @@ class GemmaAttention(nn.Module):
156 self.attn = Attention(self.num_heads,
157 self.head_dim,
158 self.scaling,
158 - num_kv_heads=self.num_kv_heads)
159 + num_kv_heads=self.num_kv_heads,
160 + cache_config=cache_config)
161
162 def forward(
163 self,
@@ -177,6 +179,7 @@ class GemmaDecoderLayer(nn.Module):
179 def __init__(
180 self,
181 config: GemmaConfig,
182 + cache_config: Optional[CacheConfig] = None,
183 quant_config: Optional[QuantizationConfig] = None,
184 ) -> None:
185 super().__init__()
@@ -188,6 +191,7 @@ class GemmaDecoderLayer(nn.Module):
191 head_dim=config.head_dim,
192 max_position_embeddings=config.max_position_embeddings,
193 rope_theta=config.rope_theta,
194 + cache_config=cache_config,
195 quant_config=quant_config,
196 )
197 self.mlp = GemmaMLP(
@@ -236,6 +240,7 @@ class GemmaModel(nn.Module):
240 def __init__(
241 self,
242 config: GemmaConfig,
243 + cache_config: Optional[CacheConfig] = None,
244 quant_config: Optional[QuantizationConfig] = None,
245 ) -> None:
246 super().__init__()
@@ -246,7 +251,7 @@ class GemmaModel(nn.Module):
251 config.hidden_size,
252 )
253 self.layers = nn.ModuleList([
249 - GemmaDecoderLayer(config, quant_config)
254 + GemmaDecoderLayer(config, cache_config, quant_config)
255 for _ in range(config.num_hidden_layers)
256 ])
257 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -309,6 +314,7 @@ class GemmaForCausalLM(nn.Module):
314 def __init__(
315 self,
316 config: GemmaConfig,
317 + cache_config: Optional[CacheConfig] = None,
318 quant_config: Optional[QuantizationConfig] = None,
319 lora_config: Optional[LoRAConfig] = None,
320 ) -> None:
@@ -316,7 +322,7 @@ class GemmaForCausalLM(nn.Module):
322 super().__init__()
323 self.config = config
324 self.quant_config = quant_config
319 - self.model = GemmaModel(config, quant_config)
325 + self.model = GemmaModel(config, cache_config, quant_config)
326 self.logits_processor = LogitsProcessor(config.vocab_size)
327 self.sampler = Sampler()
328
vllm/model_executor/models/gpt2.py
+12 -4
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import GPT2Config
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.config import CacheConfig
28 from vllm.distributed import get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -45,6 +46,7 @@ class GPT2Attention(nn.Module):
46 def __init__(
47 self,
48 config: GPT2Config,
49 + cache_config: Optional[CacheConfig] = None,
50 quant_config: Optional[QuantizationConfig] = None,
51 ):
52 super().__init__()
@@ -70,7 +72,10 @@ class GPT2Attention(nn.Module):
72 bias=True,
73 quant_config=quant_config,
74 )
73 - self.attn = Attention(self.num_heads, self.head_dim, scale=self.scale)
75 + self.attn = Attention(self.num_heads,
76 + self.head_dim,
77 + scale=self.scale,
78 + cache_config=cache_config)
79
80 def forward(
81 self,
@@ -122,6 +127,7 @@ class GPT2Block(nn.Module):
127 def __init__(
128 self,
129 config: GPT2Config,
130 + cache_config: Optional[CacheConfig] = None,
131 quant_config: Optional[QuantizationConfig] = None,
132 ):
133 super().__init__()
@@ -130,7 +136,7 @@ class GPT2Block(nn.Module):
136 hidden_size)
137
138 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
133 - self.attn = GPT2Attention(config, quant_config)
139 + self.attn = GPT2Attention(config, cache_config, quant_config)
140 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
141 self.mlp = GPT2MLP(inner_dim, config, quant_config)
142
@@ -163,6 +169,7 @@ class GPT2Model(nn.Module):
169 def __init__(
170 self,
171 config: GPT2Config,
172 + cache_config: Optional[CacheConfig] = None,
173 quant_config: Optional[QuantizationConfig] = None,
174 ):
175 super().__init__()
@@ -174,7 +181,7 @@ class GPT2Model(nn.Module):
181 self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
182 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
183 self.h = nn.ModuleList([
177 - GPT2Block(config, quant_config)
184 + GPT2Block(config, cache_config, quant_config)
185 for _ in range(config.num_hidden_layers)
186 ])
187 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -203,12 +210,13 @@ class GPT2LMHeadModel(nn.Module):
210 def __init__(
211 self,
212 config: GPT2Config,
213 + cache_config: Optional[CacheConfig] = None,
214 quant_config: Optional[QuantizationConfig] = None,
215 ):
216 super().__init__()
217 self.config = config
218 self.quant_config = quant_config
211 - self.transformer = GPT2Model(config, quant_config)
219 + self.transformer = GPT2Model(config, cache_config, quant_config)
220 self.lm_head_weight = self.transformer.wte.weight
221 self.logits_processor = LogitsProcessor(config.vocab_size)
222 self.sampler = Sampler()
vllm/model_executor/models/gpt_bigcode.py
+10 -4
@@ -25,6 +25,7 @@ from torch import nn
25 from transformers import GPTBigCodeConfig
26
27 from vllm.attention import Attention, AttentionMetadata
28 +from vllm.config import CacheConfig
29 from vllm.distributed import get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -46,6 +47,7 @@ class GPTBigCodeAttention(nn.Module):
47 def __init__(
48 self,
49 config: GPTBigCodeConfig,
50 + cache_config: Optional[CacheConfig] = None,
51 quant_config: Optional[QuantizationConfig] = None,
52 ):
53 super().__init__()
@@ -85,7 +87,8 @@ class GPTBigCodeAttention(nn.Module):
87 self.attn = Attention(self.num_heads,
88 self.head_dim,
89 scale=self.scale,
88 - num_kv_heads=self.num_kv_heads)
90 + num_kv_heads=self.num_kv_heads,
91 + cache_config=cache_config)
92
93 def forward(
94 self,
@@ -143,6 +146,7 @@ class GPTBigCodeBlock(nn.Module):
146 def __init__(
147 self,
148 config: GPTBigCodeConfig,
149 + cache_config: Optional[CacheConfig] = None,
150 quant_config: Optional[QuantizationConfig] = None,
151 ):
152 super().__init__()
@@ -151,7 +155,7 @@ class GPTBigCodeBlock(nn.Module):
155 hidden_size)
156
157 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
154 - self.attn = GPTBigCodeAttention(config, quant_config)
158 + self.attn = GPTBigCodeAttention(config, cache_config, quant_config)
159 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
160 self.mlp = GPTBigMLP(inner_dim, config, quant_config)
161
@@ -184,6 +188,7 @@ class GPTBigCodeModel(nn.Module):
188 def __init__(
189 self,
190 config: GPTBigCodeConfig,
191 + cache_config: Optional[CacheConfig] = None,
192 quant_config: Optional[QuantizationConfig] = None,
193 ):
194 super().__init__()
@@ -195,7 +200,7 @@ class GPTBigCodeModel(nn.Module):
200 self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
201 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
202 self.h = nn.ModuleList([
198 - GPTBigCodeBlock(config, quant_config)
203 + GPTBigCodeBlock(config, cache_config, quant_config)
204 for _ in range(config.num_hidden_layers)
205 ])
206 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -224,12 +229,13 @@ class GPTBigCodeForCausalLM(nn.Module):
229 def __init__(
230 self,
231 config: GPTBigCodeConfig,
232 + cache_config: Optional[CacheConfig] = None,
233 quant_config: Optional[QuantizationConfig] = None,
234 ):
235 super().__init__()
236 self.config = config
237 self.quant_config = quant_config
232 - self.transformer = GPTBigCodeModel(config, quant_config)
238 + self.transformer = GPTBigCodeModel(config, cache_config, quant_config)
239 self.lm_head_weight = self.transformer.wte.weight
240 self.logits_processor = LogitsProcessor(config.vocab_size)
241 self.sampler = Sampler()
vllm/model_executor/models/gpt_j.py
+15 -5
@@ -23,6 +23,7 @@ from torch import nn
23 from transformers import GPTJConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 +from vllm.config import CacheConfig
27 from vllm.distributed import get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -45,6 +46,7 @@ class GPTJAttention(nn.Module):
46 def __init__(
47 self,
48 config: GPTJConfig,
49 + cache_config: Optional[CacheConfig] = None,
50 quant_config: Optional[QuantizationConfig] = None,
51 ):
52 super().__init__()
@@ -83,7 +85,10 @@ class GPTJAttention(nn.Module):
85 base=rope_theta,
86 is_neox_style=False,
87 )
86 - self.attn = Attention(self.num_heads, self.head_size, scaling)
88 + self.attn = Attention(self.num_heads,
89 + self.head_size,
90 + scaling,
91 + cache_config=cache_config)
92
93 def forward(
94 self,
@@ -135,13 +140,14 @@ class GPTJBlock(nn.Module):
140 def __init__(
141 self,
142 config: GPTJConfig,
143 + cache_config: Optional[CacheConfig] = None,
144 quant_config: Optional[QuantizationConfig] = None,
145 ):
146 super().__init__()
147 inner_dim = (4 * config.n_embd
148 if config.n_inner is None else config.n_inner)
149 self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
144 - self.attn = GPTJAttention(config, quant_config)
150 + self.attn = GPTJAttention(config, cache_config, quant_config)
151 self.mlp = GPTJMLP(inner_dim, config, quant_config)
152
153 def forward(
@@ -169,6 +175,7 @@ class GPTJModel(nn.Module):
175 def __init__(
176 self,
177 config: GPTJConfig,
178 + cache_config: Optional[CacheConfig] = None,
179 quant_config: Optional[QuantizationConfig] = None,
180 ):
181 super().__init__()
@@ -178,8 +185,10 @@ class GPTJModel(nn.Module):
185 config.vocab_size,
186 self.embed_dim,
187 )
181 - self.h = nn.ModuleList(
182 - [GPTJBlock(config, quant_config) for _ in range(config.n_layer)])
188 + self.h = nn.ModuleList([
189 + GPTJBlock(config, cache_config, quant_config)
190 + for _ in range(config.n_layer)
191 + ])
192 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
193
194 def forward(
@@ -207,13 +216,14 @@ class GPTJForCausalLM(nn.Module):
216 def __init__(
217 self,
218 config: GPTJConfig,
219 + cache_config: Optional[CacheConfig] = None,
220 quant_config: Optional[QuantizationConfig] = None,
221 ):
222 super().__init__()
223 self.config = config
224 self.quant_config = quant_config
225 assert not config.tie_word_embeddings
216 - self.transformer = GPTJModel(config, quant_config)
226 + self.transformer = GPTJModel(config, cache_config, quant_config)
227 self.lm_head = ParallelLMHead(
228 config.vocab_size,
229 config.n_embd,
vllm/model_executor/models/gpt_neox.py
+12 -4
@@ -23,6 +23,7 @@ from torch import nn
23 from transformers import GPTNeoXConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 +from vllm.config import CacheConfig
27 from vllm.distributed import get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -45,6 +46,7 @@ class GPTNeoXAttention(nn.Module):
46 def __init__(
47 self,
48 config: GPTNeoXConfig,
49 + cache_config: Optional[CacheConfig] = None,
50 quant_config: Optional[QuantizationConfig] = None,
51 ):
52 super().__init__()
@@ -84,7 +86,10 @@ class GPTNeoXAttention(nn.Module):
86 max_position=max_position_embeddings,
87 base=rope_theta,
88 )
87 - self.attn = Attention(self.num_heads, self.head_size, scaling)
89 + self.attn = Attention(self.num_heads,
90 + self.head_size,
91 + scaling,
92 + cache_config=cache_config)
93
94 def forward(
95 self,
@@ -134,6 +139,7 @@ class GPTNeoXLayer(nn.Module):
139 def __init__(
140 self,
141 config: GPTNeoXConfig,
142 + cache_config: Optional[CacheConfig] = None,
143 quant_config: Optional[QuantizationConfig] = None,
144 ):
145 super().__init__()
@@ -142,7 +148,7 @@ class GPTNeoXLayer(nn.Module):
148 eps=config.layer_norm_eps)
149 self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
150 eps=config.layer_norm_eps)
145 - self.attention = GPTNeoXAttention(config, quant_config)
151 + self.attention = GPTNeoXAttention(config, cache_config, quant_config)
152 self.mlp = GPTNeoXMLP(config, quant_config)
153
154 def forward(
@@ -182,6 +188,7 @@ class GPTNeoXModel(nn.Module):
188 def __init__(
189 self,
190 config: GPTNeoXConfig,
191 + cache_config: Optional[CacheConfig] = None,
192 quant_config: Optional[QuantizationConfig] = None,
193 ):
194 super().__init__()
@@ -192,7 +199,7 @@ class GPTNeoXModel(nn.Module):
199 config.hidden_size,
200 )
201 self.layers = nn.ModuleList([
195 - GPTNeoXLayer(config, quant_config)
202 + GPTNeoXLayer(config, cache_config, quant_config)
203 for _ in range(config.num_hidden_layers)
204 ])
205 self.final_layer_norm = nn.LayerNorm(config.hidden_size,
@@ -223,12 +230,13 @@ class GPTNeoXForCausalLM(nn.Module):
230 def __init__(
231 self,
232 config,
233 + cache_config: Optional[CacheConfig] = None,
234 quant_config: Optional[QuantizationConfig] = None,
235 ):
236 super().__init__()
237 self.config = config
238 self.quant_config = quant_config
231 - self.gpt_neox = GPTNeoXModel(config, quant_config)
239 + self.gpt_neox = GPTNeoXModel(config, cache_config, quant_config)
240 self.embed_out = ParallelLMHead(
241 config.vocab_size,
242 config.hidden_size,
vllm/model_executor/models/internlm2.py
+10 -3
@@ -6,6 +6,7 @@ from torch import nn
6 from transformers import PretrainedConfig
7
8 from vllm.attention import Attention, AttentionMetadata
9 +from vllm.config import CacheConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.model_executor.layers.activation import SiluAndMul
12 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -64,6 +65,7 @@ class InternLM2Attention(nn.Module):
65 rope_theta: float = 10000,
66 rope_scaling: Optional[Dict[str, Any]] = None,
67 max_position_embeddings: int = 8192,
68 + cache_config: Optional[CacheConfig] = None,
69 quant_config: Optional[QuantizationConfig] = None,
70 ) -> None:
71 super().__init__()
@@ -114,7 +116,8 @@ class InternLM2Attention(nn.Module):
116 self.attn = Attention(self.num_heads,
117 self.head_dim,
118 self.scaling,
117 - num_kv_heads=self.num_kv_heads)
119 + num_kv_heads=self.num_kv_heads,
120 + cache_config=cache_config)
121
122 def forward(
123 self,
@@ -136,6 +139,7 @@ class InternLMDecoderLayer(nn.Module):
139 def __init__(
140 self,
141 config: PretrainedConfig,
142 + cache_config: Optional[CacheConfig] = None,
143 quant_config: Optional[QuantizationConfig] = None,
144 ) -> None:
145 super().__init__()
@@ -151,6 +155,7 @@ class InternLMDecoderLayer(nn.Module):
155 rope_theta=rope_theta,
156 rope_scaling=rope_scaling,
157 max_position_embeddings=max_position_embeddings,
158 + cache_config=cache_config,
159 quant_config=quant_config,
160 )
161 self.feed_forward = InternLM2MLP(
@@ -196,6 +201,7 @@ class InternLM2Model(nn.Module):
201 def __init__(
202 self,
203 config: PretrainedConfig,
204 + cache_config: Optional[CacheConfig] = None,
205 quant_config: Optional[QuantizationConfig] = None,
206 ) -> None:
207 super().__init__()
@@ -207,7 +213,7 @@ class InternLM2Model(nn.Module):
213 config.hidden_size,
214 )
215 self.layers = nn.ModuleList([
210 - InternLMDecoderLayer(config, quant_config)
216 + InternLMDecoderLayer(config, cache_config, quant_config)
217 for _ in range(config.num_hidden_layers)
218 ])
219 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -239,12 +245,13 @@ class InternLM2ForCausalLM(nn.Module):
245 def __init__(
246 self,
247 config: PretrainedConfig,
248 + cache_config: Optional[CacheConfig] = None,
249 quant_config: Optional[QuantizationConfig] = None,
250 ) -> None:
251 super().__init__()
252 self.config = config
253 self.quant_config = quant_config
247 - self.model = InternLM2Model(config, quant_config)
254 + self.model = InternLM2Model(config, cache_config, quant_config)
255 self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
256 self.logits_processor = LogitsProcessor(config.vocab_size)
257 self.sampler = Sampler()
vllm/model_executor/models/jais.py
+9 -3
@@ -26,6 +26,7 @@ import torch
26 from torch import nn
27
28 from vllm.attention import Attention, AttentionMetadata
29 +from vllm.config import CacheConfig
30 from vllm.distributed import (get_tensor_model_parallel_rank,
31 get_tensor_model_parallel_world_size)
32 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -69,6 +70,7 @@ class JAISAttention(nn.Module):
70 def __init__(
71 self,
72 config: JAISConfig,
73 + cache_config: Optional[CacheConfig] = None,
74 quant_config: Optional[QuantizationConfig] = None,
75 ):
76 super().__init__()
@@ -108,6 +110,7 @@ class JAISAttention(nn.Module):
110 self.head_dim,
111 scale=self.scale,
112 alibi_slopes=alibi_slopes,
113 + cache_config=cache_config,
114 )
115
116 def forward(
@@ -170,6 +173,7 @@ class JAISBlock(nn.Module):
173 def __init__(
174 self,
175 config: JAISConfig,
176 + cache_config: Optional[CacheConfig] = None,
177 quant_config: Optional[QuantizationConfig] = None,
178 ):
179 super().__init__()
@@ -178,7 +182,7 @@ class JAISBlock(nn.Module):
182 hidden_size)
183
184 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
181 - self.attn = JAISAttention(config, quant_config)
185 + self.attn = JAISAttention(config, cache_config, quant_config)
186 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
187 self.mlp = JAISMLP(inner_dim, config, quant_config)
188
@@ -211,6 +215,7 @@ class JAISModel(nn.Module):
215 def __init__(
216 self,
217 config: JAISConfig,
218 + cache_config: Optional[CacheConfig] = None,
219 quant_config: Optional[QuantizationConfig] = None,
220 ):
221 super().__init__()
@@ -228,7 +233,7 @@ class JAISModel(nn.Module):
233 else:
234 self.embeddings_scale = config.mup_embeddings_scale
235 self.h = nn.ModuleList([
231 - JAISBlock(config, quant_config)
236 + JAISBlock(config, cache_config, quant_config)
237 for _ in range(config.num_hidden_layers)
238 ])
239 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -262,12 +267,13 @@ class JAISLMHeadModel(nn.Module):
267 def __init__(
268 self,
269 config: JAISConfig,
270 + cache_config: Optional[CacheConfig] = None,
271 quant_config: Optional[QuantizationConfig] = None,
272 ):
273 super().__init__()
274 self.config = config
275 self.quant_config = quant_config
270 - self.transformer = JAISModel(config, quant_config)
276 + self.transformer = JAISModel(config, cache_config, quant_config)
277 self.lm_head_weight = self.transformer.wte.weight
278 if hasattr(config, "width_scale"):
279 self.output_logits_scale = config.width_scale
vllm/model_executor/models/llama.py
+13 -4
@@ -28,7 +28,7 @@ from torch import nn
28 from transformers import LlamaConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 -from vllm.config import LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig
32 from vllm.distributed import (get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
@@ -94,6 +94,7 @@ class LlamaAttention(nn.Module):
94 quant_config: Optional[QuantizationConfig] = None,
95 bias: bool = False,
96 sliding_window: Optional[int] = None,
97 + cache_config: Optional[CacheConfig] = None,
98 ) -> None:
99 super().__init__()
100 self.hidden_size = hidden_size
@@ -153,7 +154,8 @@ class LlamaAttention(nn.Module):
154 self.head_dim,
155 self.scaling,
156 num_kv_heads=self.num_kv_heads,
156 - sliding_window=sliding_window)
157 + sliding_window=sliding_window,
158 + cache_config=cache_config)
159
160 def forward(
161 self,
@@ -176,6 +178,7 @@ class LlamaDecoderLayer(nn.Module):
178 def __init__(
179 self,
180 config: LlamaConfig,
181 + cache_config: Optional[CacheConfig] = None,
182 quant_config: Optional[QuantizationConfig] = None,
183 ) -> None:
184 super().__init__()
@@ -204,6 +207,7 @@ class LlamaDecoderLayer(nn.Module):
207 quant_config=quant_config,
208 bias=attention_bias,
209 sliding_window=sliding_window,
210 + cache_config=cache_config,
211 )
212 self.mlp = LlamaMLP(
213 hidden_size=self.hidden_size,
@@ -251,6 +255,7 @@ class LlamaModel(nn.Module):
255 def __init__(
256 self,
257 config: LlamaConfig,
258 + cache_config: Optional[CacheConfig] = None,
259 quant_config: Optional[QuantizationConfig] = None,
260 lora_config: Optional[LoRAConfig] = None,
261 ) -> None:
@@ -267,7 +272,7 @@ class LlamaModel(nn.Module):
272 org_num_embeddings=config.vocab_size,
273 )
274 self.layers = nn.ModuleList([
270 - LlamaDecoderLayer(config, quant_config)
275 + LlamaDecoderLayer(config, cache_config, quant_config)
276 for _ in range(config.num_hidden_layers)
277 ])
278 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -332,12 +337,16 @@ class LlamaForCausalLM(nn.Module):
337 def __init__(
338 self,
339 config: LlamaConfig,
340 + cache_config: Optional[CacheConfig] = None,
341 quant_config: Optional[QuantizationConfig] = None,
342 lora_config: Optional[LoRAConfig] = None,
343 ) -> None:
344 super().__init__()
345 self.config = config
340 - self.model = LlamaModel(config, quant_config, lora_config=lora_config)
346 + self.model = LlamaModel(config,
347 + cache_config,
348 + quant_config,
349 + lora_config=lora_config)
350 self.unpadded_vocab_size = config.vocab_size
351 if lora_config:
352 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
vllm/model_executor/models/llava.py
+4 -2
@@ -7,7 +7,7 @@ from torch import nn
7 from transformers import CLIPVisionModel, LlavaConfig
8
9 from vllm.attention import AttentionMetadata
10 -from vllm.config import VisionLanguageConfig
10 +from vllm.config import CacheConfig, VisionLanguageConfig
11 from vllm.model_executor.layers.activation import get_act_fn
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 from vllm.model_executor.layers.quantization.base_config import (
@@ -62,6 +62,7 @@ class LlavaForConditionalGeneration(nn.Module):
62 def __init__(self,
63 config: "LlavaConfig",
64 vision_language_config: VisionLanguageConfig,
65 + cache_config: Optional[CacheConfig] = None,
66 quant_config: Optional["QuantizationConfig"] = None) -> None:
67 super().__init__()
68 self.config = config
@@ -85,7 +86,8 @@ class LlavaForConditionalGeneration(nn.Module):
86 projector_hidden_act=config.projector_hidden_act)
87
88 self.quant_config = quant_config
88 - self.language_model = LlamaModel(config.text_config, quant_config)
89 + self.language_model = LlamaModel(config.text_config, cache_config,
90 + quant_config)
91 self.unpadded_vocab_size = config.text_config.vocab_size
92 self.lm_head = ParallelLMHead(
93 self.unpadded_vocab_size,
vllm/model_executor/models/minicpm.py
+10 -3
@@ -28,7 +28,7 @@ import torch
28 from torch import nn
29
30 from vllm.attention import Attention, AttentionMetadata
31 -from vllm.config import LoRAConfig
31 +from vllm.config import CacheConfig, LoRAConfig
32 from vllm.distributed import (get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size,
34 tensor_model_parallel_all_reduce)
@@ -181,6 +181,7 @@ class MiniCPMAttention(nn.Module):
181 rope_theta: float = 10000,
182 rope_scaling: Optional[Dict[str, Any]] = None,
183 max_position_embeddings: int = 8192,
184 + cache_config: Optional[CacheConfig] = None,
185 quant_config: Optional[QuantizationConfig] = None,
186 ) -> None:
187 super().__init__()
@@ -234,7 +235,8 @@ class MiniCPMAttention(nn.Module):
235 self.attn = Attention(self.num_heads,
236 self.head_dim,
237 self.scaling,
237 - num_kv_heads=self.num_kv_heads)
238 + num_kv_heads=self.num_kv_heads,
239 + cache_config=cache_config)
240
241 def forward(
242 self,
@@ -259,6 +261,7 @@ class MiniCPMDecoderLayer(nn.Module):
261 def __init__(
262 self,
263 config,
264 + cache_config: Optional[CacheConfig] = None,
265 quant_config: Optional[QuantizationConfig] = None,
266 ) -> None:
267 super().__init__()
@@ -275,6 +278,7 @@ class MiniCPMDecoderLayer(nn.Module):
278 rope_theta=rope_theta,
279 rope_scaling=rope_scaling,
280 max_position_embeddings=max_position_embeddings,
281 + cache_config=cache_config,
282 quant_config=quant_config,
283 )
284 self.num_experts = getattr(self.config, "num_experts", 0)
@@ -330,6 +334,7 @@ class MiniCPMModel(nn.Module):
334 def __init__(
335 self,
336 config,
337 + cache_config: Optional[CacheConfig] = None,
338 quant_config: Optional[QuantizationConfig] = None,
339 lora_config: Optional[LoRAConfig] = None,
340 ) -> None:
@@ -346,7 +351,7 @@ class MiniCPMModel(nn.Module):
351 org_num_embeddings=config.vocab_size,
352 )
353 self.layers = nn.ModuleList([
349 - MiniCPMDecoderLayer(config, quant_config)
354 + MiniCPMDecoderLayer(config, cache_config, quant_config)
355 for _ in range(config.num_hidden_layers)
356 ])
357 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -413,6 +418,7 @@ class MiniCPMForCausalLM(nn.Module):
418 def __init__(
419 self,
420 config,
421 + cache_config: Optional[CacheConfig] = None,
422 quant_config: Optional[QuantizationConfig] = None,
423 lora_config: Optional[LoRAConfig] = None,
424 ) -> None:
@@ -421,6 +427,7 @@ class MiniCPMForCausalLM(nn.Module):
427 self.num_experts = getattr(self.config, "num_experts", 0)
428 self.quant_config = quant_config
429 self.model = MiniCPMModel(config,
430 + cache_config,
431 quant_config,
432 lora_config=lora_config)
433 unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/mixtral.py
+11 -2
@@ -29,7 +29,7 @@ from transformers import MixtralConfig
29
30 from vllm import _custom_ops as ops
31 from vllm.attention import Attention, AttentionMetadata
32 -from vllm.config import LoRAConfig
32 +from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import (get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size,
35 tensor_model_parallel_all_reduce)
@@ -252,6 +252,7 @@ class MixtralAttention(nn.Module):
252 num_kv_heads: int,
253 max_position: int = 4096 * 32,
254 rope_theta: float = 10000,
255 + cache_config: Optional[CacheConfig] = None,
256 quant_config: Optional[QuantizationConfig] = None,
257 sliding_window: Optional[int] = None) -> None:
258 super().__init__()
@@ -313,6 +314,7 @@ class MixtralAttention(nn.Module):
314 self.scaling,
315 num_kv_heads=self.num_kv_heads,
316 sliding_window=self.sliding_window,
317 + cache_config=cache_config,
318 )
319
320 def forward(
@@ -335,6 +337,7 @@ class MixtralDecoderLayer(nn.Module):
337 def __init__(
338 self,
339 config: MixtralConfig,
340 + cache_config: Optional[CacheConfig] = None,
341 quant_config: Optional[QuantizationConfig] = None,
342 ) -> None:
343 super().__init__()
@@ -348,6 +351,7 @@ class MixtralDecoderLayer(nn.Module):
351 num_kv_heads=config.num_key_value_heads,
352 rope_theta=rope_theta,
353 sliding_window=config.sliding_window,
354 + cache_config=cache_config,
355 quant_config=quant_config)
356 self.block_sparse_moe = MixtralMoE(
357 num_experts=config.num_local_experts,
@@ -394,6 +398,7 @@ class MixtralModel(nn.Module):
398 def __init__(
399 self,
400 config: MixtralConfig,
401 + cache_config: Optional[CacheConfig] = None,
402 quant_config: Optional[QuantizationConfig] = None,
403 lora_config: Optional[LoRAConfig] = None,
404 ) -> None:
@@ -410,7 +415,9 @@ class MixtralModel(nn.Module):
415 org_num_embeddings=config.vocab_size,
416 )
417 self.layers = nn.ModuleList([
413 - MixtralDecoderLayer(config, quant_config=quant_config)
418 + MixtralDecoderLayer(config,
419 + cache_config,
420 + quant_config=quant_config)
421 for _ in range(config.num_hidden_layers)
422 ])
423 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -460,12 +467,14 @@ class MixtralForCausalLM(nn.Module):
467 def __init__(
468 self,
469 config: MixtralConfig,
470 + cache_config: Optional[CacheConfig] = None,
471 quant_config: Optional[QuantizationConfig] = None,
472 lora_config: Optional[LoRAConfig] = None,
473 ) -> None:
474 super().__init__()
475 self.config = config
476 self.model = MixtralModel(config,
477 + cache_config,
478 quant_config,
479 lora_config=lora_config)
480 self.unpadded_vocab_size = config.vocab_size
vllm/model_executor/models/mixtral_quant.py
+21 -10
@@ -30,6 +30,7 @@ from torch import nn
30 from transformers import MixtralConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 +from vllm.config import CacheConfig
34 from vllm.distributed import (get_tensor_model_parallel_rank,
35 get_tensor_model_parallel_world_size,
36 tensor_model_parallel_all_reduce)
@@ -157,14 +158,17 @@ class MixtralMoE(nn.Module):
158
159 class MixtralAttention(nn.Module):
160
160 - def __init__(self,
161 - hidden_size: int,
162 - num_heads: int,
163 - num_kv_heads: int,
164 - max_position: int = 4096 * 32,
165 - rope_theta: float = 10000,
166 - quant_config: Optional[QuantizationConfig] = None,
167 - sliding_window: Optional[int] = None) -> None:
161 + def __init__(
162 + self,
163 + hidden_size: int,
164 + num_heads: int,
165 + num_kv_heads: int,
166 + max_position: int = 4096 * 32,
167 + rope_theta: float = 10000,
168 + quant_config: Optional[QuantizationConfig] = None,
169 + sliding_window: Optional[int] = None,
170 + cache_config: Optional[CacheConfig] = None,
171 + ) -> None:
172 super().__init__()
173 self.hidden_size = hidden_size
174 tp_size = get_tensor_model_parallel_world_size()
@@ -215,6 +219,7 @@ class MixtralAttention(nn.Module):
219 self.scaling,
220 num_kv_heads=self.num_kv_heads,
221 sliding_window=self.sliding_window,
222 + cache_config=cache_config,
223 )
224
225 def forward(
@@ -237,6 +242,7 @@ class MixtralDecoderLayer(nn.Module):
242 def __init__(
243 self,
244 config: MixtralConfig,
245 + cache_config: Optional[CacheConfig] = None,
246 quant_config: Optional[QuantizationConfig] = None,
247 ) -> None:
248 super().__init__()
@@ -250,6 +256,7 @@ class MixtralDecoderLayer(nn.Module):
256 num_kv_heads=config.num_key_value_heads,
257 rope_theta=rope_theta,
258 sliding_window=config.sliding_window,
259 + cache_config=cache_config,
260 quant_config=quant_config)
261 self.block_sparse_moe = MixtralMoE(config=config,
262 quant_config=quant_config)
@@ -292,6 +299,7 @@ class MixtralModel(nn.Module):
299 def __init__(
300 self,
301 config: MixtralConfig,
302 + cache_config: Optional[CacheConfig] = None,
303 quant_config: Optional[QuantizationConfig] = None,
304 ) -> None:
305 super().__init__()
@@ -303,7 +311,9 @@ class MixtralModel(nn.Module):
311 config.hidden_size,
312 )
313 self.layers = nn.ModuleList([
306 - MixtralDecoderLayer(config, quant_config=quant_config)
314 + MixtralDecoderLayer(config,
315 + cache_config,
316 + quant_config=quant_config)
317 for _ in range(config.num_hidden_layers)
318 ])
319 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -332,12 +342,13 @@ class MixtralForCausalLM(nn.Module):
342 def __init__(
343 self,
344 config: MixtralConfig,
345 + cache_config: Optional[CacheConfig] = None,
346 quant_config: Optional[QuantizationConfig] = None,
347 ) -> None:
348 super().__init__()
349 self.config = config
350 self.quant_config = quant_config
340 - self.model = MixtralModel(config, quant_config)
351 + self.model = MixtralModel(config, cache_config, quant_config)
352 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
353 self.logits_processor = LogitsProcessor(config.vocab_size)
354 self.sampler = Sampler()
vllm/model_executor/models/mpt.py
+13 -5
@@ -7,6 +7,7 @@ import torch
7 import torch.nn as nn
8
9 from vllm.attention import Attention, AttentionMetadata
10 +from vllm.config import CacheConfig
11 from vllm.distributed import (get_tensor_model_parallel_rank,
12 get_tensor_model_parallel_world_size)
13 from vllm.model_executor.layers.activation import get_act_fn
@@ -43,6 +44,7 @@ class MPTAttention(nn.Module):
44 def __init__(
45 self,
46 config: MPTConfig,
47 + cache_config: Optional[CacheConfig] = None,
48 quant_config: Optional[QuantizationConfig] = None,
49 ):
50 super().__init__()
@@ -107,7 +109,8 @@ class MPTAttention(nn.Module):
109 self.head_dim,
110 scaling,
111 alibi_slopes=alibi_slopes,
110 - num_kv_heads=self.num_kv_heads)
112 + num_kv_heads=self.num_kv_heads,
113 + cache_config=cache_config)
114
115 def forward(
116 self,
@@ -166,12 +169,13 @@ class MPTBlock(nn.Module):
169 def __init__(
170 self,
171 config: MPTConfig,
172 + cache_config: Optional[CacheConfig] = None,
173 quant_config: Optional[QuantizationConfig] = None,
174 ):
175 super().__init__()
176 hidden_size = config.d_model
177 self.norm_1 = nn.LayerNorm(hidden_size)
174 - self.attn = MPTAttention(config, quant_config)
178 + self.attn = MPTAttention(config, cache_config, quant_config)
179 self.norm_2 = nn.LayerNorm(hidden_size)
180 self.ffn = MPTMLP(config, quant_config)
181
@@ -201,6 +205,7 @@ class MPTModel(nn.Module):
205 def __init__(
206 self,
207 config: MPTConfig,
208 + cache_config: Optional[CacheConfig] = None,
209 quant_config: Optional[QuantizationConfig] = None,
210 ):
211 super().__init__()
@@ -211,8 +216,10 @@ class MPTModel(nn.Module):
216 config.vocab_size,
217 config.d_model,
218 )
214 - self.blocks = nn.ModuleList(
215 - [MPTBlock(config, quant_config) for _ in range(config.n_layers)])
219 + self.blocks = nn.ModuleList([
220 + MPTBlock(config, cache_config, quant_config)
221 + for _ in range(config.n_layers)
222 + ])
223 self.norm_f = nn.LayerNorm(config.d_model)
224 if config.no_bias:
225 for module in self.modules():
@@ -246,6 +253,7 @@ class MPTForCausalLM(nn.Module):
253 def __init__(
254 self,
255 config: MPTConfig,
256 + cache_config: Optional[CacheConfig] = None,
257 quant_config: Optional[QuantizationConfig] = None,
258 ):
259 super().__init__()
@@ -253,7 +261,7 @@ class MPTForCausalLM(nn.Module):
261 assert config.tie_word_embeddings
262 self.quant_config = quant_config
263
256 - self.transformer = MPTModel(config, quant_config)
264 + self.transformer = MPTModel(config, cache_config, quant_config)
265 self.lm_head_weight = self.transformer.wte.weight
266 self.logits_processor = LogitsProcessor(config.vocab_size)
267 self.sampler = Sampler()
vllm/model_executor/models/olmo.py
+10 -4
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers import OlmoConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.config import CacheConfig
32 from vllm.distributed import get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -55,6 +56,7 @@ class OlmoAttention(nn.Module):
56 def __init__(
57 self,
58 config: OlmoConfig,
59 + cache_config: Optional[CacheConfig] = None,
60 quant_config: Optional[QuantizationConfig] = None,
61 ):
62 super().__init__()
@@ -93,7 +95,8 @@ class OlmoAttention(nn.Module):
95 self.scaling = self.head_dim**-0.5
96 self.attn = Attention(self.num_heads,
97 self.head_dim,
96 - scale=self.scaling)
98 + scale=self.scaling,
99 + cache_config=cache_config)
100
101 # Attention output projection.
102 self.o_proj = RowParallelLinear(
@@ -175,10 +178,11 @@ class OlmoDecoderLayer(nn.Module):
178
179 def __init__(self,
180 config: OlmoConfig,
181 + cache_config: Optional[CacheConfig] = None,
182 quant_config: Optional[QuantizationConfig] = None):
183 super().__init__()
184 # Attention block.
181 - self.self_attn = OlmoAttention(config, quant_config)
185 + self.self_attn = OlmoAttention(config, cache_config, quant_config)
186
187 # MLP block.
188 self.mlp = OlmoMLP(config, quant_config)
@@ -217,6 +221,7 @@ class OlmoModel(nn.Module):
221
222 def __init__(self,
223 config: OlmoConfig,
224 + cache_config: Optional[CacheConfig] = None,
225 quant_config: Optional[QuantizationConfig] = None):
226 super().__init__()
227 self.config = config
@@ -224,7 +229,7 @@ class OlmoModel(nn.Module):
229 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
230 config.hidden_size)
231 self.layers = nn.ModuleList([
227 - OlmoDecoderLayer(config, quant_config)
232 + OlmoDecoderLayer(config, cache_config, quant_config)
233 for layer_idx in range(config.num_hidden_layers)
234 ])
235 self.norm = nn.LayerNorm(config.hidden_size,
@@ -271,10 +276,11 @@ class OlmoForCausalLM(nn.Module):
276
277 def __init__(self,
278 config: OlmoConfig,
279 + cache_config: Optional[CacheConfig] = None,
280 quant_config: Optional[QuantizationConfig] = None):
281 super().__init__()
282 self.config = config
277 - self.model = OlmoModel(config, quant_config)
283 + self.model = OlmoModel(config, cache_config, quant_config)
284 if config.tie_word_embeddings:
285 self.lm_head_weight = self.model.embed_tokens.weight
286 else:
vllm/model_executor/models/opt.py
+12 -4
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import OPTConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.config import CacheConfig
28 from vllm.distributed import get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -61,6 +62,7 @@ class OPTAttention(nn.Module):
62 embed_dim: int,
63 num_heads: int,
64 bias: bool = True,
65 + cache_config: Optional[CacheConfig] = None,
66 quant_config: Optional[QuantizationConfig] = None,
67 ) -> None:
68 super().__init__()
@@ -88,7 +90,8 @@ class OPTAttention(nn.Module):
90 )
91 self.attn = Attention(self.num_heads,
92 self.head_dim,
91 - scale=self.scaling)
93 + scale=self.scaling,
94 + cache_config=cache_config)
95
96 def forward(
97 self,
@@ -108,6 +111,7 @@ class OPTDecoderLayer(nn.Module):
111 def __init__(
112 self,
113 config: OPTConfig,
114 + cache_config: Optional[CacheConfig] = None,
115 quant_config: Optional[QuantizationConfig] = None,
116 ):
117 super().__init__()
@@ -117,6 +121,7 @@ class OPTDecoderLayer(nn.Module):
121 embed_dim=self.embed_dim,
122 num_heads=config.num_attention_heads,
123 bias=config.enable_bias,
124 + cache_config=cache_config,
125 quant_config=quant_config,
126 )
127 self.do_layer_norm_before = config.do_layer_norm_before
@@ -181,6 +186,7 @@ class OPTDecoder(nn.Module):
186 def __init__(
187 self,
188 config: OPTConfig,
189 + cache_config: Optional[CacheConfig] = None,
190 quant_config: Optional[QuantizationConfig] = None,
191 ):
192 super().__init__()
@@ -226,7 +232,7 @@ class OPTDecoder(nn.Module):
232 self.final_layer_norm = None
233
234 self.layers = nn.ModuleList([
229 - OPTDecoderLayer(config, quant_config)
235 + OPTDecoderLayer(config, cache_config, quant_config)
236 for _ in range(config.num_hidden_layers)
237 ])
238
@@ -259,10 +265,11 @@ class OPTModel(nn.Module):
265 def __init__(
266 self,
267 config: OPTConfig,
268 + cache_config: Optional[CacheConfig] = None,
269 quant_config: Optional[QuantizationConfig] = None,
270 ):
271 super().__init__()
265 - self.decoder = OPTDecoder(config, quant_config)
272 + self.decoder = OPTDecoder(config, cache_config, quant_config)
273
274 def forward(
275 self,
@@ -279,12 +286,13 @@ class OPTForCausalLM(nn.Module):
286 def __init__(
287 self,
288 config,
289 + cache_config: Optional[CacheConfig] = None,
290 quant_config: Optional[QuantizationConfig] = None,
291 ):
292 super().__init__()
293 self.config = config
294 self.quant_config = quant_config
287 - self.model = OPTModel(config, quant_config)
295 + self.model = OPTModel(config, cache_config, quant_config)
296 self.lm_head_weight = self.model.decoder.embed_tokens.weight
297 self.logits_processor = LogitsProcessor(config.vocab_size)
298 self.sampler = Sampler()
vllm/model_executor/models/orion.py
+10 -3
@@ -11,6 +11,7 @@ from torch import nn
11 from transformers import PretrainedConfig
12
13 from vllm.attention import Attention, AttentionMetadata
14 +from vllm.config import CacheConfig
15 from vllm.distributed import get_tensor_model_parallel_world_size
16 from vllm.model_executor.layers.activation import SiluAndMul
17 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -68,6 +69,7 @@ class OrionAttention(nn.Module):
69 rope_theta: float = 10000,
70 rope_scaling: Optional[Dict[str, Any]] = None,
71 max_position_embeddings: int = 8192,
72 + cache_config: Optional[CacheConfig] = None,
73 quant_config: Optional[QuantizationConfig] = None,
74 ) -> None:
75 super().__init__()
@@ -118,7 +120,8 @@ class OrionAttention(nn.Module):
120 self.attn = Attention(self.num_heads,
121 self.head_dim,
122 self.scaling,
121 - num_kv_heads=self.num_kv_heads)
123 + num_kv_heads=self.num_kv_heads,
124 + cache_config=cache_config)
125
126 def forward(
127 self,
@@ -140,6 +143,7 @@ class OrionDecoderLayer(nn.Module):
143 def __init__(
144 self,
145 config: PretrainedConfig,
146 + cache_config: Optional[CacheConfig] = None,
147 quant_config: Optional[QuantizationConfig] = None,
148 ) -> None:
149 super().__init__()
@@ -155,6 +159,7 @@ class OrionDecoderLayer(nn.Module):
159 rope_theta=rope_theta,
160 rope_scaling=rope_scaling,
161 max_position_embeddings=max_position_embeddings,
162 + cache_config=cache_config,
163 quant_config=quant_config,
164 )
165 self.mlp = OrionMLP(
@@ -202,6 +207,7 @@ class OrionModel(nn.Module):
207 def __init__(
208 self,
209 config: PretrainedConfig,
210 + cache_config: Optional[CacheConfig] = None,
211 quant_config: Optional[QuantizationConfig] = None,
212 ) -> None:
213 super().__init__()
@@ -213,7 +219,7 @@ class OrionModel(nn.Module):
219 config.hidden_size,
220 )
221 self.layers = nn.ModuleList([
216 - OrionDecoderLayer(config, quant_config)
222 + OrionDecoderLayer(config, cache_config, quant_config)
223 for _ in range(config.num_hidden_layers)
224 ])
225 self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -245,12 +251,13 @@ class OrionForCausalLM(nn.Module):
251 def __init__(
252 self,
253 config: PretrainedConfig,
254 + cache_config: Optional[CacheConfig] = None,
255 quant_config: Optional[QuantizationConfig] = None,
256 ) -> None:
257 super().__init__()
258 self.config = config
259 self.quant_config = quant_config
253 - self.model = OrionModel(config, quant_config)
260 + self.model = OrionModel(config, cache_config, quant_config)
261 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
262 self.logits_processor = LogitsProcessor(config.vocab_size)
263 self.sampler = Sampler()
vllm/model_executor/models/phi.py
+12 -4
@@ -42,6 +42,7 @@ from torch import nn
42 from transformers import PretrainedConfig
43
44 from vllm.attention import Attention, AttentionMetadata
45 +from vllm.config import CacheConfig
46 from vllm.distributed import get_tensor_model_parallel_world_size
47 from vllm.model_executor.layers.activation import get_act_fn
48 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -63,6 +64,7 @@ class PhiAttention(nn.Module):
64
65 def __init__(self,
66 config: PretrainedConfig,
67 + cache_config: Optional[CacheConfig] = None,
68 quant_config: Optional[QuantizationConfig] = None):
69 super().__init__()
70 self.total_num_heads = config.num_attention_heads
@@ -105,7 +107,10 @@ class PhiAttention(nn.Module):
107 max_position=max_position_embeddings,
108 base=rope_theta,
109 )
108 - self.attn = Attention(self.num_heads, self.head_size, scaling)
110 + self.attn = Attention(self.num_heads,
111 + self.head_size,
112 + scaling,
113 + cache_config=cache_config)
114
115 def forward(
116 self,
@@ -155,11 +160,12 @@ class PhiLayer(nn.Module):
160
161 def __init__(self,
162 config: PretrainedConfig,
163 + cache_config: Optional[CacheConfig] = None,
164 quant_config: Optional[QuantizationConfig] = None):
165 super().__init__()
166 self.input_layernorm = nn.LayerNorm(config.hidden_size,
167 eps=config.layer_norm_eps)
162 - self.self_attn = PhiAttention(config, quant_config)
168 + self.self_attn = PhiAttention(config, cache_config, quant_config)
169 self.mlp = PhiMLP(config, quant_config)
170
171 def forward(
@@ -186,6 +192,7 @@ class PhiModel(nn.Module):
192
193 def __init__(self,
194 config: PretrainedConfig,
195 + cache_config: Optional[CacheConfig] = None,
196 quant_config: Optional[QuantizationConfig] = None):
197 super().__init__()
198 self.config = config
@@ -193,7 +200,7 @@ class PhiModel(nn.Module):
200 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
201 config.hidden_size)
202 self.layers = nn.ModuleList([
196 - PhiLayer(config, quant_config)
203 + PhiLayer(config, cache_config, quant_config)
204 for _ in range(config.num_hidden_layers)
205 ])
206 self.final_layernorm = nn.LayerNorm(config.hidden_size,
@@ -225,12 +232,13 @@ class PhiForCausalLM(nn.Module):
232
233 def __init__(self,
234 config: PretrainedConfig,
235 + cache_config: Optional[CacheConfig] = None,
236 quant_config: Optional[QuantizationConfig] = None):
237 super().__init__()
238 self.config = config
239 self.quant_config = quant_config
240
233 - self.model = PhiModel(config, quant_config)
241 + self.model = PhiModel(config, cache_config, quant_config)
242
243 self.lm_head = ParallelLMHead(config.vocab_size,
244 config.hidden_size,
vllm/model_executor/models/qwen.py
+12 -3
@@ -11,6 +11,7 @@ from torch import nn
11 from transformers import PretrainedConfig
12
13 from vllm.attention import Attention, AttentionMetadata
14 +from vllm.config import CacheConfig
15 from vllm.distributed import get_tensor_model_parallel_world_size
16 from vllm.model_executor.layers.activation import SiluAndMul
17 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -68,6 +69,7 @@ class QWenAttention(nn.Module):
69 max_position_embeddings: int,
70 rope_theta: float = 10000,
71 rope_scaling: Optional[Dict[str, Any]] = None,
72 + cache_config: Optional[CacheConfig] = None,
73 quant_config: Optional[QuantizationConfig] = None,
74 ):
75 super().__init__()
@@ -101,7 +103,10 @@ class QWenAttention(nn.Module):
103 base=rope_theta,
104 rope_scaling=rope_scaling,
105 )
104 - self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
106 + self.attn = Attention(self.num_heads,
107 + self.head_dim,
108 + self.scaling,
109 + cache_config=cache_config)
110
111 def forward(
112 self,
@@ -123,6 +128,7 @@ class QWenBlock(nn.Module):
128 def __init__(
129 self,
130 config: PretrainedConfig,
131 + cache_config: Optional[CacheConfig] = None,
132 quant_config: Optional[QuantizationConfig] = None,
133 ):
134 super().__init__()
@@ -135,6 +141,7 @@ class QWenBlock(nn.Module):
141 config.max_position_embeddings,
142 rope_theta=rope_theta,
143 rope_scaling=rope_scaling,
144 + cache_config=cache_config,
145 quant_config=quant_config)
146
147 self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -175,6 +182,7 @@ class QWenModel(nn.Module):
182 def __init__(
183 self,
184 config: PretrainedConfig,
185 + cache_config: Optional[CacheConfig] = None,
186 quant_config: Optional[QuantizationConfig] = None,
187 ):
188 super().__init__()
@@ -186,7 +194,7 @@ class QWenModel(nn.Module):
194 config.hidden_size,
195 )
196 self.h = nn.ModuleList([
189 - QWenBlock(config, quant_config)
197 + QWenBlock(config, cache_config, quant_config)
198 for _ in range(config.num_hidden_layers)
199 ])
200 self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -218,12 +226,13 @@ class QWenLMHeadModel(nn.Module):
226 def __init__(
227 self,
228 config: PretrainedConfig,
229 + cache_config: Optional[CacheConfig] = None,
230 quant_config: Optional[QuantizationConfig] = None,
231 ):
232 super().__init__()
233 self.config = config
234 self.quant_config = quant_config
226 - self.transformer = QWenModel(config, quant_config)
235 + self.transformer = QWenModel(config, cache_config, quant_config)
236 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
237 self.logits_processor = LogitsProcessor(config.vocab_size)
238 self.sampler = Sampler()
vllm/model_executor/models/qwen2.py
+10 -4
@@ -29,7 +29,7 @@ from torch import nn
29 from transformers import Qwen2Config
30
31 from vllm.attention import Attention, AttentionMetadata
32 -from vllm.config import LoRAConfig
32 +from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -87,6 +87,7 @@ class Qwen2Attention(nn.Module):
87 max_position: int = 4096 * 32,
88 rope_theta: float = 10000,
89 use_sliding_window: bool = False,
90 + cache_config: Optional[CacheConfig] = None,
91 quant_config: Optional[QuantizationConfig] = None,
92 sliding_window: Optional[int] = None) -> None:
93 super().__init__()
@@ -137,7 +138,8 @@ class Qwen2Attention(nn.Module):
138 self.head_dim,
139 self.scaling,
140 num_kv_heads=self.num_kv_heads,
140 - sliding_window=self.sliding_window)
141 + sliding_window=self.sliding_window,
142 + cache_config=cache_config)
143
144 def forward(
145 self,
@@ -160,6 +162,7 @@ class Qwen2DecoderLayer(nn.Module):
162 self,
163 config: Qwen2Config,
164 layer_idx: int,
165 + cache_config: Optional[CacheConfig] = None,
166 quant_config: Optional[QuantizationConfig] = None,
167 ) -> None:
168 super().__init__()
@@ -175,6 +178,7 @@ class Qwen2DecoderLayer(nn.Module):
178 num_kv_heads=config.num_key_value_heads,
179 rope_theta=rope_theta,
180 use_sliding_window=use_sliding_window,
181 + cache_config=cache_config,
182 quant_config=quant_config,
183 sliding_window=config.sliding_window)
184 self.mlp = Qwen2MLP(
@@ -222,6 +226,7 @@ class Qwen2Model(nn.Module):
226 def __init__(
227 self,
228 config: Qwen2Config,
229 + cache_config: Optional[CacheConfig] = None,
230 quant_config: Optional[QuantizationConfig] = None,
231 ) -> None:
232 super().__init__()
@@ -234,7 +239,7 @@ class Qwen2Model(nn.Module):
239 config.hidden_size,
240 )
241 self.layers = nn.ModuleList([
237 - Qwen2DecoderLayer(config, layer_idx, quant_config)
242 + Qwen2DecoderLayer(config, layer_idx, cache_config, quant_config)
243 for layer_idx in range(config.num_hidden_layers)
244 ])
245 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -287,6 +292,7 @@ class Qwen2ForCausalLM(nn.Module):
292 def __init__(
293 self,
294 config: Qwen2Config,
295 + cache_config: Optional[CacheConfig] = None,
296 quant_config: Optional[QuantizationConfig] = None,
297 lora_config: Optional[LoRAConfig] = None,
298 ) -> None:
@@ -294,7 +300,7 @@ class Qwen2ForCausalLM(nn.Module):
300 super().__init__()
301 self.config = config
302 self.quant_config = quant_config
297 - self.model = Qwen2Model(config, quant_config)
303 + self.model = Qwen2Model(config, cache_config, quant_config)
304
305 if config.tie_word_embeddings:
306 self.lm_head_weight = self.model.embed_tokens.weight
vllm/model_executor/models/qwen2_moe.py
+13 -3
@@ -30,6 +30,7 @@ from torch import nn
30 from transformers import PretrainedConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 +from vllm.config import CacheConfig
34 from vllm.distributed import (get_tensor_model_parallel_rank,
35 get_tensor_model_parallel_world_size,
36 tensor_model_parallel_all_reduce)
@@ -187,6 +188,7 @@ class Qwen2MoeAttention(nn.Module):
188 rope_theta: float = 10000,
189 rope_scaling: Optional[Dict[str, Any]] = None,
190 max_position_embeddings: int = 8192,
191 + cache_config: Optional[CacheConfig] = None,
192 quant_config: Optional[QuantizationConfig] = None,
193 ) -> None:
194 super().__init__()
@@ -238,7 +240,8 @@ class Qwen2MoeAttention(nn.Module):
240 self.attn = Attention(self.num_heads,
241 self.head_dim,
242 self.scaling,
241 - num_kv_heads=self.num_kv_heads)
243 + num_kv_heads=self.num_kv_heads,
244 + cache_config=cache_config)
245
246 def forward(
247 self,
@@ -261,6 +264,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
264 self,
265 config: PretrainedConfig,
266 layer_idx: int,
267 + cache_config: Optional[CacheConfig] = None,
268 quant_config: Optional[QuantizationConfig] = None,
269 ) -> None:
270 super().__init__()
@@ -276,6 +280,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
280 rope_theta=rope_theta,
281 rope_scaling=rope_scaling,
282 max_position_embeddings=max_position_embeddings,
283 + cache_config=cache_config,
284 quant_config=quant_config,
285 )
286 if (config.num_experts is not None
@@ -328,6 +333,7 @@ class Qwen2MoeModel(nn.Module):
333 def __init__(
334 self,
335 config: PretrainedConfig,
336 + cache_config: Optional[CacheConfig] = None,
337 quant_config: Optional[QuantizationConfig] = None,
338 ) -> None:
339 super().__init__()
@@ -339,7 +345,10 @@ class Qwen2MoeModel(nn.Module):
345 config.hidden_size,
346 )
347 self.layers = nn.ModuleList([
342 - Qwen2MoeDecoderLayer(config, layer_idx, quant_config=quant_config)
348 + Qwen2MoeDecoderLayer(config,
349 + layer_idx,
350 + cache_config,
351 + quant_config=quant_config)
352 for layer_idx in range(config.num_hidden_layers)
353 ])
354 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -369,12 +378,13 @@ class Qwen2MoeForCausalLM(nn.Module):
378 def __init__(
379 self,
380 config: PretrainedConfig,
381 + cache_config: Optional[CacheConfig] = None,
382 quant_config: Optional[QuantizationConfig] = None,
383 ) -> None:
384 super().__init__()
385 self.config = config
386 self.quant_config = quant_config
377 - self.model = Qwen2MoeModel(config, quant_config)
387 + self.model = Qwen2MoeModel(config, cache_config, quant_config)
388 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
389 self.logits_processor = LogitsProcessor(config.vocab_size)
390 self.sampler = Sampler()
vllm/model_executor/models/stablelm.py
+10 -4
@@ -26,6 +26,7 @@ from torch import nn
26 from transformers import PretrainedConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 +from vllm.config import CacheConfig
30 from vllm.distributed import get_tensor_model_parallel_world_size
31 from vllm.model_executor.layers.activation import SiluAndMul
32 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -72,6 +73,7 @@ class StablelmAttention(nn.Module):
73
74 def __init__(self,
75 config: PretrainedConfig,
76 + cache_config: Optional[CacheConfig] = None,
77 quant_config: Optional[QuantizationConfig] = None) -> None:
78 super().__init__()
79 self.config = config
@@ -124,7 +126,8 @@ class StablelmAttention(nn.Module):
126 self.attn = Attention(self.num_heads,
127 self.head_dim,
128 self.scaling,
127 - num_kv_heads=self.num_key_value_heads)
129 + num_kv_heads=self.num_key_value_heads,
130 + cache_config=cache_config)
131
132 def forward(
133 self,
@@ -146,10 +149,11 @@ class StablelmDecoderLayer(nn.Module):
149 def __init__(
150 self,
151 config: PretrainedConfig,
152 + cache_config: Optional[CacheConfig] = None,
153 quant_config: Optional[QuantizationConfig] = None,
154 ) -> None:
155 super().__init__()
152 - self.self_attn = StablelmAttention(config)
156 + self.self_attn = StablelmAttention(config, cache_config, quant_config)
157 self.mlp = StablelmMLP(config, quant_config)
158 norm_eps = getattr(config, "norm_eps",
159 getattr(config, "layer_norm_eps", 1e-05))
@@ -188,6 +192,7 @@ class StableLMEpochModel(nn.Module):
192
193 def __init__(self,
194 config: PretrainedConfig,
195 + cache_config: Optional[CacheConfig] = None,
196 quant_config: Optional[QuantizationConfig] = None) -> None:
197 super().__init__()
198 self.embed_tokens = VocabParallelEmbedding(
@@ -195,7 +200,7 @@ class StableLMEpochModel(nn.Module):
200 config.hidden_size,
201 )
202 self.layers = nn.ModuleList([
198 - StablelmDecoderLayer(config, quant_config)
203 + StablelmDecoderLayer(config, cache_config, quant_config)
204 for _ in range(config.num_hidden_layers)
205 ])
206 norm_eps = getattr(config, "norm_eps",
@@ -227,12 +232,13 @@ class StablelmForCausalLM(nn.Module):
232 def __init__(
233 self,
234 config: PretrainedConfig,
235 + cache_config: Optional[CacheConfig] = None,
236 quant_config: Optional[QuantizationConfig] = None,
237 ) -> None:
238 super().__init__()
239 self.config = config
240 self.quant_config = quant_config
235 - self.model = StableLMEpochModel(config, quant_config)
241 + self.model = StableLMEpochModel(config, cache_config, quant_config)
242 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
243 self.logits_processor = LogitsProcessor(config.vocab_size)
244 self.sampler = Sampler()
vllm/model_executor/models/starcoder2.py
+15 -3
@@ -25,6 +25,7 @@ from torch import nn
25 from transformers import Starcoder2Config
26
27 from vllm.attention import Attention, AttentionMetadata
28 +from vllm.config import CacheConfig
29 from vllm.distributed import get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -46,6 +47,7 @@ class Starcoder2Attention(nn.Module):
47
48 def __init__(self,
49 config: Starcoder2Config,
50 + cache_config: Optional[CacheConfig] = None,
51 quant_config: Optional[QuantizationConfig] = None):
52 super().__init__()
53 self.config = config
@@ -101,6 +103,7 @@ class Starcoder2Attention(nn.Module):
103 self.scaling,
104 num_kv_heads=self.num_kv_heads,
105 sliding_window=self.sliding_window,
106 + cache_config=cache_config,
107 )
108
109 def forward(
@@ -150,10 +153,13 @@ class Starcoder2DecoderLayer(nn.Module):
153
154 def __init__(self,
155 config: Starcoder2Config,
156 + cache_config: Optional[CacheConfig] = None,
157 quant_config: Optional[QuantizationConfig] = None):
158 super().__init__()
159 self.hidden_size = config.hidden_size
156 - self.self_attn = Starcoder2Attention(config, quant_config=quant_config)
160 + self.self_attn = Starcoder2Attention(config,
161 + cache_config,
162 + quant_config=quant_config)
163 self.mlp = Starcoder2MLP(config, quant_config=quant_config)
164 self.input_layernorm = nn.LayerNorm(config.hidden_size,
165 eps=config.norm_epsilon)
@@ -191,6 +197,7 @@ class Starcoder2Model(nn.Module):
197
198 def __init__(self,
199 config: Starcoder2Config,
200 + cache_config: Optional[CacheConfig] = None,
201 quant_config: Optional[QuantizationConfig] = None):
202 super().__init__()
203 self.config = config
@@ -201,7 +208,9 @@ class Starcoder2Model(nn.Module):
208 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
209 config.hidden_size)
210 self.layers = nn.ModuleList([
204 - Starcoder2DecoderLayer(config, quant_config=quant_config)
211 + Starcoder2DecoderLayer(config,
212 + cache_config,
213 + quant_config=quant_config)
214 for _ in range(config.num_hidden_layers)
215 ])
216 self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
@@ -226,10 +235,13 @@ class Starcoder2ForCausalLM(nn.Module):
235
236 def __init__(self,
237 config: Starcoder2Config,
238 + cache_config: Optional[CacheConfig] = None,
239 quant_config: Optional[QuantizationConfig] = None):
240 super().__init__()
241 self.config = config
232 - self.model = Starcoder2Model(config, quant_config=quant_config)
242 + self.model = Starcoder2Model(config,
243 + cache_config,
244 + quant_config=quant_config)
245 self.vocab_size = config.vocab_size
246 self.unpadded_vocab_size = config.vocab_size
247 if config.tie_word_embeddings:
vllm/model_executor/models/xverse.py
+10 -4
@@ -27,7 +27,7 @@ from torch import nn
27 from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 -from vllm.config import LoRAConfig
30 +from vllm.config import CacheConfig, LoRAConfig
31 from vllm.distributed import get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import SiluAndMul
33 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -89,6 +89,7 @@ class XverseAttention(nn.Module):
89 quant_config: Optional[QuantizationConfig] = None,
90 bias: bool = False,
91 sliding_window: Optional[int] = None,
92 + cache_config: Optional[CacheConfig] = None,
93 ) -> None:
94 super().__init__()
95 self.hidden_size = hidden_size
@@ -133,7 +134,8 @@ class XverseAttention(nn.Module):
134 self.head_dim,
135 self.scaling,
136 num_kv_heads=self.num_kv_heads,
136 - sliding_window=sliding_window)
137 + sliding_window=sliding_window,
138 + cache_config=cache_config)
139
140 def forward(
141 self,
@@ -155,6 +157,7 @@ class XverseDecoderLayer(nn.Module):
157 def __init__(
158 self,
159 config: PretrainedConfig,
160 + cache_config: Optional[CacheConfig] = None,
161 quant_config: Optional[QuantizationConfig] = None,
162 ) -> None:
163 super().__init__()
@@ -175,6 +178,7 @@ class XverseDecoderLayer(nn.Module):
178 quant_config=quant_config,
179 bias=getattr(config, "bias", False),
180 sliding_window=sliding_window,
181 + cache_config=cache_config,
182 )
183 self.mlp = XverseMLP(
184 hidden_size=self.hidden_size,
@@ -221,6 +225,7 @@ class XverseModel(nn.Module):
225 def __init__(
226 self,
227 config: PretrainedConfig,
228 + cache_config: Optional[CacheConfig] = None,
229 quant_config: Optional[QuantizationConfig] = None,
230 lora_config: Optional[LoRAConfig] = None,
231 ) -> None:
@@ -237,7 +242,7 @@ class XverseModel(nn.Module):
242 org_num_embeddings=config.vocab_size,
243 )
244 self.layers = nn.ModuleList([
240 - XverseDecoderLayer(config, quant_config)
245 + XverseDecoderLayer(config, cache_config, quant_config)
246 for _ in range(config.num_hidden_layers)
247 ])
248 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -295,13 +300,14 @@ class XverseForCausalLM(nn.Module):
300 def __init__(
301 self,
302 config: PretrainedConfig,
303 + cache_config: Optional[CacheConfig] = None,
304 quant_config: Optional[QuantizationConfig] = None,
305 lora_config=None,
306 ) -> None:
307 super().__init__()
308 self.config = config
309 self.quant_config = quant_config
304 - self.model = XverseModel(config, quant_config)
310 + self.model = XverseModel(config, cache_config, quant_config)
311 self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
312 self.logits_processor = LogitsProcessor(config.vocab_size)
313 self.sampler = Sampler()
vllm/worker/cache_engine.py
+11 -3
@@ -31,7 +31,7 @@ class CacheEngine:
31
32 self.head_size = model_config.get_head_size()
33 self.num_layers = model_config.get_num_layers(parallel_config)
34 - self.num_heads = model_config.get_num_kv_heads(parallel_config)
34 + self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
35
36 self.block_size = cache_config.block_size
37 self.num_gpu_blocks = cache_config.num_gpu_blocks
@@ -43,7 +43,15 @@ class CacheEngine:
43 self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
44
45 # Get attention backend.
46 - self.attn_backend = get_attn_backend(model_config.dtype)
46 + self.attn_backend = get_attn_backend(
47 + model_config.get_num_attention_heads(parallel_config),
48 + self.head_size,
49 + self.num_kv_heads,
50 + model_config.get_sliding_window(),
51 + model_config.dtype,
52 + cache_config.cache_dtype,
53 + self.block_size,
54 + )
55
56 # Initialize the cache.
57 self.gpu_cache = self._allocate_kv_cache(self.num_gpu_blocks, "cuda")
@@ -56,7 +64,7 @@ class CacheEngine:
64 ) -> List[torch.Tensor]:
65 """Allocates KV cache on the specified device."""
66 kv_cache_shape = self.attn_backend.get_kv_cache_shape(
59 - num_blocks, self.block_size, self.num_heads, self.head_size)
67 + num_blocks, self.block_size, self.num_kv_heads, self.head_size)
68 pin_memory = is_pin_memory_available() if device == "cpu" else False
69 kv_cache: List[torch.Tensor] = []
70 for _ in range(self.num_layers):
vllm/worker/cpu_model_runner.py
+11 -4
@@ -53,7 +53,15 @@ class CPUModelRunner:
53 self.kv_cache_dtype = kv_cache_dtype
54 self.sliding_window = model_config.get_sliding_window()
55 self.block_size = cache_config.block_size
56 - self.attn_backend = get_attn_backend(self.model_config.dtype)
56 + self.attn_backend = get_attn_backend(
57 + self.model_config.get_num_attention_heads(self.parallel_config),
58 + self.model_config.get_head_size(),
59 + self.model_config.get_num_kv_heads(self.parallel_config),
60 + self.model_config.get_sliding_window(),
61 + self.model_config.dtype,
62 + self.kv_cache_dtype,
63 + self.block_size,
64 + )
65
66 # Lazy initialization.
67 self.model: nn.Module # Set after init_Model
@@ -66,7 +74,8 @@ class CPUModelRunner:
74 vision_language_config=self.vision_language_config,
75 lora_config=self.lora_config,
76 parallel_config=self.parallel_config,
69 - scheduler_config=self.scheduler_config)
77 + scheduler_config=self.scheduler_config,
78 + cache_config=self.cache_config)
79
80 def _prepare_prompt(
81 self,
@@ -158,7 +167,6 @@ class CPUModelRunner:
167 decode_metadata=None,
168 block_tables=torch.tensor([]),
169 slot_mapping=slot_mapping,
161 - kv_cache_dtype=self.kv_cache_dtype,
170 )
171 return (input_tokens, input_positions, attn_metadata, seq_lens,
172 multi_modal_input)
@@ -242,7 +250,6 @@ class CPUModelRunner:
250 prefill_metadata=None,
251 decode_metadata=None,
252 block_tables=block_tables,
245 - kv_cache_dtype=self.kv_cache_dtype,
253 )
254 return (
255 input_tokens,
vllm/worker/cpu_worker.py
+9 -1
@@ -53,7 +53,15 @@ class CPUCacheEngine:
53 self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
54
55 # Get attention backend.
56 - self.attn_backend = get_attn_backend(model_config.dtype)
56 + self.attn_backend = get_attn_backend(
57 + self.model_config.get_num_attention_heads(self.parallel_config),
58 + self.model_config.get_head_size(),
59 + self.model_config.get_num_kv_heads(self.parallel_config),
60 + self.model_config.get_sliding_window(),
61 + self.model_config.dtype,
62 + cache_config.cache_dtype,
63 + self.block_size,
64 + )
65
66 # Initialize the cache.
67 self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks)
vllm/worker/embedding_model_runner.py
-1
@@ -235,7 +235,6 @@ class EmbeddingModelRunner(ModelRunner):
235 num_decode_tokens=num_decode_tokens,
236 prefill_metadata=prefill_attn_metadata,
237 decode_metadata=decode_attn_metadata,
238 - kv_cache_dtype=self.kv_cache_dtype,
238 )
239
240 return (input_tokens, input_positions, attn_metadata, pooling_metadata,
vllm/worker/model_runner.py
+11 -4
@@ -141,10 +141,18 @@ class ModelRunner:
141 self.graph_block_tables = np.zeros(
142 (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
143 dtype=np.int32)
144 - self.attn_backend = get_attn_backend(self.model_config.dtype)
144 + self.attn_backend = get_attn_backend(
145 + self.model_config.get_num_attention_heads(self.parallel_config),
146 + self.model_config.get_head_size(),
147 + self.model_config.get_num_kv_heads(self.parallel_config),
148 + self.model_config.get_sliding_window(),
149 + self.model_config.dtype,
150 + self.kv_cache_dtype,
151 + self.block_size,
152 + )
153
154 # Lazy initialization
147 - self.model: torch.nn.Module # Set after load_model
155 + self.model: nn.Module # Set after load_model
156 # Set if the backend is flashinfer.
157 self.flashinfer_workspace_buffer: torch.Tensor
158 # Set after load_model.
@@ -160,6 +168,7 @@ class ModelRunner:
168 vision_language_config=self.vision_language_config,
169 parallel_config=self.parallel_config,
170 scheduler_config=self.scheduler_config,
171 + cache_config=self.cache_config,
172 )
173
174 self.model_memory_usage = m.consumed_memory
@@ -753,7 +762,6 @@ class ModelRunner:
762 num_decode_tokens=num_decode_tokens,
763 prefill_metadata=prefill_attn_metadata,
764 decode_metadata=decode_attn_metadata,
756 - kv_cache_dtype=self.kv_cache_dtype,
765 )
766
767 return (input_tokens, input_positions, attn_metadata,
@@ -965,7 +973,6 @@ class ModelRunner:
973 slot_mapping=slot_mapping[:batch_size],
974 prefill_metadata=None,
975 decode_metadata=decode_metadata,
968 - kv_cache_dtype=self.kv_cache_dtype,
976 )
977
978 if self.lora_config: