A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
TP/quantization/weight loading refactor part 2 - Refactor quantized linear logic and extend quantization support to all models (#1622)
Refactor the tensor parallelism, quantization, and weight-loading codes. Summary of the new features enabled by this PR: - **All models** are able to be quantized with AWQ and SqueezeLLM, and [soon GPTQ](https://github.com/vllm-project/vllm/pull/1580). - Model loading code became much simpler. - Support model parallelism for all MQA/GQA models when the number of key/value heads is smaller than the tensor parallel size.
Zhuohan Li committed
Nov 15, 2023 at 22:50 UTC
7076fa1c9f5769469bc2671afaca5af604a9bed3
36 files changed
+2159
-2508
vllm/config.py
+30
-19
@@ -140,8 +140,8 @@ class ModelConfig:
140
# FIXME(woosuk): This may not be true for all models.
141
return self.hf_config.hidden_size // self.hf_config.num_attention_heads
142
143
- def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
144
- """Returns the number of KV heads per GPU worker."""
143
+ def get_total_num_kv_heads(self) -> int:
144
+ """Returns the total number of KV heads."""
145
# For GPTBigCode & Falcon:
146
# NOTE: for falcon, when new_decoder_architecture is True, the
147
# multi_query flag is ignored and we use n_head_kv for the number of
@@ -155,23 +155,34 @@ class ModelConfig:
155
# Multi-query attention, only one KV head.
156
# Currently, tensor parallelism is not supported in this case.
157
return 1
158
- # For Falcon:
159
- if getattr(self.hf_config, "n_head_kv", None) is not None:
160
- return (self.hf_config.n_head_kv //
161
- parallel_config.tensor_parallel_size)
162
- if getattr(self.hf_config, "num_kv_heads", None) is not None:
163
- return (self.hf_config.num_kv_heads //
164
- parallel_config.tensor_parallel_size)
165
- # For LLaMA-2:
166
- if getattr(self.hf_config, "num_key_value_heads", None) is not None:
167
- return (self.hf_config.num_key_value_heads //
168
- parallel_config.tensor_parallel_size)
169
- # For ChatGLM-2:
170
- if getattr(self.hf_config, "multi_query_group_num", None) is not None:
171
- return (self.hf_config.multi_query_group_num //
172
- parallel_config.tensor_parallel_size)
173
- total_num_attention_heads = self.hf_config.num_attention_heads
174
- return total_num_attention_heads // parallel_config.tensor_parallel_size
158
+
159
+ attributes = [
160
+ # For Falcon:
161
+ "n_head_kv",
162
+ "num_kv_heads",
163
+ # For LLaMA-2:
164
+ "num_key_value_heads",
165
+ # For ChatGLM:
166
+ "multi_query_group_num",
167
+ ]
168
+ for attr in attributes:
169
+ num_kv_heads = getattr(self.hf_config, attr, None)
170
+ if num_kv_heads is not None:
171
+ return num_kv_heads
172
+
173
+ # For non-grouped-query attention models, the number of KV heads is
174
+ # equal to the number of attention heads.
175
+ return self.hf_config.num_attention_heads
176
+
177
+ def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
178
+ """Returns the number of KV heads per GPU."""
179
+ total_num_kv_heads = self.get_total_num_kv_heads()
180
+ # If tensor parallelism is used, we divide the number of KV heads by
181
+ # the tensor parallel size. We will replicate the KV heads in the
182
+ # case where the number of KV heads is smaller than the tensor
183
+ # parallel size so each GPU has at least one KV head.
184
+ return max(1,
185
+ total_num_kv_heads // parallel_config.tensor_parallel_size)
186
187
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
188
total_num_hidden_layers = self.hf_config.num_hidden_layers
vllm/engine/async_llm_engine.py
+2
-2
@@ -142,10 +142,10 @@ class RequestTracker:
142
143
self._request_streams[request_id].finish()
144
145
- def get_new_and_finished_requests(self) -> Tuple[List[dict], Set[str]]:
145
+ def get_new_and_finished_requests(self) -> Tuple[List[Dict], Set[str]]:
146
"""Get the new requests and finished requests to be
147
sent to the engine."""
148
- new_requests: List[dict] = []
148
+ new_requests: List[Dict] = []
149
finished_requests: Set[str] = set()
150
151
while not self._finished_requests.empty():
vllm/model_executor/layers/linear.py
new
+541
@@ -0,0 +1,541 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Dict, List, Optional
3
+
4
+import torch
5
+import torch.nn.functional as F
6
+from torch.nn.parameter import Parameter
7
+
8
+from vllm.model_executor.parallel_utils.parallel_state import (
9
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
10
+from vllm.model_executor.parallel_utils.communication_op import (
11
+ tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
12
+from vllm.model_executor.parallel_utils.utils import (
13
+ divide, split_tensor_along_last_dim)
14
+from vllm.model_executor.utils import set_weight_attrs
15
+from vllm.logger import init_logger
16
+
17
+logger = init_logger(__name__)
18
+
19
+
20
+class LinearMethodBase(ABC):
21
+ """Base class for different (maybe quantized) linear methods."""
22
+
23
+ @abstractmethod
24
+ def create_weights(self, input_size: int, output_size: int,
25
+ params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
26
+ """Create weights for a linear layer."""
27
+ raise NotImplementedError
28
+
29
+ @abstractmethod
30
+ def apply_weights(self,
31
+ weights: Dict[str, torch.Tensor],
32
+ x: torch.Tensor,
33
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
34
+ """Apply the weights to the input tensor."""
35
+ raise NotImplementedError
36
+
37
+
38
+class UnquantizedLinearMethod(LinearMethodBase):
39
+ """Linear method without quantization.
40
+
41
+ Args:
42
+ separate_bias_add: If true, add bias separately after matrix
43
+ multiplication.
44
+ """
45
+
46
+ def __init__(self, separate_bias_add: bool = False):
47
+ self.separate_bias_add = separate_bias_add
48
+
49
+ def create_weights(self, input_size: int, output_size: int,
50
+ params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
51
+ weight = Parameter(torch.empty(output_size,
52
+ input_size,
53
+ device=torch.cuda.current_device(),
54
+ dtype=params_dtype),
55
+ requires_grad=False)
56
+ set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
57
+ return {"weight": weight}
58
+
59
+ def apply_weights(self,
60
+ weights: Dict[str, torch.Tensor],
61
+ x: torch.Tensor,
62
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
63
+ weight = weights["weight"]
64
+ if self.separate_bias_add:
65
+ if bias:
66
+ return F.linear(x, weight) + bias
67
+ return F.linear(x, weight)
68
+ return F.linear(x, weight, bias)
69
+
70
+
71
+class ReplicatedLinear(torch.nn.Module):
72
+ """Replicated linear layer.
73
+
74
+ Args:
75
+ input_size: input dimension of the linear layer.
76
+ output_size: output dimension of the linear layer.
77
+ bias: If true, add bias.
78
+ skip_bias_add: If true, skip adding bias but instead return it.
79
+ params_dtype: Data type for the parameters.
80
+ linear_method: (Maybe quantized) linear method.
81
+ """
82
+
83
+ def __init__(
84
+ self,
85
+ input_size: int,
86
+ output_size: int,
87
+ bias: bool = True,
88
+ skip_bias_add: bool = False,
89
+ params_dtype: Optional[torch.dtype] = None,
90
+ linear_method: Optional[LinearMethodBase] = None,
91
+ ):
92
+ super().__init__()
93
+
94
+ # Keep input parameters
95
+ self.input_size = input_size
96
+ self.output_size = output_size
97
+ self.skip_bias_add = skip_bias_add
98
+ if params_dtype is None:
99
+ params_dtype = torch.get_default_dtype()
100
+ self.params_dtype = params_dtype
101
+ if linear_method is None:
102
+ linear_method = UnquantizedLinearMethod()
103
+ self.linear_method = linear_method
104
+ self.linear_weights = self.linear_method.create_weights(
105
+ self.input_size, self.output_size, self.params_dtype)
106
+ for name, weight in self.linear_weights.items():
107
+ self.register_parameter(name, weight)
108
+ if bias:
109
+ self.bias = Parameter(
110
+ torch.empty(self.output_size,
111
+ device=torch.cuda.current_device(),
112
+ dtype=self.params_dtype))
113
+ set_weight_attrs(self.bias, {"output_dim": 0})
114
+ else:
115
+ self.register_parameter("bias", None)
116
+
117
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
118
+ bias = self.bias if not self.skip_bias_add else None
119
+ output = self.linear_method.apply_weights(self.linear_weights, x, bias)
120
+ output_bias = self.bias if self.skip_bias_add else None
121
+ return output, output_bias
122
+
123
+
124
+class ColumnParallelLinear(torch.nn.Module):
125
+ """Linear layer with column parallelism.
126
+
127
+ The linear layer is defined as Y = XA + b. A is parallelized along
128
+ its second dimension as A = [A_1, ..., A_p].
129
+
130
+ Args:
131
+ input_size: first dimension of matrix A.
132
+ output_size: second dimension of matrix A.
133
+ bias: If true, add bias.
134
+ gather_output: If true, call all-gather on output and make Y available
135
+ to all GPUs, otherwise, every GPU will have its output
136
+ which is Y_i = XA_i
137
+ skip_bias_add: This was added to enable performance optimizations where
138
+ bias can be fused with other element-wise operations. we
139
+ skip adding bias but instead return it.
140
+ params_dtype: Data type for the parameters.
141
+ linear_method: (Maybe quantized) linear method.
142
+ """
143
+
144
+ def __init__(
145
+ self,
146
+ input_size: int,
147
+ output_size: int,
148
+ bias: bool = True,
149
+ gather_output: bool = False,
150
+ skip_bias_add: bool = False,
151
+ params_dtype: Optional[torch.dtype] = None,
152
+ linear_method: Optional[LinearMethodBase] = None,
153
+ ):
154
+ super().__init__()
155
+
156
+ # Keep input parameters
157
+ self.input_size = input_size
158
+ self.output_size = output_size
159
+ self.gather_output = gather_output
160
+ # Divide the weight matrix along the last dimension.
161
+ tp_size = get_tensor_model_parallel_world_size()
162
+ self.output_size_per_partition = divide(output_size, tp_size)
163
+ self.skip_bias_add = skip_bias_add
164
+ if params_dtype is None:
165
+ params_dtype = torch.get_default_dtype()
166
+ self.params_dtype = params_dtype
167
+ if linear_method is None:
168
+ linear_method = UnquantizedLinearMethod()
169
+ self.linear_method = linear_method
170
+ self.linear_weights = self.linear_method.create_weights(
171
+ self.input_size, self.output_size_per_partition, self.params_dtype)
172
+ for name, weight in self.linear_weights.items():
173
+ self.register_parameter(name, weight)
174
+ set_weight_attrs(weight, {"weight_loader": self.weight_loader})
175
+ if bias:
176
+ self.bias = Parameter(
177
+ torch.empty(self.output_size_per_partition,
178
+ device=torch.cuda.current_device(),
179
+ dtype=params_dtype))
180
+ set_weight_attrs(self.bias, {
181
+ "output_dim": 0,
182
+ "weight_loader": self.weight_loader,
183
+ })
184
+ else:
185
+ self.register_parameter("bias", None)
186
+
187
+ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
188
+ tp_rank = get_tensor_model_parallel_rank()
189
+ output_dim = getattr(param, "output_dim", None)
190
+ param_data = param.data
191
+ if output_dim is not None:
192
+ shard_size = param_data.shape[output_dim]
193
+ start_idx = tp_rank * shard_size
194
+ loaded_weight = loaded_weight.narrow(output_dim, start_idx,
195
+ shard_size)
196
+ assert param_data.shape == loaded_weight.shape
197
+ param_data.copy_(loaded_weight)
198
+
199
+ def forward(self, input_):
200
+ bias = self.bias if not self.skip_bias_add else None
201
+
202
+ # Matrix multiply.
203
+ output_parallel = self.linear_method.apply_weights(
204
+ self.linear_weights, input_, bias)
205
+ if self.gather_output:
206
+ # All-gather across the partitions.
207
+ output = tensor_model_parallel_all_gather(output_parallel)
208
+ else:
209
+ output = output_parallel
210
+ output_bias = self.bias if self.skip_bias_add else None
211
+ return output, output_bias
212
+
213
+
214
+class MergedColumnParallelLinear(ColumnParallelLinear):
215
+ """Packed linear layers with column parallelism.
216
+
217
+ Similar to ColumnParallelLinear, but the weight matrix is concatenated
218
+ along the output dimension. When the weight matrix is loaded, the
219
+ different partitions are sharded separately.
220
+
221
+ Args:
222
+ input_size: input dimension of the linear layer.
223
+ output_sizes: list of output dimensions of the linear layer.
224
+ bias: If true, add bias.
225
+ gather_output: If true, call all-gather on output and make the output
226
+ available to all GPUs, otherwise, every GPU will have
227
+ its own output.
228
+ skip_bias_add: This was added to enable performance optimizations where
229
+ bias can be fused with other element-wise operations. we
230
+ skip adding bias but instead return it.
231
+ params_dtype: Data type for the parameters.
232
+ linear_method: (Maybe quantized) linear method.
233
+ """
234
+
235
+ def __init__(
236
+ self,
237
+ input_size: int,
238
+ output_sizes: List[int],
239
+ bias: bool = True,
240
+ gather_output: bool = False,
241
+ skip_bias_add: bool = False,
242
+ params_dtype: Optional[torch.dtype] = None,
243
+ linear_method: Optional[LinearMethodBase] = None,
244
+ ):
245
+ self.output_sizes = output_sizes
246
+ tp_size = get_tensor_model_parallel_world_size()
247
+ assert all(output_size % tp_size == 0 for output_size in output_sizes)
248
+ super().__init__(input_size, sum(output_sizes), bias, gather_output,
249
+ skip_bias_add, params_dtype, linear_method)
250
+
251
+ def weight_loader(self,
252
+ param: Parameter,
253
+ loaded_weight: torch.Tensor,
254
+ loaded_shard_id: Optional[int] = None):
255
+ param_data = param.data
256
+ output_dim = getattr(param, "output_dim", None)
257
+ if loaded_shard_id is None:
258
+ # Loaded weight is already packed.
259
+ if output_dim is None:
260
+ assert param_data.shape == loaded_weight.shape
261
+ param_data.copy_(loaded_weight)
262
+ return
263
+ current_shard_offset = 0
264
+ shard_offsets = []
265
+ for i, output_size in enumerate(self.output_sizes):
266
+ shard_offsets.append((i, current_shard_offset, output_size))
267
+ current_shard_offset += output_size
268
+ packed_dim = getattr(param, "packed_dim", None)
269
+ for shard_id, shard_offset, shard_size in shard_offsets:
270
+ # If quantized, we need to adjust the offset and size to account
271
+ # for the packing.
272
+ if packed_dim == output_dim:
273
+ shard_size = shard_size // param.pack_factor
274
+ shard_offset = shard_offset // param.pack_factor
275
+ loaded_weight_shard = loaded_weight.narrow(
276
+ output_dim, shard_offset, shard_size)
277
+ self.weight_loader(param, loaded_weight_shard, shard_id)
278
+ return
279
+
280
+ assert loaded_shard_id < len(self.output_sizes)
281
+ tp_rank = get_tensor_model_parallel_rank()
282
+ tp_size = get_tensor_model_parallel_world_size()
283
+ if output_dim is not None:
284
+ shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
285
+ shard_size = self.output_sizes[loaded_shard_id] // tp_size
286
+ # If quantized, we need to adjust the offset and size to account
287
+ # for the packing.
288
+ packed_dim = getattr(param, "packed_dim", None)
289
+ if packed_dim == output_dim:
290
+ shard_size = shard_size // param.pack_factor
291
+ shard_offset = shard_offset // param.pack_factor
292
+ param_data = param_data.narrow(output_dim, shard_offset,
293
+ shard_size)
294
+ start_idx = tp_rank * shard_size
295
+ loaded_weight = loaded_weight.narrow(output_dim, start_idx,
296
+ shard_size)
297
+ else:
298
+ logger.warning(
299
+ "Loading a weight without `output_dim` attribute in "
300
+ "MergedColumnParallelLinear, assume the weight is "
301
+ "the same for all partitions.")
302
+ assert param_data.shape == loaded_weight.shape
303
+ param_data.copy_(loaded_weight)
304
+
305
+
306
+class QKVParallelLinear(ColumnParallelLinear):
307
+ """Linear layers for the attention's QKV transformation.
308
+
309
+ Linear layers for the linear transformation of the query, key, and value
310
+ vectors in the attention layer. The weight matrix is concatenated along
311
+ the output dimension. The layer is parallelized along the head dimension.
312
+ When the number of key/value heads is smaller than the number of query
313
+ heads (e.g., multi-query/grouped-query attention), the key/value head may
314
+ be replicated while the query heads are partitioned.
315
+
316
+ Args:
317
+ hidden_size: input hidden state size of the transformer.
318
+ head_size: size of each attention head.
319
+ total_num_heads: total number of attention query heads.
320
+ total_num_kv_heads: total number of attention key/value heads. If
321
+ None, assume total_num_kv_heads = total_num_heads.
322
+ bias: If true, add bias.
323
+ skip_bias_add: This was added to enable performance optimizations where
324
+ bias can be fused with other element-wise operations. we
325
+ skip adding bias but instead return it.
326
+ params_dtype: Data type for the parameters.
327
+ linear_method: (Maybe quantized) linear method.
328
+ """
329
+
330
+ def __init__(
331
+ self,
332
+ hidden_size: int,
333
+ head_size: int,
334
+ total_num_heads: int,
335
+ total_num_kv_heads: Optional[int] = None,
336
+ bias: bool = True,
337
+ skip_bias_add: bool = False,
338
+ params_dtype: Optional[torch.dtype] = None,
339
+ linear_method: Optional[LinearMethodBase] = None,
340
+ ):
341
+ self.hidden_size = hidden_size
342
+ self.head_size = head_size
343
+ self.total_num_heads = total_num_heads
344
+ if total_num_kv_heads is None:
345
+ total_num_kv_heads = total_num_heads
346
+ self.total_num_kv_heads = total_num_kv_heads
347
+ # Divide the weight matrix along the last dimension.
348
+ tp_size = get_tensor_model_parallel_world_size()
349
+ self.num_heads = divide(self.total_num_heads, tp_size)
350
+ if tp_size >= self.total_num_kv_heads:
351
+ self.num_kv_heads = 1
352
+ self.num_kv_head_replicas = divide(tp_size,
353
+ self.total_num_kv_heads)
354
+ else:
355
+ self.num_kv_heads = divide(self.total_num_kv_heads, tp_size)
356
+ self.num_kv_head_replicas = 1
357
+ input_size = self.hidden_size
358
+ output_size = (self.num_heads +
359
+ 2 * self.num_kv_heads) * tp_size * self.head_size
360
+ super().__init__(input_size, output_size, bias, False, skip_bias_add,
361
+ params_dtype, linear_method)
362
+
363
+ def weight_loader(self,
364
+ param: Parameter,
365
+ loaded_weight: torch.Tensor,
366
+ loaded_shard_id: Optional[str] = None):
367
+ param_data = param.data
368
+ output_dim = getattr(param, "output_dim", None)
369
+ if loaded_shard_id is None:
370
+ # Loaded weight is already packed.
371
+ if output_dim is None:
372
+ assert param_data.shape == loaded_weight.shape
373
+ param_data.copy_(loaded_weight)
374
+ return
375
+ shard_offsets = [
376
+ # (shard_id, shard_offset, shard_size)
377
+ ("q", 0, self.total_num_heads * self.head_size),
378
+ ("k", self.total_num_heads * self.head_size,
379
+ self.total_num_kv_heads * self.head_size),
380
+ ("v", (self.total_num_heads + self.total_num_kv_heads) *
381
+ self.head_size, self.total_num_kv_heads * self.head_size),
382
+ ]
383
+ packed_dim = getattr(param, "packed_dim", None)
384
+ for shard_id, shard_offset, shard_size in shard_offsets:
385
+ # If quantized, we need to adjust the offset and size to account
386
+ # for the packing.
387
+ if packed_dim == output_dim:
388
+ shard_size = shard_size // param.pack_factor
389
+ shard_offset = shard_offset // param.pack_factor
390
+ loaded_weight_shard = loaded_weight.narrow(
391
+ output_dim, shard_offset, shard_size)
392
+ self.weight_loader(param, loaded_weight_shard, shard_id)
393
+ return
394
+
395
+ tp_rank = get_tensor_model_parallel_rank()
396
+ assert loaded_shard_id in ["q", "k", "v"]
397
+ if output_dim is not None:
398
+ if loaded_shard_id == "q":
399
+ shard_offset = 0
400
+ shard_size = self.num_heads * self.head_size
401
+ elif loaded_shard_id == "k":
402
+ shard_offset = self.num_heads * self.head_size
403
+ shard_size = self.num_kv_heads * self.head_size
404
+ elif loaded_shard_id == "v":
405
+ shard_offset = (self.num_heads +
406
+ self.num_kv_heads) * self.head_size
407
+ shard_size = self.num_kv_heads * self.head_size
408
+ # If quantized, we need to adjust the offset and size to account
409
+ # for the packing.
410
+ packed_dim = getattr(param, "packed_dim", None)
411
+ if packed_dim == output_dim:
412
+ shard_size = shard_size // param.pack_factor
413
+ shard_offset = shard_offset // param.pack_factor
414
+ param_data = param_data.narrow(output_dim, shard_offset,
415
+ shard_size)
416
+ shard_id = tp_rank // self.num_kv_head_replicas
417
+ start_idx = shard_id * shard_size
418
+ loaded_weight = loaded_weight.narrow(output_dim, start_idx,
419
+ shard_size)
420
+ else:
421
+ logger.warning(
422
+ "Loading a weight without `output_dim` attribute in "
423
+ "QKVParallelLinear, assume the weight is the same "
424
+ "for all partitions.")
425
+ assert param_data.shape == loaded_weight.shape
426
+ param_data.copy_(loaded_weight)
427
+
428
+
429
+class RowParallelLinear(torch.nn.Module):
430
+ """Linear layer with row parallelism.
431
+
432
+ The linear layer is defined as Y = XA + b. A is parallelized along
433
+ its first dimension and X along its second dimension as:
434
+ - -
435
+ | A_1 |
436
+ | . |
437
+ A = | . | X = [X_1, ..., X_p]
438
+ | . |
439
+ | A_p |
440
+ - -
441
+ Arguments:
442
+ input_size: first dimension of matrix A.
443
+ output_size: second dimension of matrix A.
444
+ bias: If true, add bias. Note that bias is not parallelized.
445
+ input_is_parallel: If true, we assume that the input is already
446
+ split across the GPUs and we do not split
447
+ again.
448
+ skip_bias_add: This was added to enable performance optimization where
449
+ bias can be fused with other element-wise operations.
450
+ We skip adding bias but instead return it.
451
+ params_dtype: Data type for the parameters.
452
+ linear_method: (Maybe quantized) linear method.
453
+ """
454
+
455
+ def __init__(
456
+ self,
457
+ input_size: int,
458
+ output_size: int,
459
+ bias: bool = True,
460
+ input_is_parallel: bool = True,
461
+ skip_bias_add: bool = False,
462
+ params_dtype: Optional[torch.dtype] = None,
463
+ reduce_results: bool = True,
464
+ linear_method: Optional[LinearMethodBase] = None,
465
+ ):
466
+ super().__init__()
467
+ # Keep input parameters
468
+ self.input_size = input_size
469
+ self.output_size = output_size
470
+ self.input_is_parallel = input_is_parallel
471
+ self.reduce_results = reduce_results
472
+ if params_dtype is None:
473
+ params_dtype = torch.get_default_dtype()
474
+ self.params_dtype = params_dtype
475
+
476
+ # Divide the weight matrix along the last dimension.
477
+ self.tp_size = get_tensor_model_parallel_world_size()
478
+ self.input_size_per_partition = divide(input_size, self.tp_size)
479
+ self.skip_bias_add = skip_bias_add
480
+ if linear_method is None:
481
+ linear_method = UnquantizedLinearMethod()
482
+ self.linear_method = linear_method
483
+ self.linear_weights = self.linear_method.create_weights(
484
+ self.input_size_per_partition, self.output_size, self.params_dtype)
485
+ for name, weight in self.linear_weights.items():
486
+ self.register_parameter(name, weight)
487
+ set_weight_attrs(weight, {"weight_loader": self.weight_loader})
488
+
489
+ if not reduce_results and (bias and not skip_bias_add):
490
+ raise ValueError("When not reduce the results, adding bias to the "
491
+ "results can lead to incorrect results")
492
+
493
+ if bias:
494
+ self.bias = Parameter(
495
+ torch.empty(self.output_size,
496
+ device=torch.cuda.current_device(),
497
+ dtype=params_dtype))
498
+ set_weight_attrs(self.bias, {
499
+ "output_dim": 0,
500
+ "weight_loader": self.weight_loader,
501
+ })
502
+ else:
503
+ self.register_parameter("bias", None)
504
+
505
+ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
506
+ tp_rank = get_tensor_model_parallel_rank()
507
+ input_dim = getattr(param, "input_dim", None)
508
+ param_data = param.data
509
+ if input_dim is not None:
510
+ shard_size = param_data.shape[input_dim]
511
+ start_idx = tp_rank * shard_size
512
+ loaded_weight = loaded_weight.narrow(input_dim, start_idx,
513
+ shard_size)
514
+ assert param_data.shape == loaded_weight.shape
515
+ param_data.copy_(loaded_weight)
516
+
517
+ def forward(self, input_):
518
+ # Set up backprop all-reduce.
519
+ if self.input_is_parallel:
520
+ input_parallel = input_
521
+ else:
522
+ tp_rank = get_tensor_model_parallel_rank()
523
+ splitted_input = split_tensor_along_last_dim(
524
+ input_, num_partitions=self.tp_size)
525
+ input_parallel = splitted_input[tp_rank].contiguous()
526
+
527
+ # Matrix multiply.
528
+ output_parallel = self.linear_method.apply_weights(
529
+ self.linear_weights, input_parallel)
530
+ if self.reduce_results and self.tp_size > 1:
531
+ output_ = tensor_model_parallel_all_reduce(output_parallel)
532
+ else:
533
+ output_ = output_parallel
534
+
535
+ if not self.skip_bias_add:
536
+ output = output_ + self.bias if self.bias is not None else output_
537
+ output_bias = None
538
+ else:
539
+ output = output_
540
+ output_bias = self.bias
541
+ return output, output_bias
vllm/model_executor/layers/quantization/__init__.py
new
+22
@@ -0,0 +1,22 @@
1
+from typing import Type
2
+
3
+from vllm.model_executor.layers.quantization.awq import AWQConfig
4
+from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
5
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
6
+
7
+_QUANTIZATION_CONFIG_REGISTRY = {
8
+ "awq": AWQConfig,
9
+ "squeezellm": SqueezeLLMConfig,
10
+}
11
+
12
+
13
+def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
14
+ if quantization not in _QUANTIZATION_CONFIG_REGISTRY:
15
+ raise ValueError(f"Invalid quantization method: {quantization}")
16
+ return _QUANTIZATION_CONFIG_REGISTRY[quantization]
17
+
18
+
19
+__all__ = [
20
+ "QuantizationConfig",
21
+ "get_quantization_config",
22
+]
vllm/model_executor/layers/quantization/awq.py
new
+155
@@ -0,0 +1,155 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm import quantization_ops
7
+from vllm.model_executor.layers.linear import (LinearMethodBase,
8
+ set_weight_attrs)
9
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
10
+
11
+
12
+class AWQConfig(QuantizationConfig):
13
+ """Config class for AWQ.
14
+
15
+ Reference: https://arxiv.org/abs/2306.00978
16
+ """
17
+
18
+ def __init__(
19
+ self,
20
+ weight_bits: int,
21
+ group_size: int,
22
+ zero_point: bool,
23
+ ) -> None:
24
+ self.weight_bits = weight_bits
25
+ self.group_size = group_size
26
+ self.zero_point = zero_point
27
+
28
+ if self.weight_bits != 4:
29
+ raise ValueError(
30
+ "Currently, only 4-bit weight quantization is supported for "
31
+ f"AWQ, but got {self.weight_bits} bits.")
32
+ self.pack_factor = 32 // self.weight_bits
33
+
34
+ def __repr__(self) -> str:
35
+ return (f"AWQConfig(weight_bits={self.weight_bits}, "
36
+ f"group_size={self.group_size}, "
37
+ f"zero_point={self.zero_point})")
38
+
39
+ def get_name(self) -> str:
40
+ return "awq"
41
+
42
+ def get_supported_act_dtypes(self) -> List[torch.dtype]:
43
+ return [torch.half]
44
+
45
+ def get_min_capability(self) -> int:
46
+ # The AWQ kernel only supports Turing or newer GPUs.
47
+ return 75
48
+
49
+ @staticmethod
50
+ def get_config_filenames() -> List[str]:
51
+ return [
52
+ "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
53
+ "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq # pylint: disable=line-too-long
54
+ ]
55
+
56
+ @classmethod
57
+ def from_config(cls, config: Dict[str, Any]) -> "AWQConfig":
58
+ weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
59
+ group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
60
+ zero_point = cls.get_from_keys(config, ["zero_point"])
61
+ return cls(weight_bits, group_size, zero_point)
62
+
63
+ def get_linear_method(self) -> "AWQLinearMethod":
64
+ return AWQLinearMethod(self)
65
+
66
+
67
+class AWQLinearMethod(LinearMethodBase):
68
+ """Linear method for AWQ.
69
+
70
+ Args:
71
+ quant_config: The AWQ quantization config.
72
+ """
73
+
74
+ def __init__(self, quant_config: AWQConfig):
75
+ self.quant_config = quant_config
76
+
77
+ def create_weights(self, input_size: int, output_size: int,
78
+ params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
79
+ if input_size % self.quant_config.group_size != 0:
80
+ raise ValueError(
81
+ "The input size is not aligned with the quantized "
82
+ "weight shape. This can be caused by too large "
83
+ "tensor parallel size.")
84
+ if output_size % self.quant_config.pack_factor != 0:
85
+ raise ValueError(
86
+ "The output size is not aligned with the quantized "
87
+ "weight shape. This can be caused by too large "
88
+ "tensor parallel size.")
89
+
90
+ qweight = Parameter(
91
+ torch.empty(
92
+ input_size,
93
+ output_size // self.quant_config.pack_factor,
94
+ device="cuda",
95
+ dtype=torch.int32,
96
+ ),
97
+ requires_grad=False,
98
+ )
99
+ set_weight_attrs(
100
+ qweight, {
101
+ "input_dim": 0,
102
+ "output_dim": 1,
103
+ "packed_dim": 1,
104
+ "pack_factor": self.quant_config.pack_factor,
105
+ })
106
+ qzeros = Parameter(
107
+ torch.empty(
108
+ input_size // self.quant_config.group_size,
109
+ output_size // self.quant_config.pack_factor,
110
+ device="cuda",
111
+ dtype=torch.int32,
112
+ ),
113
+ requires_grad=False,
114
+ )
115
+ set_weight_attrs(
116
+ qzeros, {
117
+ "input_dim": 0,
118
+ "output_dim": 1,
119
+ "packed_dim": 1,
120
+ "pack_factor": self.quant_config.pack_factor,
121
+ })
122
+ scales = Parameter(
123
+ torch.empty(
124
+ input_size // self.quant_config.group_size,
125
+ output_size,
126
+ device="cuda",
127
+ dtype=params_dtype,
128
+ ),
129
+ requires_grad=False,
130
+ )
131
+ set_weight_attrs(scales, {
132
+ "input_dim": 0,
133
+ "output_dim": 1,
134
+ })
135
+ return {
136
+ "qweight": qweight,
137
+ "qzeros": qzeros,
138
+ "scales": scales,
139
+ }
140
+
141
+ def apply_weights(self,
142
+ weights: Dict[str, torch.Tensor],
143
+ x: torch.Tensor,
144
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
145
+ qweight = weights["qweight"]
146
+ qzeros = weights["qzeros"]
147
+ scales = weights["scales"]
148
+ pack_factor = self.quant_config.pack_factor
149
+ out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
150
+ reshaped_x = x.reshape(-1, x.shape[-1])
151
+ out = quantization_ops.awq_gemm(reshaped_x, qweight, scales, qzeros,
152
+ pack_factor)
153
+ if bias is not None:
154
+ out = out + bias
155
+ return out.reshape(out_shape)
vllm/model_executor/layers/quantization/base_config.py
new
+56
@@ -0,0 +1,56 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Any, Dict, List
3
+
4
+import torch
5
+
6
+from vllm.model_executor.layers.linear import LinearMethodBase
7
+
8
+
9
+class QuantizationConfig(ABC):
10
+ """Base class for quantization configs."""
11
+
12
+ @abstractmethod
13
+ def get_name(self) -> str:
14
+ """Name of the quantization method."""
15
+ raise NotImplementedError
16
+
17
+ @abstractmethod
18
+ def get_supported_act_dtypes(self) -> List[torch.dtype]:
19
+ """List of supported activation dtypes."""
20
+ raise NotImplementedError
21
+
22
+ @abstractmethod
23
+ def get_min_capability(self) -> int:
24
+ """Minimum GPU capability to support the quantization method.
25
+
26
+ E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
27
+ This requirement is due to the custom CUDA kernels used by the
28
+ quantization method.
29
+ """
30
+ raise NotImplementedError
31
+
32
+ @staticmethod
33
+ @abstractmethod
34
+ def get_config_filenames() -> List[str]:
35
+ """List of filenames to search for in the model directory."""
36
+ raise NotImplementedError
37
+
38
+ @classmethod
39
+ @abstractmethod
40
+ def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig":
41
+ """Create a config class from the model's quantization config."""
42
+ raise NotImplementedError
43
+
44
+ @staticmethod
45
+ def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
46
+ """Get a value from the model's quantization config."""
47
+ for key in keys:
48
+ if key in config:
49
+ return config[key]
50
+ raise ValueError(f"Cannot find any of {keys} in the model's "
51
+ "quantization config.")
52
+
53
+ @abstractmethod
54
+ def get_linear_method(self) -> LinearMethodBase:
55
+ """Get the linear method to use for the quantized linear layer."""
56
+ raise NotImplementedError
vllm/model_executor/layers/quantization/squeezellm.py
new
+121
@@ -0,0 +1,121 @@
1
+from typing import Any, Dict, List, Optional
2
+
3
+import torch
4
+from torch.nn.parameter import Parameter
5
+
6
+from vllm import quantization_ops
7
+from vllm.model_executor.layers.linear import (LinearMethodBase,
8
+ set_weight_attrs)
9
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
10
+
11
+
12
+class SqueezeLLMConfig(QuantizationConfig):
13
+ """Config class for SqueezeLLM.
14
+
15
+ Reference: https://arxiv.org/pdf/2306.07629
16
+ """
17
+
18
+ def __init__(
19
+ self,
20
+ weight_bits: int,
21
+ ) -> None:
22
+ self.weight_bits = weight_bits
23
+
24
+ if self.weight_bits != 4:
25
+ raise ValueError(
26
+ "Currently, only 4-bit weight quantization is supported for "
27
+ f"SqueezeLLM, but got {self.weight_bits} bits.")
28
+
29
+ self.pack_factor = 32 // self.weight_bits
30
+
31
+ def __repr__(self) -> str:
32
+ return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
33
+
34
+ def get_name(self) -> str:
35
+ return "squeezellm"
36
+
37
+ def get_supported_act_dtypes(self) -> List[torch.dtype]:
38
+ return [torch.half]
39
+
40
+ def get_min_capability(self) -> int:
41
+ return 70
42
+
43
+ @staticmethod
44
+ def get_config_filenames() -> List[str]:
45
+ return ["quant_config.json"]
46
+
47
+ @classmethod
48
+ def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
49
+ weight_bits = cls.get_from_keys(config, ["wbits"])
50
+ return cls(weight_bits)
51
+
52
+ def get_linear_method(self) -> "SqueezeLLMLinearMethod":
53
+ return SqueezeLLMLinearMethod(self)
54
+
55
+
56
+class SqueezeLLMLinearMethod(LinearMethodBase):
57
+ """Linear method for SqueezeLLM.
58
+
59
+ Args:
60
+ quant_config: The SqueezeLLM quantization config.
61
+ """
62
+
63
+ def __init__(self, quant_config: SqueezeLLMConfig):
64
+ self.quant_config = quant_config
65
+
66
+ def create_weights(self, input_size: int, output_size: int,
67
+ params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
68
+ if input_size % self.quant_config.pack_factor != 0:
69
+ raise ValueError(
70
+ "The input size is not aligned with the quantized "
71
+ "weight shape. This can be caused by too large "
72
+ "tensor parallel size.")
73
+ qweight = Parameter(
74
+ torch.empty(
75
+ input_size // self.quant_config.pack_factor,
76
+ output_size,
77
+ device="cuda",
78
+ dtype=torch.int32,
79
+ ),
80
+ requires_grad=False,
81
+ )
82
+ set_weight_attrs(
83
+ qweight, {
84
+ "input_dim": 0,
85
+ "output_dim": 1,
86
+ "packed_dim": 0,
87
+ "pack_factor": self.quant_config.pack_factor,
88
+ })
89
+ lookup_table = Parameter(
90
+ torch.empty(
91
+ output_size,
92
+ self.quant_config.weight_bits**2,
93
+ device="cuda",
94
+ dtype=params_dtype,
95
+ ),
96
+ requires_grad=False,
97
+ )
98
+ set_weight_attrs(lookup_table, {
99
+ "output_dim": 0,
100
+ })
101
+ return {
102
+ "qweight": qweight,
103
+ "lookup_table": lookup_table,
104
+ }
105
+
106
+ def apply_weights(self,
107
+ weights: Dict[str, torch.Tensor],
108
+ x: torch.Tensor,
109
+ bias: Optional[torch.Tensor] = None) -> torch.Tensor:
110
+ qweight = weights["qweight"]
111
+ lookup_table = weights["lookup_table"]
112
+ out_shape = x.shape[:-1] + (qweight.shape[-1], )
113
+ reshaped_x = x.reshape(-1, x.shape[-1])
114
+ # NOTE: The output tensor should be zero-initialized.
115
+ out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
116
+ quantization_ops.squeezellm_gemm(reshaped_x, qweight, out,
117
+ lookup_table)
118
+
119
+ if bias is not None:
120
+ out = out + bias
121
+ return out.reshape(out_shape)
vllm/model_executor/layers/quantized_linear/__init__.py
deleted
-41
@@ -1,41 +0,0 @@
1
-from vllm.model_executor.layers.quantized_linear.awq import (
2
- AWQColumnParallelLinear, AWQRowParallelLinear)
3
-from vllm.model_executor.layers.quantized_linear.squeezellm import (
4
- SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear)
5
-from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
6
- RowParallelLinear)
7
-
8
-_QUANTIZED_LINEAR_REGISTRY = {
9
- "awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
10
- "squeezellm":
11
- (SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear),
12
-}
13
-
14
-
15
-class ParallelLinear:
16
-
17
- @classmethod
18
- def column(cls, *args, **kwargs) -> ColumnParallelLinear:
19
- quant_config = kwargs.get("quant_config", None)
20
- if quant_config is None:
21
- return ColumnParallelLinear(*args, **kwargs)
22
-
23
- name = quant_config.get_name()
24
- if name not in _QUANTIZED_LINEAR_REGISTRY:
25
- raise ValueError(f"No quantized linear is found for {name}")
26
-
27
- quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][0]
28
- return quant_linear_cls(*args, **kwargs)
29
-
30
- @classmethod
31
- def row(cls, *args, **kwargs) -> RowParallelLinear:
32
- quant_config = kwargs.get("quant_config", None)
33
- if quant_config is None:
34
- return RowParallelLinear(*args, **kwargs)
35
-
36
- name = quant_config.get_name()
37
- if name not in _QUANTIZED_LINEAR_REGISTRY:
38
- raise ValueError(f"No quantized linear is found for {name}")
39
-
40
- quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][1]
41
- return quant_linear_cls(*args, **kwargs)
vllm/model_executor/layers/quantized_linear/awq.py
deleted
-106
@@ -1,106 +0,0 @@
1
-from typing import Optional
2
-
3
-import torch
4
-from torch.nn.parameter import Parameter
5
-
6
-from vllm import quantization_ops
7
-from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8
- RowParallelLinear)
9
-
10
-
11
-class AWQColumnParallelLinear(ColumnParallelLinear):
12
-
13
- def create_weights(self, dtype: torch.dtype) -> None:
14
- assert self.input_size % self.quant_config.group_size == 0
15
- if self.output_size_per_partition % self.quant_config.pack_factor != 0:
16
- raise ValueError(
17
- "The tensor parallel size is not aligned with the quantized "
18
- "weight shape. Please use a different tensor parallel size.")
19
- self.qweight = Parameter(
20
- torch.empty(
21
- self.input_size,
22
- self.output_size_per_partition //
23
- self.quant_config.pack_factor,
24
- device="cuda",
25
- dtype=torch.int32,
26
- ),
27
- requires_grad=False,
28
- )
29
- self.qzeros = Parameter(
30
- torch.empty(
31
- self.input_size // self.quant_config.group_size,
32
- self.output_size_per_partition //
33
- self.quant_config.pack_factor,
34
- device="cuda",
35
- dtype=torch.int32,
36
- ),
37
- requires_grad=False,
38
- )
39
- self.scales = Parameter(
40
- torch.empty(
41
- self.input_size // self.quant_config.group_size,
42
- self.output_size_per_partition,
43
- device="cuda",
44
- dtype=dtype,
45
- ),
46
- requires_grad=False,
47
- )
48
-
49
- def apply_weights(
50
- self,
51
- x: torch.Tensor,
52
- bias: Optional[torch.Tensor],
53
- ) -> torch.Tensor:
54
- pack_factor = self.quant_config.pack_factor
55
- out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
56
- reshaped_x = x.reshape(-1, x.shape[-1])
57
- out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
58
- self.qzeros, pack_factor)
59
- if bias is not None:
60
- out = out + bias
61
- return out.reshape(out_shape)
62
-
63
-
64
-class AWQRowParallelLinear(RowParallelLinear):
65
-
66
- def create_weights(self, dtype: torch.dtype) -> None:
67
- assert self.output_size % self.quant_config.pack_factor == 0
68
- if self.input_size_per_partition % self.quant_config.group_size != 0:
69
- raise ValueError(
70
- "The tensor parallel size is not aligned with the quantized "
71
- "weight shape. Please use a different tensor parallel size.")
72
- self.qweight = Parameter(
73
- torch.empty(
74
- self.input_size_per_partition,
75
- self.output_size // self.quant_config.pack_factor,
76
- device="cuda",
77
- dtype=torch.int32,
78
- ),
79
- requires_grad=False,
80
- )
81
- self.qzeros = Parameter(
82
- torch.empty(
83
- self.input_size_per_partition // self.quant_config.group_size,
84
- self.output_size // self.quant_config.pack_factor,
85
- device="cuda",
86
- dtype=torch.int32,
87
- ),
88
- requires_grad=False,
89
- )
90
- self.scales = Parameter(
91
- torch.empty(
92
- self.input_size_per_partition // self.quant_config.group_size,
93
- self.output_size,
94
- device="cuda",
95
- dtype=dtype,
96
- ),
97
- requires_grad=False,
98
- )
99
-
100
- def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
101
- pack_factor = self.quant_config.pack_factor
102
- out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
103
- reshaped_x = x.reshape(-1, x.shape[-1])
104
- out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
105
- self.qzeros, pack_factor)
106
- return out.reshape(out_shape)
vllm/model_executor/layers/quantized_linear/squeezellm.py
deleted
-84
@@ -1,84 +0,0 @@
1
-from typing import Optional
2
-
3
-import torch
4
-from torch.nn.parameter import Parameter
5
-
6
-from vllm import quantization_ops
7
-from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8
- RowParallelLinear)
9
-
10
-
11
-class SqueezeLLMColumnParallelLinear(ColumnParallelLinear):
12
-
13
- def create_weights(self, dtype: torch.dtype) -> None:
14
- assert self.input_size % self.quant_config.pack_factor == 0
15
- self.qweight = Parameter(
16
- torch.empty(
17
- self.input_size // self.quant_config.pack_factor,
18
- self.output_size_per_partition,
19
- device="cuda",
20
- dtype=torch.int32,
21
- ),
22
- requires_grad=False,
23
- )
24
- self.lookup_table = Parameter(
25
- torch.empty(
26
- self.output_size_per_partition,
27
- self.quant_config.weight_bits**2,
28
- device="cuda",
29
- dtype=dtype,
30
- ),
31
- requires_grad=False,
32
- )
33
-
34
- def apply_weights(
35
- self,
36
- x: torch.Tensor,
37
- bias: Optional[torch.Tensor],
38
- ) -> torch.Tensor:
39
- out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
40
- reshaped_x = x.reshape(-1, x.shape[-1])
41
- # NOTE: The output tensor should be zero-initialized.
42
- out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
43
- quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
44
- self.lookup_table)
45
-
46
- if bias is not None:
47
- out = out + bias
48
- return out.reshape(out_shape)
49
-
50
-
51
-class SqueezeLLMRowParallelLinear(RowParallelLinear):
52
-
53
- def create_weights(self, dtype: torch.dtype) -> None:
54
- if self.input_size_per_partition % self.quant_config.pack_factor != 0:
55
- raise ValueError(
56
- "The tensor parallel size is not aligned with the quantized "
57
- "weight shape. Please use a different tensor parallel size.")
58
- self.qweight = Parameter(
59
- torch.empty(
60
- self.input_size_per_partition // self.quant_config.pack_factor,
61
- self.output_size,
62
- device="cuda",
63
- dtype=torch.int32,
64
- ),
65
- requires_grad=False,
66
- )
67
- self.lookup_table = Parameter(
68
- torch.empty(
69
- self.output_size,
70
- self.quant_config.weight_bits**2,
71
- device="cuda",
72
- dtype=dtype,
73
- ),
74
- requires_grad=False,
75
- )
76
-
77
- def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
78
- reshaped_x = x.reshape(-1, x.shape[-1])
79
- out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
80
- # NOTE: The output tensor should be zero-initialized.
81
- out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
82
- quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
83
- self.lookup_table)
84
- return out.reshape(out_shape)
vllm/model_executor/layers/vocab_parallel_embedding.py
new
+139
@@ -0,0 +1,139 @@
1
+from typing import Optional, Sequence
2
+
3
+import torch
4
+import torch.nn.functional as F
5
+from torch.nn.parameter import Parameter
6
+
7
+from vllm.model_executor.parallel_utils.parallel_state import (
8
+ get_tensor_model_parallel_rank,
9
+ get_tensor_model_parallel_world_size,
10
+)
11
+from vllm.model_executor.parallel_utils.utils import divide
12
+from vllm.model_executor.parallel_utils.communication_op import (
13
+ tensor_model_parallel_all_reduce)
14
+from vllm.model_executor.utils import set_weight_attrs
15
+
16
+
17
+def pad_vocab_size(vocab_size: int, pad_to: int = 64) -> int:
18
+ """Pad the vocab size to the given value."""
19
+ return ((vocab_size + pad_to - 1) // pad_to) * pad_to
20
+
21
+
22
+def vocab_range_from_per_partition_vocab_size(per_partition_vocab_size: int,
23
+ rank: int) -> Sequence[int]:
24
+ index_f = rank * per_partition_vocab_size
25
+ index_l = index_f + per_partition_vocab_size
26
+ return index_f, index_l
27
+
28
+
29
+def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
30
+ world_size: int) -> Sequence[int]:
31
+ per_partition_vocab_size = divide(global_vocab_size, world_size)
32
+ return vocab_range_from_per_partition_vocab_size(per_partition_vocab_size,
33
+ rank)
34
+
35
+
36
+class VocabParallelEmbedding(torch.nn.Module):
37
+ """Embedding parallelized in the vocabulary dimension.
38
+
39
+ Adapted from torch.nn.Embedding, note that we pad the vocabulary size to
40
+ make sure it is divisible by the number of model parallel GPUs.
41
+
42
+ Args:
43
+ num_embeddings: vocabulary size.
44
+ embedding_dim: size of hidden state.
45
+ params_dtype: type of the parameters.
46
+ """
47
+
48
+ def __init__(self,
49
+ num_embeddings: int,
50
+ embedding_dim: int,
51
+ params_dtype: Optional[torch.dtype] = None):
52
+ super().__init__()
53
+
54
+ # Keep the input dimensions.
55
+ self.num_embeddings = num_embeddings
56
+ self.num_embeddings_padded = pad_vocab_size(num_embeddings)
57
+ self.embedding_dim = embedding_dim
58
+ if params_dtype is None:
59
+ params_dtype = torch.get_default_dtype()
60
+ self.tp_size = get_tensor_model_parallel_world_size()
61
+ # Divide the weight matrix along the vocaburaly dimension.
62
+ self.vocab_start_index, self.vocab_end_index = (
63
+ vocab_range_from_global_vocab_size(
64
+ self.num_embeddings_padded, get_tensor_model_parallel_rank(),
65
+ self.tp_size))
66
+ self.num_embeddings_per_partition = (self.vocab_end_index -
67
+ self.vocab_start_index)
68
+ self.weight = Parameter(
69
+ torch.empty(self.num_embeddings_per_partition,
70
+ self.embedding_dim,
71
+ device=torch.cuda.current_device(),
72
+ dtype=params_dtype))
73
+ set_weight_attrs(self.weight, {
74
+ "parallel_dim": 0,
75
+ "weight_loader": self.weight_loader
76
+ })
77
+
78
+ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
79
+ parallel_dim = param.parallel_dim
80
+ assert loaded_weight.shape[parallel_dim] == self.num_embeddings
81
+ loaded_weight = loaded_weight[self.vocab_start_index:self.
82
+ vocab_end_index]
83
+ param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
84
+
85
+ def forward(self, input_):
86
+ if self.tp_size > 1:
87
+ # Build the mask.
88
+ input_mask = ((input_ < self.vocab_start_index) |
89
+ (input_ >= self.vocab_end_index))
90
+ # Mask the input.
91
+ masked_input = input_.clone() - self.vocab_start_index
92
+ masked_input[input_mask] = 0
93
+ else:
94
+ masked_input = input_
95
+ # Get the embeddings.
96
+ output_parallel = F.embedding(masked_input, self.weight)
97
+ # Mask the output embedding.
98
+ if self.tp_size > 1:
99
+ output_parallel[input_mask, :] = 0.0
100
+ # Reduce across all the model parallel GPUs.
101
+ output = tensor_model_parallel_all_reduce(output_parallel)
102
+ return output
103
+
104
+
105
+class ParallelLMHead(VocabParallelEmbedding):
106
+ """Parallelized LM head.
107
+
108
+ Output logits weight matrices used in the Sampler. The weight and bias
109
+ tensors are padded to make sure they are divisible by the number of
110
+ model parallel GPUs.
111
+
112
+ Args:
113
+ num_embeddings: vocabulary size.
114
+ embedding_dim: size of hidden state.
115
+ bias: whether to use bias.
116
+ params_dtype: type of the parameters.
117
+ """
118
+
119
+ def __init__(self,
120
+ num_embeddings: int,
121
+ embedding_dim: int,
122
+ bias: bool = False,
123
+ params_dtype: Optional[torch.dtype] = None):
124
+ super().__init__(num_embeddings, embedding_dim, params_dtype)
125
+ if bias:
126
+ self.bias = Parameter(
127
+ torch.empty(self.num_embeddings_per_partition,
128
+ device=torch.cuda.current_device(),
129
+ dtype=params_dtype))
130
+ set_weight_attrs(self.bias, {
131
+ "parallel_dim": 0,
132
+ "weight_loader": self.weight_loader
133
+ })
134
+ else:
135
+ self.register_parameter("bias", None)
136
+
137
+ def forward(self, input_):
138
+ del input_
139
+ raise RuntimeError("LMHead's weights should be used in the sampler.")
vllm/model_executor/model_loader.py
+4
-16
@@ -37,13 +37,6 @@ _MODEL_REGISTRY = {
37
"YiForCausalLM": YiForCausalLM,
38
}
39
40
-# FIXME(woosuk): Remove this once all models support quantization.
41
-_MODEL_CLASSES_SUPPORT_QUANTIZATION = [
42
- LlamaForCausalLM,
43
- MistralForCausalLM,
44
- YiForCausalLM,
45
-]
46
-
40
41
@contextlib.contextmanager
42
def _set_default_torch_dtype(dtype: torch.dtype):
@@ -67,12 +60,9 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
60
def get_model(model_config: ModelConfig) -> nn.Module:
61
model_class = _get_model_architecture(model_config.hf_config)
62
70
- # Get the quantization config.
71
- quant_config = None
63
+ # Get the (maybe quantized) linear method.
64
+ linear_method = None
65
if model_config.quantization is not None:
73
- if model_class not in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
74
- raise ValueError(
75
- f"Quantization is not supported for {model_class}.")
66
quant_config = get_quant_config(model_config.quantization,
67
model_config.model,
68
model_config.download_dir)
@@ -90,14 +80,12 @@ def get_model(model_config: ModelConfig) -> nn.Module:
80
f"{model_config.dtype} is not supported for quantization "
81
f"method {model_config.quantization}. Supported dtypes: "
82
f"{supported_dtypes}")
83
+ linear_method = quant_config.get_linear_method()
84
85
with _set_default_torch_dtype(model_config.dtype):
86
# Create a model instance.
87
# The weights will be initialized as empty tensors.
97
- if model_class in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
98
- model = model_class(model_config.hf_config, quant_config)
99
- else:
100
- model = model_class(model_config.hf_config)
88
+ model = model_class(model_config.hf_config, linear_method)
89
if model_config.load_format == "dummy":
90
model = model.cuda()
91
# NOTE(woosuk): For accurate performance evaluation, we assign
vllm/model_executor/models/aquila.py
+64
-102
@@ -33,15 +33,17 @@ from torch import nn
33
from vllm.model_executor.input_metadata import InputMetadata
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
36
+from vllm.model_executor.layers.linear import (LinearMethodBase,
37
+ MergedColumnParallelLinear,
38
+ QKVParallelLinear,
39
+ RowParallelLinear)
40
from vllm.model_executor.layers.sampler import Sampler
37
-from vllm.model_executor.weight_utils import (
38
- hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
39
- load_tensor_parallel_weights)
41
+from vllm.model_executor.layers.vocab_parallel_embedding import (
42
+ VocabParallelEmbedding, ParallelLMHead)
43
from vllm.model_executor.parallel_utils.parallel_state import (
41
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43
- ColumnParallelLinear,
44
- RowParallelLinear)
44
+ get_tensor_model_parallel_world_size)
45
+from vllm.model_executor.weight_utils import (default_weight_loader,
46
+ hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
48
from vllm.transformers_utils.configs.aquila import AquilaConfig
49
@@ -55,20 +57,17 @@ class AquilaMLP(nn.Module):
57
hidden_size: int,
58
intermediate_size: int,
59
hidden_act: str,
60
+ linear_method: Optional[LinearMethodBase] = None,
61
):
62
super().__init__()
60
- self.gate_up_proj = ColumnParallelLinear(
61
- hidden_size,
62
- 2 * intermediate_size,
63
+ self.gate_up_proj = MergedColumnParallelLinear(
64
+ hidden_size, [intermediate_size] * 2,
65
bias=False,
64
- gather_output=False,
65
- )
66
- self.down_proj = RowParallelLinear(
67
- intermediate_size,
68
- hidden_size,
69
- bias=False,
70
- input_is_parallel=True,
71
- )
66
+ linear_method=linear_method)
67
+ self.down_proj = RowParallelLinear(intermediate_size,
68
+ hidden_size,
69
+ bias=False,
70
+ linear_method=linear_method)
71
if hidden_act != "silu":
72
raise ValueError(f"Unsupported activation: {hidden_act}. "
73
"Only silu is supported for now.")
@@ -111,6 +110,7 @@ class AquilaAttention(nn.Module):
110
rope_theta: float = 10000,
111
max_position_embeddings: int = 8192,
112
rope_scaling: Optional[Dict[str, Any]] = None,
113
+ linear_method: Optional[LinearMethodBase] = None,
114
):
115
super().__init__()
116
self.hidden_size = hidden_size
@@ -128,29 +128,29 @@ class AquilaAttention(nn.Module):
128
self.rope_theta = rope_theta
129
self.max_position_embeddings = max_position_embeddings
130
131
- self.qkv_proj = ColumnParallelLinear(
131
+ self.qkv_proj = QKVParallelLinear(
132
hidden_size,
133
- (self.total_num_heads + 2 * self.total_num_kv_heads) *
133
self.head_dim,
134
+ self.total_num_heads,
135
+ self.total_num_kv_heads,
136
bias=False,
136
- gather_output=False,
137
+ linear_method=linear_method,
138
)
139
self.o_proj = RowParallelLinear(
140
self.total_num_heads * self.head_dim,
141
hidden_size,
142
bias=False,
142
- input_is_parallel=True,
143
+ linear_method=linear_method,
144
)
145
self.attn = PagedAttentionWithRoPE(
146
self.num_heads,
147
self.head_dim,
148
self.scaling,
148
- rotary_dim=self.head_dim,
149
base=self.rope_theta,
150
max_position=self.max_position_embeddings,
151
+ rotary_dim=self.head_dim,
152
num_kv_heads=self.num_kv_heads,
152
- rope_scaling=rope_scaling,
153
- )
153
+ rope_scaling=rope_scaling)
154
155
def forward(
156
self,
@@ -171,7 +171,11 @@ class AquilaAttention(nn.Module):
171
172
class AquilaDecoderLayer(nn.Module):
173
174
- def __init__(self, config: AquilaConfig):
174
+ def __init__(
175
+ self,
176
+ config: AquilaConfig,
177
+ linear_method: Optional[LinearMethodBase] = None,
178
+ ):
179
super().__init__()
180
self.hidden_size = config.hidden_size
181
rope_theta = getattr(config, "rope_theta", 10000)
@@ -185,11 +189,13 @@ class AquilaDecoderLayer(nn.Module):
189
rope_theta=rope_theta,
190
max_position_embeddings=max_position_embeddings,
191
rope_scaling=rope_scaling,
192
+ linear_method=linear_method,
193
)
194
self.mlp = AquilaMLP(
195
hidden_size=self.hidden_size,
196
intermediate_size=config.intermediate_size,
197
hidden_act=config.hidden_act,
198
+ linear_method=linear_method,
199
)
200
self.input_layernorm = AquilaRMSNorm(config.hidden_size,
201
eps=config.rms_norm_eps)
@@ -226,19 +232,22 @@ class AquilaDecoderLayer(nn.Module):
232
233
class AquilaModel(nn.Module):
234
229
- def __init__(self, config: AquilaConfig):
235
+ def __init__(
236
+ self,
237
+ config: AquilaConfig,
238
+ linear_method: Optional[LinearMethodBase] = None,
239
+ ):
240
super().__init__()
241
self.config = config
242
self.padding_idx = config.pad_token_id
243
self.vocab_size = config.vocab_size
234
-
235
- #vocab_size = ((config.vocab_size + 63) // 64) * 64
244
self.embed_tokens = VocabParallelEmbedding(
245
config.vocab_size,
246
config.hidden_size,
247
)
248
self.layers = nn.ModuleList([
241
- AquilaDecoderLayer(config) for _ in range(config.num_hidden_layers)
249
+ AquilaDecoderLayer(config, linear_method)
250
+ for _ in range(config.num_hidden_layers)
251
])
252
self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
253
@@ -271,17 +280,16 @@ class AquilaModel(nn.Module):
280
281
class AquilaForCausalLM(nn.Module):
282
274
- def __init__(self, config):
283
+ def __init__(
284
+ self,
285
+ config,
286
+ linear_method: Optional[LinearMethodBase] = None,
287
+ ):
288
super().__init__()
289
self.config = config
277
- self.model = AquilaModel(config)
278
- vocab_size = ((config.vocab_size + 63) // 64) * 64
279
- self.lm_head = ColumnParallelLinear(
280
- config.hidden_size,
281
- vocab_size,
282
- bias=False,
283
- gather_output=False,
284
- )
290
+ self.linear_method = linear_method
291
+ self.model = AquilaModel(config, linear_method)
292
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
293
self.sampler = Sampler(config.vocab_size)
294
295
def forward(
@@ -298,79 +306,33 @@ class AquilaForCausalLM(nn.Module):
306
input_metadata)
307
return next_tokens
308
301
- _column_parallel_weights = [
302
- "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
303
- ]
304
- _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
305
-
309
def load_weights(self,
310
model_name_or_path: str,
311
cache_dir: Optional[str] = None,
312
load_format: str = "auto",
313
revision: Optional[str] = None):
311
- tp_size = get_tensor_model_parallel_world_size()
312
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
313
- q_proj_shard_size = (self.config.hidden_size // tp_size)
314
- kv_proj_shard_size = (self.config.hidden_size //
315
- self.config.num_attention_heads *
316
- self.config.num_key_value_heads // tp_size)
317
- attention_weight_specs = [
318
- # (weight_name, shard_size, offset)
319
- ("q_proj", q_proj_shard_size, 0),
320
- ("k_proj", kv_proj_shard_size, q_proj_shard_size),
321
- ("v_proj", kv_proj_shard_size,
322
- q_proj_shard_size + kv_proj_shard_size),
314
+ stacked_params_mapping = [
315
+ # (param_name, shard_name, shard_id)
316
+ ("qkv_proj", "q_proj", "q"),
317
+ ("qkv_proj", "k_proj", "k"),
318
+ ("qkv_proj", "v_proj", "v"),
319
+ ("gate_up_proj", "gate_proj", 0),
320
+ ("gate_up_proj", "up_proj", 1),
321
]
324
- state_dict = self.state_dict()
325
-
322
+ params_dict = dict(self.named_parameters())
323
for name, loaded_weight in hf_model_weights_iterator(
324
model_name_or_path, cache_dir, load_format, revision):
325
if "rotary_emb.inv_freq" in name:
326
continue
330
-
331
- is_attention_weight = False
332
- for weight_name, shard_size, offset in attention_weight_specs:
327
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
328
if weight_name not in name:
329
continue
335
- param = state_dict[name.replace(weight_name, "qkv_proj")]
336
-
337
- loaded_weight = loaded_weight[
338
- shard_size * tensor_model_parallel_rank:shard_size *
339
- (tensor_model_parallel_rank + 1)]
340
- param_slice = param.data[offset:offset + shard_size]
341
- assert param_slice.shape == loaded_weight.shape
342
-
343
- param_slice.copy_(loaded_weight)
344
- is_attention_weight = True
330
+ param = params_dict[name.replace(weight_name, param_name)]
331
+ weight_loader = param.weight_loader
332
+ weight_loader(param, loaded_weight, shard_id)
333
break
346
- if is_attention_weight:
347
- continue
348
-
349
- is_gate_up_weight = False
350
- for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
351
- if weight_name not in name:
352
- continue
353
- param = state_dict[name.replace(weight_name, "gate_up_proj")]
354
- shard_size = param.shape[0] // 2
355
- loaded_weight = loaded_weight[
356
- shard_size * tensor_model_parallel_rank:shard_size *
357
- (tensor_model_parallel_rank + 1)]
358
- param_slice = param.data[shard_size * stride_id:shard_size *
359
- (stride_id + 1)]
360
- assert param_slice.shape == loaded_weight.shape
361
- param_slice.copy_(loaded_weight)
362
- is_gate_up_weight = True
363
- break
364
- if is_gate_up_weight:
365
- continue
366
-
367
- param = state_dict[name]
368
- if "embed_tokens" in name or "lm_head" in name:
369
- load_padded_tensor_parallel_vocab(param, loaded_weight,
370
- tensor_model_parallel_rank)
371
- continue
372
-
373
- load_tensor_parallel_weights(param, loaded_weight, name,
374
- self._column_parallel_weights,
375
- self._row_parallel_weights,
376
- tensor_model_parallel_rank)
334
+ else:
335
+ param = params_dict[name]
336
+ weight_loader = getattr(param, "weight_loader",
337
+ default_weight_loader)
338
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/baichuan.py
+65
-89
@@ -30,18 +30,20 @@ from torch import nn
30
31
from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.activation import SiluAndMul
33
-from vllm.model_executor.layers.layernorm import RMSNorm
33
from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
34
PagedAttentionWithALiBi)
35
+from vllm.model_executor.layers.layernorm import RMSNorm
36
+from vllm.model_executor.layers.linear import (LinearMethodBase,
37
+ MergedColumnParallelLinear,
38
+ QKVParallelLinear,
39
+ RowParallelLinear)
40
from vllm.model_executor.layers.sampler import Sampler
37
-from vllm.model_executor.weight_utils import (
38
- convert_pyslice_to_tensor, hf_model_weights_iterator,
39
- load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
41
+from vllm.model_executor.layers.vocab_parallel_embedding import (
42
+ VocabParallelEmbedding, ParallelLMHead)
43
from vllm.model_executor.parallel_utils.parallel_state import (
44
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43
- ColumnParallelLinear,
44
- RowParallelLinear)
45
+from vllm.model_executor.weight_utils import (default_weight_loader,
46
+ hf_model_weights_iterator)
47
from vllm.sequence import SamplerOutput
48
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
49
@@ -80,20 +82,17 @@ class BaiChuanMLP(nn.Module):
82
hidden_size: int,
83
intermediate_size: int,
84
hidden_act: str,
85
+ linear_method: Optional[LinearMethodBase] = None,
86
):
87
super().__init__()
85
- self.gate_up_proj = ColumnParallelLinear(
86
- hidden_size,
87
- 2 * intermediate_size,
88
- bias=False,
89
- gather_output=False,
90
- )
91
- self.down_proj = RowParallelLinear(
92
- intermediate_size,
93
- hidden_size,
88
+ self.gate_up_proj = MergedColumnParallelLinear(
89
+ hidden_size, [intermediate_size] * 2,
90
bias=False,
95
- input_is_parallel=True,
96
- )
91
+ linear_method=linear_method)
92
+ self.down_proj = RowParallelLinear(intermediate_size,
93
+ hidden_size,
94
+ bias=False,
95
+ linear_method=linear_method)
96
if hidden_act != "silu":
97
raise ValueError(f"Unsupported activation: {hidden_act}. "
98
"Only silu is supported for now.")
@@ -116,6 +115,7 @@ class BaiChuanAttention(nn.Module):
115
position_embedding: str,
116
rope_theta: float = 10000,
117
max_position_embeddings: int = 8192,
118
+ linear_method: Optional[LinearMethodBase] = None,
119
):
120
super().__init__()
121
self.hidden_size = hidden_size
@@ -131,17 +131,19 @@ class BaiChuanAttention(nn.Module):
131
self.max_position_embeddings = max_position_embeddings
132
133
# pylint: disable=invalid-name
134
- self.W_pack = ColumnParallelLinear(
134
+ self.W_pack = QKVParallelLinear(
135
hidden_size,
136
- 3 * hidden_size,
136
+ self.head_dim,
137
+ self.total_num_heads,
138
+ self.total_num_heads,
139
bias=False,
138
- gather_output=False,
140
+ linear_method=linear_method,
141
)
142
self.o_proj = RowParallelLinear(
143
self.total_num_heads * self.head_dim,
144
hidden_size,
145
bias=False,
144
- input_is_parallel=True,
146
+ linear_method=linear_method,
147
)
148
# Create the alibi slopes and slice them.
149
if self.postion_embedding == "ALIBI":
@@ -188,7 +190,10 @@ class BaiChuanAttention(nn.Module):
190
191
class BaiChuanDecoderLayer(nn.Module):
192
191
- def __init__(self, config: BaiChuanConfig, position_embedding: str):
193
+ def __init__(self,
194
+ config: BaiChuanConfig,
195
+ position_embedding: str,
196
+ linear_method: Optional[LinearMethodBase] = None):
197
super().__init__()
198
self.hidden_size = config.hidden_size
199
rope_theta = getattr(config, "rope_theta", 10000)
@@ -200,11 +205,13 @@ class BaiChuanDecoderLayer(nn.Module):
205
position_embedding=position_embedding,
206
rope_theta=rope_theta,
207
max_position_embeddings=max_position_embeddings,
208
+ linear_method=linear_method,
209
)
210
self.mlp = BaiChuanMLP(
211
hidden_size=self.hidden_size,
212
intermediate_size=config.intermediate_size,
213
hidden_act=config.hidden_act,
214
+ linear_method=linear_method,
215
)
216
self.input_layernorm = RMSNorm(config.hidden_size,
217
eps=config.rms_norm_eps)
@@ -241,7 +248,10 @@ class BaiChuanDecoderLayer(nn.Module):
248
249
class BaiChuanModel(nn.Module):
250
244
- def __init__(self, config: BaiChuanConfig, position_embedding: str):
251
+ def __init__(self,
252
+ config: BaiChuanConfig,
253
+ position_embedding: str,
254
+ linear_method: Optional[LinearMethodBase] = None):
255
super().__init__()
256
self.config = config
257
self.padding_idx = config.pad_token_id
@@ -252,7 +262,7 @@ class BaiChuanModel(nn.Module):
262
config.hidden_size,
263
)
264
self.layers = nn.ModuleList([
255
- BaiChuanDecoderLayer(config, position_embedding)
265
+ BaiChuanDecoderLayer(config, position_embedding, linear_method)
266
for _ in range(config.num_hidden_layers)
267
])
268
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -285,16 +295,15 @@ class BaiChuanModel(nn.Module):
295
296
class BaiChuanBaseForCausalLM(nn.Module):
297
288
- def __init__(self, config, position_embedding: str):
298
+ def __init__(self,
299
+ config,
300
+ position_embedding: str,
301
+ linear_method: Optional[LinearMethodBase] = None):
302
super().__init__()
303
self.config = config
291
- self.model = BaiChuanModel(config, position_embedding)
292
- self.lm_head = ColumnParallelLinear(
293
- config.hidden_size,
294
- config.vocab_size,
295
- bias=False,
296
- gather_output=False,
297
- )
304
+ self.linear_method = linear_method
305
+ self.model = BaiChuanModel(config, position_embedding, linear_method)
306
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
307
self.sampler = Sampler(config.vocab_size)
308
309
def forward(
@@ -311,79 +320,46 @@ class BaiChuanBaseForCausalLM(nn.Module):
320
input_metadata)
321
return next_tokens
322
314
- _column_parallel_weights = []
315
- _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
316
-
323
def load_weights(self,
324
model_name_or_path: str,
325
cache_dir: Optional[str] = None,
326
load_format: str = "auto",
327
revision: Optional[str] = None):
322
- tp_world_size = get_tensor_model_parallel_world_size()
323
- tp_rank = get_tensor_model_parallel_rank()
324
- state_dict = self.state_dict()
325
-
328
+ stacked_params_mapping = [
329
+ # (param_name, shard_name, shard_id)
330
+ ("gate_up_proj", "gate_proj", 0),
331
+ ("gate_up_proj", "up_proj", 1),
332
+ ]
333
+ params_dict = dict(self.named_parameters())
334
for name, loaded_weight in hf_model_weights_iterator(
335
model_name_or_path, cache_dir, load_format, revision):
336
if "rotary_emb.inv_freq" in name:
337
continue
330
-
331
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
332
-
333
- if "W_pack" in name:
334
- total_num_heads = self.config.num_attention_heads
335
- hidden_size = self.config.hidden_size
336
- head_size = hidden_size // total_num_heads
337
- num_heads = total_num_heads // tp_world_size
338
- head_start = tp_rank * num_heads
339
- head_end = (tp_rank + 1) * num_heads
340
-
341
- loaded_weight = loaded_weight.view(3, total_num_heads,
342
- head_size, hidden_size)
343
- loaded_weight = loaded_weight[:, head_start:head_end, :, :]
344
- loaded_weight = loaded_weight.reshape(-1, hidden_size)
345
-
346
- is_gate_up_weight = False
347
- for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
338
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
339
if weight_name not in name:
340
continue
350
- param = state_dict[name.replace(weight_name, "gate_up_proj")]
351
- shard_size = param.shape[0] // 2
352
- loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
353
- (tp_rank + 1)]
354
- param_slice = param.data[shard_size * stride_id:shard_size *
355
- (stride_id + 1)]
356
- assert param_slice.shape == loaded_weight.shape
357
- param_slice.copy_(loaded_weight)
358
- is_gate_up_weight = True
341
+ param = params_dict[name.replace(weight_name, param_name)]
342
+ weight_loader = param.weight_loader
343
+ weight_loader(param, loaded_weight, shard_id)
344
break
360
- if is_gate_up_weight:
361
- continue
362
-
363
- param = state_dict[name]
364
-
365
- if "embed_tokens" in name or "lm_head" in name:
366
- load_padded_tensor_parallel_vocab(param, loaded_weight,
367
- tp_rank)
368
- continue
369
-
370
- load_tensor_parallel_weights(
371
- param,
372
- loaded_weight,
373
- name,
374
- self._column_parallel_weights,
375
- self._row_parallel_weights,
376
- tp_rank,
377
- )
345
+ else:
346
+ param = params_dict[name]
347
+ weight_loader = getattr(param, "weight_loader",
348
+ default_weight_loader)
349
+ weight_loader(param, loaded_weight)
350
351
352
class BaichuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 13b
353
382
- def __init__(self, config):
383
- super().__init__(config, "ALIBI")
354
+ def __init__(self,
355
+ config,
356
+ linear_method: Optional[LinearMethodBase] = None):
357
+ super().__init__(config, "ALIBI", linear_method)
358
359
360
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 7b
361
388
- def __init__(self, config):
389
- super().__init__(config, "ROPE")
362
+ def __init__(self,
363
+ config,
364
+ linear_method: Optional[LinearMethodBase] = None):
365
+ super().__init__(config, "ROPE", linear_method)
vllm/model_executor/models/bloom.py
+69
-63
@@ -30,14 +30,17 @@ from transformers import BloomConfig
30
from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import get_act_fn
32
from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
33
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
+ LinearMethodBase,
35
+ QKVParallelLinear,
36
+ RowParallelLinear)
37
from vllm.model_executor.layers.sampler import Sampler
34
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35
- load_tensor_parallel_weights)
38
+from vllm.model_executor.layers.vocab_parallel_embedding import (
39
+ VocabParallelEmbedding)
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39
- ColumnParallelLinear,
40
- RowParallelLinear)
42
+from vllm.model_executor.weight_utils import (default_weight_loader,
43
+ hf_model_weights_iterator)
44
from vllm.sequence import SamplerOutput
45
46
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -70,7 +73,11 @@ def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
73
74
class BloomAttention(nn.Module):
75
73
- def __init__(self, config: BloomConfig):
76
+ def __init__(
77
+ self,
78
+ config: BloomConfig,
79
+ linear_method: Optional[LinearMethodBase] = None,
80
+ ):
81
super().__init__()
82
self.hidden_size = config.hidden_size
83
self.total_num_heads = config.n_head
@@ -81,17 +88,18 @@ class BloomAttention(nn.Module):
88
assert self.total_num_heads % tp_world_size == 0
89
self.num_heads = self.total_num_heads // tp_world_size
90
84
- self.query_key_value = ColumnParallelLinear(
91
+ self.query_key_value = QKVParallelLinear(
92
self.hidden_size,
86
- 3 * self.hidden_size,
93
+ self.head_dim,
94
+ self.total_num_heads,
95
bias=True,
88
- gather_output=False,
96
+ linear_method=linear_method,
97
)
98
self.dense = RowParallelLinear(
99
self.hidden_size,
100
self.hidden_size,
101
bias=True,
94
- input_is_parallel=True,
102
+ linear_method=linear_method,
103
)
104
105
# Create the alibi slopes and slice them.
@@ -125,19 +133,23 @@ class BloomAttention(nn.Module):
133
134
class BloomMLP(nn.Module):
135
128
- def __init__(self, config: BloomConfig):
136
+ def __init__(
137
+ self,
138
+ config: BloomConfig,
139
+ linear_method: Optional[LinearMethodBase] = None,
140
+ ):
141
super().__init__()
142
hidden_size = config.hidden_size
143
self.dense_h_to_4h = ColumnParallelLinear(
144
hidden_size,
145
4 * hidden_size,
134
- gather_output=False,
146
+ linear_method=linear_method,
147
)
148
self.act = get_act_fn("gelu")
149
self.dense_4h_to_h = RowParallelLinear(
150
4 * hidden_size,
151
hidden_size,
140
- input_is_parallel=True,
152
+ linear_method=linear_method,
153
)
154
155
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -149,16 +161,20 @@ class BloomMLP(nn.Module):
161
162
class BloomBlock(nn.Module):
163
152
- def __init__(self, config: BloomConfig):
164
+ def __init__(
165
+ self,
166
+ config: BloomConfig,
167
+ linear_method: Optional[LinearMethodBase] = None,
168
+ ):
169
super().__init__()
170
hidden_size = config.hidden_size
171
172
self.input_layernorm = nn.LayerNorm(hidden_size,
173
eps=config.layer_norm_epsilon)
158
- self.self_attention = BloomAttention(config)
174
+ self.self_attention = BloomAttention(config, linear_method)
175
self.post_attention_layernorm = nn.LayerNorm(
176
hidden_size, eps=config.layer_norm_epsilon)
161
- self.mlp = BloomMLP(config)
177
+ self.mlp = BloomMLP(config, linear_method)
178
self.apply_residual_connection_post_layernorm = (
179
config.apply_residual_connection_post_layernorm)
180
@@ -203,7 +219,11 @@ class BloomBlock(nn.Module):
219
220
class BloomModel(nn.Module):
221
206
- def __init__(self, config: BloomConfig):
222
+ def __init__(
223
+ self,
224
+ config: BloomConfig,
225
+ linear_method: Optional[LinearMethodBase] = None,
226
+ ):
227
super().__init__()
228
self.embed_dim = config.hidden_size
229
@@ -216,8 +236,10 @@ class BloomModel(nn.Module):
236
self.embed_dim, eps=config.layer_norm_epsilon)
237
238
# Transformer blocks
219
- self.h = nn.ModuleList(
220
- [BloomBlock(config) for _ in range(config.num_hidden_layers)])
239
+ self.h = nn.ModuleList([
240
+ BloomBlock(config, linear_method)
241
+ for _ in range(config.num_hidden_layers)
242
+ ])
243
244
# Final Layer Norm
245
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -251,12 +273,15 @@ class BloomModel(nn.Module):
273
274
class BloomForCausalLM(nn.Module):
275
254
- def __init__(self, config: BloomConfig):
276
+ def __init__(
277
+ self,
278
+ config: BloomConfig,
279
+ linear_method: Optional[LinearMethodBase] = None,
280
+ ):
281
super().__init__()
282
self.config = config
257
- self.transformer = BloomModel(config)
258
- # TODO(zhuohan): create a new weight after implementing pipeline
259
- # parallelism
283
+ self.linear_method = linear_method
284
+ self.transformer = BloomModel(config, linear_method)
285
self.lm_head_weight = self.transformer.word_embeddings.weight
286
self.sampler = Sampler(config.vocab_size)
287
@@ -274,55 +299,36 @@ class BloomForCausalLM(nn.Module):
299
input_metadata)
300
return next_tokens
301
277
- _column_parallel_weights = [
278
- "word_embeddings.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"
279
- ]
280
- _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
281
-
302
def load_weights(self,
303
model_name_or_path: str,
304
cache_dir: Optional[str] = None,
305
load_format: str = "auto",
306
revision: Optional[str] = None):
287
- tp_rank = get_tensor_model_parallel_rank()
288
- state_dict = self.state_dict()
307
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
308
for name, loaded_weight in hf_model_weights_iterator(
309
model_name_or_path, cache_dir, load_format, revision):
310
if name == "lm_head.weight":
292
- # Since hidden_states are parallelized, we need to
293
- # load lm_head.weight in parallel.
294
- self._column_parallel_weights.append(name)
295
- # If lm_head is provided, use it instead.
296
- param = self.lm_head_weight
297
- else:
298
- if not name.startswith("transformer."):
299
- name = "transformer." + name
300
- param = state_dict[name]
311
+ continue
312
+ if not name.startswith("transformer."):
313
+ name = "transformer." + name
314
+ param = params_dict[name]
315
316
if "query_key_value" in name:
303
- # NOTE(woosuk): BLOOM's fused QKV has the shape of
304
- # [num_heads * 3 * head_size, hidden_size], while the
305
- # required shape is [3 * num_heads * head_size, hidden_size].
317
+ # NOTE: BLOOM's fused QKV's output_dim has the shape of
318
+ # (num_heads * 3 * head_size), while the
319
+ # required shape is (3 * num_heads * head_size).
320
# Thus, we need weight conversion.
307
- shard_size = param.shape[0]
308
- start = shard_size * tp_rank
309
- end = shard_size * (tp_rank + 1)
310
- loaded_weight = loaded_weight[start:end]
311
-
321
+ output_dim = getattr(param, "output_dim", None)
322
num_heads = self.config.num_attention_heads
313
- hidden_size = self.config.hidden_size
314
- head_size = hidden_size // num_heads
315
- if "query_key_value.weight" in name:
316
- loaded_weight = loaded_weight.view(-1, 3, head_size,
317
- hidden_size)
318
- loaded_weight = loaded_weight.transpose(0, 1)
319
- loaded_weight = loaded_weight.reshape(-1, hidden_size)
320
- elif "query_key_value.bias" in name:
321
- loaded_weight = loaded_weight.view(-1, 3, head_size)
322
- loaded_weight = loaded_weight.transpose(0, 1)
323
- loaded_weight = loaded_weight.reshape(-1)
324
- else:
325
- raise ValueError(f"Unexpected weight name: {name}")
326
- load_tensor_parallel_weights(param, loaded_weight, name,
327
- self._column_parallel_weights,
328
- self._row_parallel_weights, tp_rank)
323
+ if output_dim is not None:
324
+ loaded_weight_shape = loaded_weight.shape
325
+ loaded_weight = loaded_weight.view(
326
+ loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
327
+ loaded_weight_shape[output_dim + 1:])
328
+ loaded_weight = loaded_weight.transpose(
329
+ output_dim, output_dim + 1)
330
+ loaded_weight = loaded_weight.reshape(loaded_weight_shape)
331
+
332
+ weight_loader = getattr(param, "weight_loader",
333
+ default_weight_loader)
334
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/chatglm.py
+79
-111
@@ -6,32 +6,28 @@
6
The input of the model is flattened to a 1D tensor of tokens. The model uses
7
InputMetadata to extract the original 2D shape of the input.
8
"""
9
-from typing import Dict, List, Optional, Tuple
9
+from typing import List, Optional, Tuple
10
11
import torch
12
from torch import nn
13
from torch.nn import LayerNorm
14
15
from vllm.model_executor.input_metadata import InputMetadata
16
+from vllm.model_executor.layers.activation import SiluAndMul
17
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
18
from vllm.model_executor.layers.layernorm import RMSNorm
18
-from vllm.model_executor.layers.activation import SiluAndMul
19
+from vllm.model_executor.layers.linear import (LinearMethodBase,
20
+ MergedColumnParallelLinear,
21
+ QKVParallelLinear,
22
+ RowParallelLinear)
23
from vllm.model_executor.layers.sampler import Sampler
20
-from vllm.model_executor.weight_utils import (
21
- hf_model_weights_iterator,
22
- load_tensor_parallel_weights,
23
-)
24
+from vllm.model_executor.layers.vocab_parallel_embedding import (
25
+ VocabParallelEmbedding, ParallelLMHead)
26
from vllm.model_executor.parallel_utils.parallel_state import (
25
- get_tensor_model_parallel_rank,
26
- get_tensor_model_parallel_world_size,
27
-)
28
-from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
29
-from vllm.model_executor.parallel_utils.layers import (
30
- ColumnParallelLinear,
31
- RowParallelLinear,
32
-)
33
-from vllm.sequence import SequenceOutputs
34
-
27
+ get_tensor_model_parallel_world_size)
28
+from vllm.model_executor.weight_utils import (default_weight_loader,
29
+ hf_model_weights_iterator)
30
+from vllm.sequence import SamplerOutput
31
from vllm.transformers_utils.configs import ChatGLMConfig
32
33
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -39,7 +35,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
35
36
class GLMAttention(nn.Module):
37
42
- def __init__(self, config):
38
+ def __init__(
39
+ self,
40
+ config,
41
+ linear_method: Optional[LinearMethodBase] = None,
42
+ ):
43
super().__init__()
44
self.hidden_size = config.hidden_size
45
tp_size = get_tensor_model_parallel_world_size()
@@ -50,25 +50,33 @@ class GLMAttention(nn.Module):
50
self.total_num_kv_heads = (config.multi_query_group_num
51
if config.multi_query_attention else
52
config.num_attention_heads)
53
- assert self.total_num_kv_heads % tp_size == 0
54
- self.num_kv_heads = self.total_num_kv_heads // tp_size
53
+ if self.total_num_kv_heads >= tp_size:
54
+ # Number of KV heads is greater than TP size, so we partition
55
+ # the KV heads across multiple tensor parallel GPUs.
56
+ assert self.total_num_kv_heads % tp_size == 0
57
+ else:
58
+ # Number of KV heads is less than TP size, so we replicate
59
+ # the KV heads across multiple tensor parallel GPUs.
60
+ assert tp_size % self.total_num_kv_heads == 0
61
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
62
self.head_dim = config.hidden_size // self.total_num_heads
63
self.q_size = self.num_heads * self.head_dim
64
self.kv_size = self.num_kv_heads * self.head_dim
65
self.scaling = self.head_dim**-0.5
66
60
- self.query_key_value = ColumnParallelLinear(
61
- config.hidden_size,
62
- (self.total_num_heads + 2 * self.total_num_kv_heads) *
67
+ self.query_key_value = QKVParallelLinear(
68
+ self.hidden_size,
69
self.head_dim,
64
- bias=config.add_qkv_bias,
65
- gather_output=False,
70
+ self.total_num_heads,
71
+ self.total_num_kv_heads,
72
+ bias=config.add_bias_linear or config.add_qkv_bias,
73
+ linear_method=linear_method,
74
)
75
self.dense = RowParallelLinear(
76
self.total_num_heads * self.head_dim,
77
config.hidden_size,
78
bias=config.add_bias_linear,
71
- input_is_parallel=True,
79
+ linear_method=linear_method,
80
)
81
82
self.attn = PagedAttentionWithRoPE(
@@ -78,7 +86,6 @@ class GLMAttention(nn.Module):
86
rotary_dim=self.head_dim // 2,
87
num_kv_heads=self.num_kv_heads,
88
is_neox_style=False,
81
- # is_glm_style=True
89
)
90
91
def forward(
@@ -117,17 +124,21 @@ class GLMMLP(nn.Module):
124
state back into h hidden dimension.
125
"""
126
120
- def __init__(self, config):
127
+ def __init__(
128
+ self,
129
+ config,
130
+ linear_method: Optional[LinearMethodBase] = None,
131
+ ):
132
super().__init__()
133
134
self.add_bias = config.add_bias_linear
135
136
# Project to 4h.
126
- self.dense_h_to_4h = ColumnParallelLinear(
137
+ self.dense_h_to_4h = MergedColumnParallelLinear(
138
config.hidden_size,
128
- config.ffn_hidden_size * 2,
139
+ [config.ffn_hidden_size] * 2,
140
bias=config.add_bias_linear,
130
- gather_output=False,
141
+ linear_method=linear_method,
142
)
143
144
self.activation_func = SiluAndMul()
@@ -137,7 +148,7 @@ class GLMMLP(nn.Module):
148
config.ffn_hidden_size,
149
config.hidden_size,
150
bias=config.add_bias_linear,
140
- input_is_parallel=True,
151
+ linear_method=linear_method,
152
)
153
154
def forward(self, hidden_states):
@@ -159,6 +170,7 @@ class GLMBlock(nn.Module):
170
def __init__(
171
self,
172
config,
173
+ linear_method: Optional[LinearMethodBase] = None,
174
):
175
super().__init__()
176
self.apply_residual_connection_post_layernorm = (
@@ -172,7 +184,7 @@ class GLMBlock(nn.Module):
184
eps=config.layernorm_epsilon)
185
186
# Self attention.
175
- self.self_attention = GLMAttention(config)
187
+ self.self_attention = GLMAttention(config, linear_method)
188
self.hidden_dropout = config.hidden_dropout
189
190
# Layernorm on the attention output
@@ -180,7 +192,7 @@ class GLMBlock(nn.Module):
192
config.hidden_size, eps=config.layernorm_epsilon)
193
194
# MLP
183
- self.mlp = GLMMLP(config)
195
+ self.mlp = GLMMLP(config, linear_method)
196
197
def forward(
198
self,
@@ -227,7 +239,11 @@ class GLMBlock(nn.Module):
239
class GLMTransformer(nn.Module):
240
"""Transformer class."""
241
230
- def __init__(self, config):
242
+ def __init__(
243
+ self,
244
+ config,
245
+ linear_method: Optional[LinearMethodBase] = None,
246
+ ):
247
super().__init__()
248
self.post_layer_norm = config.post_layer_norm
249
@@ -236,7 +252,7 @@ class GLMTransformer(nn.Module):
252
253
# Transformer layers.
254
self.layers = nn.ModuleList(
239
- [GLMBlock(config) for i in range(self.num_layers)])
255
+ [GLMBlock(config, linear_method) for i in range(self.num_layers)])
256
257
if self.post_layer_norm:
258
layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -274,7 +290,11 @@ class GLMTransformer(nn.Module):
290
291
class ChatGLMModel(nn.Module):
292
277
- def __init__(self, config):
293
+ def __init__(
294
+ self,
295
+ config,
296
+ linear_method: Optional[LinearMethodBase] = None,
297
+ ):
298
super().__init__()
299
300
self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
@@ -283,15 +303,10 @@ class ChatGLMModel(nn.Module):
303
self.num_layers = config.num_layers
304
self.multi_query_group_num = config.multi_query_group_num
305
self.kv_channels = config.kv_channels
286
- self.encoder = GLMTransformer(config)
306
+ self.encoder = GLMTransformer(config, linear_method)
307
288
- self.output_layer = ColumnParallelLinear(
289
- config.hidden_size,
290
- config.padded_vocab_size,
291
- bias=False,
292
- gather_output=False,
293
- params_dtype=config.torch_dtype,
294
- )
308
+ self.output_layer = ParallelLMHead(config.padded_vocab_size,
309
+ config.hidden_size)
310
311
def forward(
312
self,
@@ -317,10 +332,15 @@ class ChatGLMModel(nn.Module):
332
333
class ChatGLMForCausalLM(nn.Module):
334
320
- def __init__(self, config: ChatGLMConfig):
335
+ def __init__(
336
+ self,
337
+ config: ChatGLMConfig,
338
+ linear_method: Optional[LinearMethodBase] = None,
339
+ ):
340
super().__init__()
341
self.config: ChatGLMConfig = config
323
- self.transformer = ChatGLMModel(config)
342
+ self.linear_method = linear_method
343
+ self.transformer = ChatGLMModel(config, linear_method)
344
self.lm_head_weight = self.transformer.output_layer.weight
345
self.sampler = Sampler(config.padded_vocab_size)
346
@@ -331,78 +351,26 @@ class ChatGLMForCausalLM(nn.Module):
351
kv_caches: List[KVCache],
352
input_metadata: InputMetadata,
353
cache_events: Optional[List[torch.cuda.Event]],
334
- ) -> Dict[int, SequenceOutputs]:
354
+ ) -> SamplerOutput:
355
hidden_states = self.transformer(input_ids, positions, kv_caches,
356
input_metadata, cache_events)
357
next_tokens = self.sampler(self.lm_head_weight, hidden_states,
358
input_metadata)
359
return next_tokens
360
341
- _column_parallel_weights = [
342
- "output_layer.weight",
343
- "embedding.weight",
344
- ]
345
- _row_parallel_weights = ["dense_4h_to_h", "self_attention.dense"]
346
-
347
- def load_weights(
348
- self,
349
- model_name_or_path: str,
350
- cache_dir: Optional[str] = None,
351
- load_format: str = "auto",
352
- revision: Optional[str] = None,
353
- ):
354
- tp_rank = get_tensor_model_parallel_rank()
355
- tp_size = get_tensor_model_parallel_world_size()
356
-
357
- q_proj_shard_size = self.config.hidden_size // tp_size
358
- kv_proj_shard_size = (self.config.hidden_size //
359
- self.config.num_attention_heads *
360
- self.config.multi_query_group_num // tp_size)
361
-
362
- mlp_hidden_shard_size = self.config.ffn_hidden_size // tp_size
363
-
364
- state_dict = self.state_dict()
361
+ def load_weights(self,
362
+ model_name_or_path: str,
363
+ cache_dir: Optional[str] = None,
364
+ load_format: str = "auto",
365
+ revision: Optional[str] = None):
366
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
367
for name, loaded_weight in hf_model_weights_iterator(
368
model_name_or_path, cache_dir, load_format, revision):
369
+ if "rotary_pos_emb.inv_freq" in name:
370
+ continue
371
if "word_embeddings" in name:
372
name = name.replace(".word_embeddings", "")
369
-
370
- if name in state_dict:
371
- param = state_dict[name]
372
- if "query_key_value" in name:
373
- q_offset = q_proj_shard_size * tp_rank
374
- k_offset = (q_proj_shard_size * tp_size +
375
- kv_proj_shard_size * tp_rank)
376
- v_offset = (q_proj_shard_size * tp_size +
377
- kv_proj_shard_size * (tp_size + tp_rank))
378
- wq = loaded_weight[q_offset:q_offset + q_proj_shard_size]
379
- wk = loaded_weight[k_offset:k_offset + kv_proj_shard_size]
380
- wv = loaded_weight[v_offset:v_offset + kv_proj_shard_size]
381
- loaded_weight = torch.cat([wq, wk, wv], dim=0)
382
- param.data.copy_(loaded_weight)
383
- continue
384
-
385
- if "dense_h_to_4h" in name:
386
- w_gate = loaded_weight[mlp_hidden_shard_size *
387
- tp_rank:mlp_hidden_shard_size *
388
- (tp_rank + 1)]
389
- w_proj = loaded_weight[mlp_hidden_shard_size *
390
- (tp_size +
391
- tp_rank):mlp_hidden_shard_size *
392
- (tp_size + tp_rank + 1)]
393
- loaded_weight = torch.cat([w_gate, w_proj], dim=0)
394
- param.data.copy_(loaded_weight)
395
- continue
396
-
397
- load_tensor_parallel_weights(
398
- param,
399
- loaded_weight,
400
- name,
401
- self._column_parallel_weights,
402
- self._row_parallel_weights,
403
- tp_rank,
404
- )
405
- elif name == "transformer.rotary_pos_emb.inv_freq":
406
- continue
407
- else:
408
- print("Warning never found tensor's name:", name)
373
+ param = params_dict[name]
374
+ weight_loader = getattr(param, "weight_loader",
375
+ default_weight_loader)
376
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/falcon.py
+93
-150
@@ -30,17 +30,19 @@ from vllm.model_executor.input_metadata import InputMetadata
30
from vllm.model_executor.layers.attention import (PagedAttention,
31
PagedAttentionWithALiBi,
32
PagedAttentionWithRoPE)
33
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
+ LinearMethodBase,
35
+ QKVParallelLinear,
36
+ RowParallelLinear)
37
from vllm.model_executor.layers.sampler import Sampler
34
-from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
35
- hf_model_weights_iterator,
36
- load_tensor_parallel_weights)
37
-from vllm.model_executor.parallel_utils.parallel_state import (
38
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40
- ColumnParallelLinear,
41
- RowParallelLinear)
38
+from vllm.model_executor.layers.vocab_parallel_embedding import (
39
+ VocabParallelEmbedding, ParallelLMHead)
40
from vllm.model_executor.parallel_utils.communication_op import (
41
tensor_model_parallel_all_reduce)
42
+from vllm.model_executor.parallel_utils.parallel_state import (
43
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.weight_utils import (default_weight_loader,
45
+ hf_model_weights_iterator)
46
from vllm.sequence import SamplerOutput
47
from vllm.transformers_utils.configs import RWConfig
48
@@ -48,19 +50,6 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
50
FalconConfig = Union[HF_FalconConfig, RWConfig]
51
52
51
-# NOTE(Hesslow): Unfortunately we did not fuse matmul and bias during
52
-# training, this means that there's one additional quantization to bfloat16
53
-# between the operations. In order not to degrade the quality of our HF-port,
54
-# we keep these characteristics in the final model.
55
-class FalconLinear(nn.Linear):
56
-
57
- def forward(self, x: torch.Tensor) -> torch.Tensor:
58
- hidden_states = x @ self.weight.T
59
- if self.bias is None:
60
- return hidden_states
61
- return hidden_states + self.bias
62
-
63
-
53
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
54
closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
55
base = torch.tensor(2**(-(2**-(math.log2(closest_power_of_2) - 3))),
@@ -86,7 +75,11 @@ def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
75
76
class FalconAttention(nn.Module):
77
89
- def __init__(self, config: FalconConfig):
78
+ def __init__(
79
+ self,
80
+ config: FalconConfig,
81
+ linear_method: Optional[LinearMethodBase] = None,
82
+ ):
83
super().__init__()
84
85
self.hidden_size = config.hidden_size
@@ -103,41 +96,29 @@ class FalconAttention(nn.Module):
96
97
if self.new_decoder_architecture:
98
self.total_num_kv_heads = config.num_kv_heads
106
- assert self.total_num_heads % tp_size == 0
107
- self.num_kv_heads = self.total_num_kv_heads // tp_size
108
- self.query_key_value = ColumnParallelLinear(
109
- self.hidden_size,
110
- (self.total_num_heads + 2 * self.total_num_kv_heads) *
111
- self.head_dim,
112
- bias=config.bias,
113
- gather_output=False,
114
- skip_bias_add=True,
115
- )
99
elif self.multi_query:
100
self.total_num_kv_heads = 1
118
- self.num_kv_heads = 1
119
- self.query = ColumnParallelLinear(
120
- self.hidden_size,
121
- self.total_num_heads * self.head_dim,
122
- bias=config.bias,
123
- gather_output=False,
124
- skip_bias_add=True,
125
- )
126
- self.key_value = FalconLinear(self.hidden_size,
127
- 2 * self.head_dim,
128
- bias=config.bias)
101
else:
102
self.total_num_kv_heads = self.total_num_heads
131
- self.num_kv_heads = self.num_heads
132
- self.query_key_value = ColumnParallelLinear(
133
- self.hidden_size,
134
- (self.total_num_heads + 2 * self.total_num_kv_heads) *
135
- self.head_dim,
136
- bias=config.bias,
137
- gather_output=False,
138
- skip_bias_add=True,
139
- )
103
+ if self.total_num_kv_heads >= tp_size:
104
+ # Number of KV heads is greater than TP size, so we partition
105
+ # the KV heads across multiple tensor parallel GPUs.
106
+ assert self.total_num_kv_heads % tp_size == 0
107
+ else:
108
+ # Number of KV heads is less than TP size, so we replicate
109
+ # the KV heads across multiple tensor parallel GPUs.
110
+ assert tp_size % self.total_num_kv_heads == 0
111
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112
113
+ self.query_key_value = QKVParallelLinear(
114
+ self.hidden_size,
115
+ self.head_dim,
116
+ self.total_num_heads,
117
+ self.total_num_kv_heads,
118
+ bias=config.bias,
119
+ skip_bias_add=True,
120
+ linear_method=linear_method,
121
+ )
122
self.q_size = self.num_heads * self.head_dim
123
self.kv_size = self.num_kv_heads * self.head_dim
124
@@ -149,7 +130,6 @@ class FalconAttention(nn.Module):
130
self.hidden_size,
131
self.hidden_size,
132
bias=config.bias,
152
- input_is_parallel=True,
133
skip_bias_add=True,
134
reduce_results=self.reduce_row_parallel_results)
135
@@ -196,18 +176,10 @@ class FalconAttention(nn.Module):
176
input_metadata: InputMetadata,
177
cache_event: Optional[torch.cuda.Event],
178
) -> torch.Tensor:
199
- if not self.new_decoder_architecture and self.multi_query:
200
- q, bias = self.query(hidden_states)
201
- if bias is not None:
202
- q += bias
203
- kv = self.key_value(hidden_states)
204
- k, v = kv.split([self.kv_size, self.kv_size], dim=-1)
205
- else:
206
- qkv, bias = self.query_key_value(hidden_states)
207
- if bias is not None:
208
- qkv += bias
209
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size],
210
- dim=-1)
179
+ qkv, bias = self.query_key_value(hidden_states)
180
+ if bias is not None:
181
+ qkv += bias
182
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
183
k_cache, v_cache = kv_cache
184
if self.use_rotary:
185
attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
@@ -221,15 +193,19 @@ class FalconAttention(nn.Module):
193
194
class FalconMLP(nn.Module):
195
224
- def __init__(self, config: FalconConfig):
196
+ def __init__(
197
+ self,
198
+ config: FalconConfig,
199
+ linear_method: Optional[LinearMethodBase] = None,
200
+ ):
201
super().__init__()
202
hidden_size = config.hidden_size
203
204
self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
205
4 * hidden_size,
206
bias=config.bias,
231
- gather_output=False,
232
- skip_bias_add=True)
207
+ skip_bias_add=True,
208
+ linear_method=linear_method)
209
self.act = nn.GELU()
210
self.reduce_row_parallel_results = not (config.new_decoder_architecture
211
or config.parallel_attn)
@@ -237,9 +213,9 @@ class FalconMLP(nn.Module):
213
4 * hidden_size,
214
hidden_size,
215
bias=config.bias,
240
- input_is_parallel=True,
216
skip_bias_add=True,
242
- reduce_results=self.reduce_row_parallel_results)
217
+ reduce_results=self.reduce_row_parallel_results,
218
+ linear_method=linear_method)
219
220
def forward(self, x: torch.Tensor) -> torch.Tensor:
221
# NOTE(zhuohan): Following huggingface, we do not fuse bias add here.
@@ -253,12 +229,16 @@ class FalconMLP(nn.Module):
229
230
class FalconDecoderLayer(nn.Module):
231
256
- def __init__(self, config: FalconConfig):
232
+ def __init__(
233
+ self,
234
+ config: FalconConfig,
235
+ linear_method: Optional[LinearMethodBase] = None,
236
+ ):
237
super().__init__()
238
hidden_size = config.hidden_size
239
self.num_heads = config.num_attention_heads
260
- self.self_attention = FalconAttention(config)
261
- self.mlp = FalconMLP(config)
240
+ self.self_attention = FalconAttention(config, linear_method)
241
+ self.mlp = FalconMLP(config, linear_method)
242
self.config = config
243
244
if config.new_decoder_architecture:
@@ -334,7 +314,11 @@ class FalconDecoderLayer(nn.Module):
314
315
class FalconModel(nn.Module):
316
337
- def __init__(self, config: FalconConfig):
317
+ def __init__(
318
+ self,
319
+ config: FalconConfig,
320
+ linear_method: Optional[LinearMethodBase] = None,
321
+ ):
322
super().__init__()
323
self.config = config
324
self.embed_dim = config.hidden_size
@@ -349,7 +333,8 @@ class FalconModel(nn.Module):
333
334
# Transformer blocks
335
self.h = nn.ModuleList([
352
- FalconDecoderLayer(config) for _ in range(config.num_hidden_layers)
336
+ FalconDecoderLayer(config, linear_method)
337
+ for _ in range(config.num_hidden_layers)
338
])
339
340
# Final Layer Norm
@@ -383,15 +368,18 @@ class FalconModel(nn.Module):
368
369
class FalconForCausalLM(nn.Module):
370
386
- def __init__(self, config: FalconConfig):
371
+ def __init__(
372
+ self,
373
+ config: FalconConfig,
374
+ linear_method: Optional[LinearMethodBase] = None,
375
+ ):
376
super().__init__()
377
self.config = config
389
- self.transformer = FalconModel(config)
390
- self.lm_head = ColumnParallelLinear(
391
- config.hidden_size,
378
+ self.linear_method = linear_method
379
+ self.transformer = FalconModel(config, linear_method)
380
+ self.lm_head = ParallelLMHead(
381
config.vocab_size,
393
- bias=False,
394
- gather_output=False,
382
+ config.hidden_size,
383
)
384
self.sampler = Sampler(config.vocab_size)
385
@@ -415,89 +403,44 @@ class FalconForCausalLM(nn.Module):
403
404
return next_tokens
405
418
- _column_parallel_weights = [
419
- "word_embeddings.weight", "lm_head.weight", "dense_h_to_4h.weight",
420
- "dense_h_to_4h.bias"
421
- ]
422
- _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
423
-
406
def load_weights(self,
407
model_name_or_path: str,
408
cache_dir: Optional[str] = None,
409
load_format: str = "auto",
410
revision: Optional[str] = None):
429
- tp_size = (get_tensor_model_parallel_world_size())
430
- tp_rank = get_tensor_model_parallel_rank()
431
-
432
- hidden_size = self.config.hidden_size
411
total_num_heads = self.config.num_attention_heads
434
- num_heads = total_num_heads // tp_size
435
- head_size = hidden_size // total_num_heads
436
- head_start = tp_rank * num_heads
437
- head_end = (tp_rank + 1) * num_heads
412
if self.config.new_decoder_architecture:
413
total_num_kv_heads = self.config.num_kv_heads
440
- num_kv_heads = total_num_kv_heads // tp_size
441
- separated_q_kv = False
442
- kv_head_start = tp_rank * num_kv_heads
443
- kv_head_end = (tp_rank + 1) * num_kv_heads
414
elif self.config.multi_query:
415
total_num_kv_heads = 1
446
- num_kv_heads = 1
447
- separated_q_kv = True
448
- kv_head_start = 0
449
- kv_head_end = 1
416
else:
417
total_num_kv_heads = total_num_heads
452
- num_kv_heads = total_num_kv_heads // tp_size
453
- separated_q_kv = False
454
- kv_head_start = tp_rank * num_kv_heads
455
- kv_head_end = (tp_rank + 1) * num_kv_heads
418
num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
457
- state_dict = self.state_dict()
458
-
419
+ params_dict = dict(self.named_parameters())
420
for name, loaded_weight in hf_model_weights_iterator(
421
model_name_or_path, cache_dir, load_format, revision):
422
+ param = params_dict[name]
423
if "query_key_value" in name:
462
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
463
- loaded_weight_size = loaded_weight.size()
424
+ output_dim = getattr(param, "output_dim", None)
425
+ loaded_weight_shape = loaded_weight.shape
426
loaded_weight = loaded_weight.view(
465
- total_num_kv_heads, num_query_heads_per_kv_head + 2,
466
- head_size, *loaded_weight_size[1:])
467
-
468
- wq = loaded_weight[:, :-2].reshape(-1, *loaded_weight_size[1:])
469
- wk = loaded_weight[:, [-2]].reshape(-1,
470
- *loaded_weight_size[1:])
471
- wv = loaded_weight[:, [-1]].reshape(-1,
472
- *loaded_weight_size[1:])
473
-
474
- wq = wq[head_size * head_start:head_size * head_end]
475
- wk = wk[head_size * kv_head_start:head_size * kv_head_end]
476
- wv = wv[head_size * kv_head_start:head_size * kv_head_end]
477
-
478
- if separated_q_kv:
479
- loaded_weight_q = wq
480
- loaded_weight_kv = torch.cat([wk, wv], dim=0)
481
- q_weight_name = name.replace("query_key_value", "query")
482
- kv_weight_name = name.replace("query_key_value",
483
- "key_value")
484
- load_tensor_parallel_weights(state_dict[q_weight_name],
485
- loaded_weight_q,
486
- q_weight_name,
487
- self._column_parallel_weights,
488
- self._row_parallel_weights,
489
- tp_rank)
490
- load_tensor_parallel_weights(state_dict[kv_weight_name],
491
- loaded_weight_kv,
492
- kv_weight_name,
493
- self._column_parallel_weights,
494
- self._row_parallel_weights,
495
- tp_rank)
496
- continue
497
- else:
498
- loaded_weight = torch.cat([wq, wk, wv], dim=0)
499
-
500
- param = state_dict[name]
501
- load_tensor_parallel_weights(param, loaded_weight, name,
502
- self._column_parallel_weights,
503
- self._row_parallel_weights, tp_rank)
427
+ loaded_weight_shape[:output_dim] +
428
+ (total_num_kv_heads, num_query_heads_per_kv_head + 2, -1) +
429
+ loaded_weight_shape[output_dim + 1:])
430
+ wq = loaded_weight.narrow(
431
+ output_dim + 1, 0, num_query_heads_per_kv_head).reshape(
432
+ *loaded_weight_shape[:output_dim], -1,
433
+ *loaded_weight_shape[output_dim + 1:])
434
+ wk = loaded_weight.narrow(
435
+ output_dim + 1, num_query_heads_per_kv_head,
436
+ 1).reshape(*loaded_weight_shape[:output_dim], -1,
437
+ *loaded_weight_shape[output_dim + 1:])
438
+ wv = loaded_weight.narrow(
439
+ output_dim + 1, num_query_heads_per_kv_head + 1,
440
+ 1).reshape(*loaded_weight_shape[:output_dim], -1,
441
+ *loaded_weight_shape[output_dim + 1:])
442
+ loaded_weight = torch.cat([wq, wk, wv], dim=output_dim)
443
+
444
+ weight_loader = getattr(param, "weight_loader",
445
+ default_weight_loader)
446
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt2.py
+52
-79
@@ -30,15 +30,17 @@ from transformers import GPT2Config
30
from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import get_act_fn
32
from vllm.model_executor.layers.attention import PagedAttention
33
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
+ LinearMethodBase,
35
+ QKVParallelLinear,
36
+ RowParallelLinear)
37
from vllm.model_executor.layers.sampler import Sampler
34
-from vllm.model_executor.weight_utils import (
35
- convert_pyslice_to_tensor, hf_model_weights_iterator,
36
- load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
38
+from vllm.model_executor.layers.vocab_parallel_embedding import (
39
+ VocabParallelEmbedding)
40
from vllm.model_executor.parallel_utils.parallel_state import (
38
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40
- ColumnParallelLinear,
41
- RowParallelLinear)
41
+ get_tensor_model_parallel_world_size)
42
+from vllm.model_executor.weight_utils import (default_weight_loader,
43
+ hf_model_weights_iterator)
44
from vllm.sequence import SamplerOutput
45
46
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -46,7 +48,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
48
49
class GPT2Attention(nn.Module):
50
49
- def __init__(self, config: GPT2Config):
51
+ def __init__(
52
+ self,
53
+ config: GPT2Config,
54
+ linear_method: Optional[LinearMethodBase] = None,
55
+ ):
56
super().__init__()
57
self.hidden_size = config.hidden_size
58
total_num_heads = config.num_attention_heads
@@ -57,17 +63,18 @@ class GPT2Attention(nn.Module):
63
self.head_dim = self.hidden_size // total_num_heads
64
self.scale = self.head_dim**-0.5
65
60
- self.c_attn = ColumnParallelLinear(
66
+ self.c_attn = QKVParallelLinear(
67
self.hidden_size,
62
- 3 * self.hidden_size,
68
+ self.head_dim,
69
+ total_num_heads,
70
bias=True,
64
- gather_output=False,
71
+ linear_method=linear_method,
72
)
73
self.c_proj = RowParallelLinear(
74
self.hidden_size,
75
self.hidden_size,
76
bias=True,
70
- input_is_parallel=True,
77
+ linear_method=linear_method,
78
)
79
self.attn = PagedAttention(self.num_heads,
80
self.head_dim,
@@ -95,6 +102,7 @@ class GPT2MLP(nn.Module):
102
self,
103
intermediate_size: int,
104
config: GPT2Config,
105
+ linear_method: Optional[LinearMethodBase] = None,
106
):
107
super().__init__()
108
hidden_size = config.hidden_size
@@ -102,13 +110,13 @@ class GPT2MLP(nn.Module):
110
hidden_size,
111
intermediate_size,
112
bias=True,
105
- gather_output=False,
113
+ linear_method=linear_method,
114
)
115
self.c_proj = RowParallelLinear(
116
intermediate_size,
117
hidden_size,
118
bias=True,
111
- input_is_parallel=True,
119
+ linear_method=linear_method,
120
)
121
self.act = get_act_fn(config.activation_function)
122
@@ -121,16 +129,20 @@ class GPT2MLP(nn.Module):
129
130
class GPT2Block(nn.Module):
131
124
- def __init__(self, config: GPT2Config):
132
+ def __init__(
133
+ self,
134
+ config: GPT2Config,
135
+ linear_method: Optional[LinearMethodBase] = None,
136
+ ):
137
super().__init__()
138
hidden_size = config.hidden_size
139
inner_dim = (config.n_inner if config.n_inner is not None else 4 *
140
hidden_size)
141
142
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
131
- self.attn = GPT2Attention(config)
143
+ self.attn = GPT2Attention(config, linear_method)
144
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
133
- self.mlp = GPT2MLP(inner_dim, config)
145
+ self.mlp = GPT2MLP(inner_dim, config, linear_method)
146
147
def forward(
148
self,
@@ -160,24 +172,23 @@ class GPT2Block(nn.Module):
172
173
class GPT2Model(nn.Module):
174
163
- def __init__(self, config: GPT2Config):
175
+ def __init__(
176
+ self,
177
+ config: GPT2Config,
178
+ linear_method: Optional[LinearMethodBase] = None,
179
+ ):
180
super().__init__()
181
self.config = config
182
assert not config.add_cross_attention
183
assert not config.scale_attn_by_inverse_layer_idx
184
assert not config.reorder_and_upcast_attn
185
self.embed_dim = config.hidden_size
170
-
171
- # Optimization: While the vocab size of GPT-2 is 50257, we extend it
172
- # to 50304 in order to make it divisible by 64.
173
- # This improves performance since GPUs are faster if the dimension
174
- # is divisible by 64. In addition, it allows us to shard the embedding
175
- # layer across 2, 4, 8, or more GPUs.
176
- vocab_size = ((config.vocab_size + 63) // 64) * 64
177
- self.wte = VocabParallelEmbedding(vocab_size, self.embed_dim)
186
+ self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
187
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
179
- self.h = nn.ModuleList(
180
- [GPT2Block(config) for _ in range(config.num_hidden_layers)])
188
+ self.h = nn.ModuleList([
189
+ GPT2Block(config, linear_method)
190
+ for _ in range(config.num_hidden_layers)
191
+ ])
192
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
193
194
def forward(
@@ -207,12 +218,15 @@ class GPT2Model(nn.Module):
218
219
class GPT2LMHeadModel(nn.Module):
220
210
- def __init__(self, config: GPT2Config):
221
+ def __init__(
222
+ self,
223
+ config: GPT2Config,
224
+ linear_method: Optional[LinearMethodBase] = None,
225
+ ):
226
super().__init__()
227
self.config = config
213
- self.transformer = GPT2Model(config)
214
- # TODO(zhuohan): create a new weight after implementing pipeline
215
- # parallelism
228
+ self.linear_method = linear_method
229
+ self.transformer = GPT2Model(config, linear_method)
230
self.lm_head_weight = self.transformer.wte.weight
231
self.sampler = Sampler(config.vocab_size)
232
@@ -230,19 +244,12 @@ class GPT2LMHeadModel(nn.Module):
244
input_metadata)
245
return next_tokens
246
233
- _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
234
- _row_parallel_weights = ["c_proj.weight"]
235
-
247
def load_weights(self,
248
model_name_or_path: str,
249
cache_dir: Optional[str] = None,
250
load_format: str = "auto",
251
revision: Optional[str] = None):
241
- tensor_model_parallel_world_size = (
242
- get_tensor_model_parallel_world_size())
243
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
244
- state_dict = self.state_dict()
245
-
252
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
253
for name, loaded_weight in hf_model_weights_iterator(
254
model_name_or_path, cache_dir, load_format, revision):
255
if "lm_head.weight" in name:
@@ -253,53 +260,19 @@ class GPT2LMHeadModel(nn.Module):
260
# Skip attention mask.
261
# NOTE: "c_attn.bias" should not be skipped.
262
continue
256
-
263
if not name.startswith("transformer."):
264
name = "transformer." + name
259
-
260
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
261
-
265
+ param = params_dict[name]
266
# The HF's GPT-2 implementation uses Conv1D instead of Linear.
267
# Because of this, we need to transpose the weights.
268
+ # Note(zhuohan): the logic below might break quantized models.
269
for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
270
if conv1d_weight_name not in name:
271
continue
272
if not name.endswith(".weight"):
273
continue
274
loaded_weight = loaded_weight.t()
270
- param = state_dict[name]
271
-
272
- if name == "transformer.wte.weight":
273
- load_padded_tensor_parallel_vocab(param, loaded_weight,
274
- tensor_model_parallel_rank)
275
- continue
276
-
277
- # For the fused QKV linear layer, manually shard the weights.
278
- if "c_attn" in name:
279
- # GPT-2's fused QKV has the shape of
280
- # [3 * num_heads * head_size, hidden_size].
281
- # When tensor parallelism is used, we shard the weights along
282
- # the head dimension.
283
- total_num_heads = self.config.num_attention_heads
284
- hidden_size = self.config.hidden_size
285
- head_size = hidden_size // total_num_heads
286
- num_heads = total_num_heads // tensor_model_parallel_world_size
287
- head_start = tensor_model_parallel_rank * num_heads
288
- head_end = (tensor_model_parallel_rank + 1) * num_heads
275
290
- if name.endswith(".weight"):
291
- loaded_weight = loaded_weight.view(3, total_num_heads,
292
- head_size, hidden_size)
293
- loaded_weight = loaded_weight[:, head_start:head_end, :, :]
294
- loaded_weight = loaded_weight.reshape(-1, hidden_size)
295
- elif name.endswith(".bias"):
296
- loaded_weight = loaded_weight.view(3, total_num_heads,
297
- head_size)
298
- loaded_weight = loaded_weight[:, head_start:head_end, :]
299
- loaded_weight = loaded_weight.reshape(-1)
300
- else:
301
- raise ValueError(f"Unexpected parameter name {name}")
302
- load_tensor_parallel_weights(param, loaded_weight, name,
303
- self._column_parallel_weights,
304
- self._row_parallel_weights,
305
- tensor_model_parallel_rank)
276
+ weight_loader = getattr(param, "weight_loader",
277
+ default_weight_loader)
278
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_bigcode.py
+63
-126
@@ -31,15 +31,17 @@ from transformers import GPTBigCodeConfig
31
from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.activation import get_act_fn
33
from vllm.model_executor.layers.attention import PagedAttention
34
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
+ LinearMethodBase,
36
+ QKVParallelLinear,
37
+ RowParallelLinear)
38
from vllm.model_executor.layers.sampler import Sampler
35
-from vllm.model_executor.weight_utils import (
36
- convert_pyslice_to_tensor, hf_model_weights_iterator,
37
- load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
39
+from vllm.model_executor.layers.vocab_parallel_embedding import (
40
+ VocabParallelEmbedding)
41
from vllm.model_executor.parallel_utils.parallel_state import (
39
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
41
- ColumnParallelLinear,
42
- RowParallelLinear)
42
+ get_tensor_model_parallel_world_size)
43
+from vllm.model_executor.weight_utils import (default_weight_loader,
44
+ hf_model_weights_iterator)
45
from vllm.sequence import SamplerOutput
46
47
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -47,7 +49,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
49
50
class GPTBigCodeAttention(nn.Module):
51
50
- def __init__(self, config: GPTBigCodeConfig):
52
+ def __init__(
53
+ self,
54
+ config: GPTBigCodeConfig,
55
+ linear_method: Optional[LinearMethodBase] = None,
56
+ ):
57
super().__init__()
58
self.hidden_size = config.hidden_size
59
total_num_heads = config.num_attention_heads
@@ -61,32 +67,26 @@ class GPTBigCodeAttention(nn.Module):
67
68
self.multi_query = config.multi_query
69
if self.multi_query:
70
+ total_num_kv_heads = 1
71
self.num_kv_heads = 1
65
- self.kv_dim = self.head_dim
66
- self.c_attn_q = ColumnParallelLinear(
67
- self.hidden_size,
68
- self.hidden_size,
69
- bias=True,
70
- gather_output=False,
71
- )
72
- self.c_attn_kv = nn.Linear(self.hidden_size,
73
- 2 * self.kv_dim,
74
- bias=True)
72
else:
73
+ total_num_kv_heads = total_num_heads
74
self.num_kv_heads = self.num_heads
77
- self.kv_dim = self.num_kv_heads * self.head_dim
78
- self.c_attn = ColumnParallelLinear(
79
- self.hidden_size,
80
- self.hidden_size + 2 * self.kv_dim,
81
- bias=True,
82
- gather_output=False,
83
- )
75
+ self.kv_dim = self.head_dim * self.num_kv_heads
76
+ self.c_attn = QKVParallelLinear(
77
+ self.hidden_size,
78
+ self.head_dim,
79
+ total_num_heads,
80
+ total_num_kv_heads,
81
+ bias=True,
82
+ linear_method=linear_method,
83
+ )
84
85
self.c_proj = RowParallelLinear(
86
self.hidden_size,
87
self.hidden_size,
88
bias=True,
89
- input_is_parallel=True,
89
+ linear_method=linear_method,
90
)
91
self.attn = PagedAttention(self.num_heads,
92
self.head_dim,
@@ -100,17 +100,14 @@ class GPTBigCodeAttention(nn.Module):
100
input_metadata: InputMetadata,
101
cache_event: Optional[torch.cuda.Event],
102
) -> torch.Tensor:
103
- if self.multi_query:
104
- q, _ = self.c_attn_q(hidden_states)
105
- kv = self.c_attn_kv(hidden_states)
106
- k, v = kv.split([self.kv_dim, self.kv_dim], dim=-1)
107
- else:
108
- qkv, _ = self.c_attn(hidden_states)
109
- q, k, v = qkv.split([
103
+ qkv, _ = self.c_attn(hidden_states)
104
+ q, k, v = qkv.split(
105
+ [
106
self.hidden_size // self.tensor_model_parallel_world_size,
107
self.kv_dim, self.kv_dim
108
],
113
- dim=-1)
109
+ dim=-1,
110
+ )
111
key_cache, value_cache = kv_cache
112
attn_output = self.attn(q, k, v, key_cache, value_cache,
113
input_metadata, cache_event)
@@ -124,6 +121,7 @@ class GPTBigMLP(nn.Module):
121
self,
122
intermediate_size: int,
123
config: GPTBigCodeConfig,
124
+ linear_method: Optional[LinearMethodBase] = None,
125
):
126
super().__init__()
127
hidden_size = config.hidden_size
@@ -131,13 +129,13 @@ class GPTBigMLP(nn.Module):
129
hidden_size,
130
intermediate_size,
131
bias=True,
134
- gather_output=False,
132
+ linear_method=linear_method,
133
)
134
self.c_proj = RowParallelLinear(
135
intermediate_size,
136
hidden_size,
137
bias=True,
140
- input_is_parallel=True,
138
+ linear_method=linear_method,
139
)
140
self.act = get_act_fn(config.activation_function)
141
@@ -150,16 +148,20 @@ class GPTBigMLP(nn.Module):
148
149
class GPTBigCodeBlock(nn.Module):
150
153
- def __init__(self, config: GPTBigCodeConfig):
151
+ def __init__(
152
+ self,
153
+ config: GPTBigCodeConfig,
154
+ linear_method: Optional[LinearMethodBase] = None,
155
+ ):
156
super().__init__()
157
hidden_size = config.hidden_size
158
inner_dim = (config.n_inner if config.n_inner is not None else 4 *
159
hidden_size)
160
161
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
160
- self.attn = GPTBigCodeAttention(config)
162
+ self.attn = GPTBigCodeAttention(config, linear_method)
163
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
162
- self.mlp = GPTBigMLP(inner_dim, config)
164
+ self.mlp = GPTBigMLP(inner_dim, config, linear_method)
165
166
def forward(
167
self,
@@ -189,23 +191,23 @@ class GPTBigCodeBlock(nn.Module):
191
192
class GPTBigCodeModel(nn.Module):
193
192
- def __init__(self, config: GPTBigCodeConfig):
194
+ def __init__(
195
+ self,
196
+ config: GPTBigCodeConfig,
197
+ linear_method: Optional[LinearMethodBase] = None,
198
+ ):
199
super().__init__()
200
self.config = config
201
assert not config.add_cross_attention
202
203
self.embed_dim = config.hidden_size
204
199
- # Optimization: While the vocab size of GPT-2 is 50257, we extend it
200
- # to 50304 in order to make it divisible by 64.
201
- # This improves performance since GPUs are faster if the dimension
202
- # is divisible by 64. In addition, it allows us to shard the embedding
203
- # layer across 2, 4, 8, or more GPUs.
204
- vocab_size = ((config.vocab_size + 63) // 64) * 64
205
- self.wte = VocabParallelEmbedding(vocab_size, self.embed_dim)
205
+ self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
206
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
207
- self.h = nn.ModuleList(
208
- [GPTBigCodeBlock(config) for _ in range(config.num_hidden_layers)])
207
+ self.h = nn.ModuleList([
208
+ GPTBigCodeBlock(config, linear_method)
209
+ for _ in range(config.num_hidden_layers)
210
+ ])
211
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
212
213
def forward(
@@ -235,12 +237,15 @@ class GPTBigCodeModel(nn.Module):
237
238
class GPTBigCodeForCausalLM(nn.Module):
239
238
- def __init__(self, config: GPTBigCodeConfig):
240
+ def __init__(
241
+ self,
242
+ config: GPTBigCodeConfig,
243
+ linear_method: Optional[LinearMethodBase] = None,
244
+ ):
245
super().__init__()
246
self.config = config
241
- self.transformer = GPTBigCodeModel(config)
242
- # TODO(zhuohan): create a new weight after implementing pipeline
243
- # parallelism
247
+ self.linear_method = linear_method
248
+ self.transformer = GPTBigCodeModel(config, linear_method)
249
self.lm_head_weight = self.transformer.wte.weight
250
self.sampler = Sampler(config.vocab_size)
251
@@ -258,89 +263,21 @@ class GPTBigCodeForCausalLM(nn.Module):
263
input_metadata)
264
return next_tokens
265
261
- _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
262
- _row_parallel_weights = ["c_proj.weight"]
263
-
266
def load_weights(self,
267
model_name_or_path: str,
268
cache_dir: Optional[str] = None,
269
load_format: str = "auto",
270
revision: Optional[str] = None):
269
- tensor_model_parallel_world_size = (
270
- get_tensor_model_parallel_world_size())
271
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
272
- state_dict = self.state_dict()
273
-
271
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
272
for name, loaded_weight in hf_model_weights_iterator(
273
model_name_or_path, cache_dir, load_format, revision):
274
if "lm_head.weight" in name:
277
- # GPT-2 ties the weights of the embedding layer and the final
278
- # linear layer.
275
continue
276
if ".attn.bias" in name:
277
# Skip attention mask.
278
# NOTE: "c_attn.bias" should not be skipped.
279
continue
284
-
285
- if not name.startswith("transformer."):
286
- name = "transformer." + name
287
-
288
- # For the fused QKV linear layer, manually shard the weights.
289
- if "c_attn" in name:
290
- # GPT-2's fused QKV has the shape of
291
- # [3 * num_heads * head_size, hidden_size].
292
- # When tensor parallelism is used, we shard the weights along
293
- # the head dimension.
294
- total_num_heads = self.config.num_attention_heads
295
- total_num_kv_heads = (1 if self.config.multi_query else
296
- total_num_heads)
297
- hidden_size = self.config.hidden_size
298
- head_size = hidden_size // total_num_heads
299
- total_kv_size = head_size * total_num_kv_heads
300
- num_heads = total_num_heads // tensor_model_parallel_world_size
301
- head_start = tensor_model_parallel_rank * num_heads
302
- head_end = (tensor_model_parallel_rank + 1) * num_heads
303
-
304
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
305
- wq, wk, wv = torch.split(
306
- loaded_weight, [hidden_size, total_kv_size, total_kv_size],
307
- dim=0)
308
-
309
- wq = wq[head_size * head_start:head_size * head_end]
310
- if not self.config.multi_query:
311
- # Split the heads when using normal multi-head attention
312
- wk = wk[head_size * head_start:head_size * head_end]
313
- wv = wv[head_size * head_start:head_size * head_end]
314
- loaded_weight = torch.cat([wq, wk, wv], dim=0)
315
- else:
316
- # For multi-query attention, we split the query
317
- # but replicate the key and value.
318
- loaded_weight_q = wq
319
- loaded_weight_kv = torch.cat([wk, wv], dim=0)
320
- q_weight_name = name.replace("c_attn", "c_attn_q")
321
- kv_weight_name = name.replace("c_attn", "c_attn_kv")
322
- load_tensor_parallel_weights(state_dict[q_weight_name],
323
- loaded_weight_q,
324
- q_weight_name,
325
- self._column_parallel_weights,
326
- self._row_parallel_weights,
327
- tensor_model_parallel_rank)
328
- load_tensor_parallel_weights(state_dict[kv_weight_name],
329
- loaded_weight_kv,
330
- kv_weight_name,
331
- self._column_parallel_weights,
332
- self._row_parallel_weights,
333
- tensor_model_parallel_rank)
334
- continue
335
-
336
- param = state_dict[name]
337
-
338
- if name == "transformer.wte.weight":
339
- load_padded_tensor_parallel_vocab(param, loaded_weight,
340
- tensor_model_parallel_rank)
341
- continue
342
-
343
- load_tensor_parallel_weights(param, loaded_weight, name,
344
- self._column_parallel_weights,
345
- self._row_parallel_weights,
346
- tensor_model_parallel_rank)
280
+ param = params_dict[name]
281
+ weight_loader = getattr(param, "weight_loader",
282
+ default_weight_loader)
283
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_j.py
+69
-53
@@ -29,14 +29,17 @@ from transformers import GPTJConfig
29
from vllm.model_executor.input_metadata import InputMetadata
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
+ LinearMethodBase,
34
+ QKVParallelLinear,
35
+ RowParallelLinear)
36
from vllm.model_executor.layers.sampler import Sampler
33
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34
- load_tensor_parallel_weights)
37
+from vllm.model_executor.layers.vocab_parallel_embedding import (
38
+ VocabParallelEmbedding, ParallelLMHead)
39
from vllm.model_executor.parallel_utils.parallel_state import (
36
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38
- ColumnParallelLinear,
39
- RowParallelLinear)
40
+ get_tensor_model_parallel_world_size)
41
+from vllm.model_executor.weight_utils import (default_weight_loader,
42
+ hf_model_weights_iterator)
43
from vllm.sequence import SamplerOutput
44
45
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -44,23 +47,28 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
47
48
class GPTJAttention(nn.Module):
49
47
- def __init__(self, config: GPTJConfig):
50
+ def __init__(
51
+ self,
52
+ config: GPTJConfig,
53
+ linear_method: Optional[LinearMethodBase] = None,
54
+ ):
55
super().__init__()
56
self.total_num_heads = config.num_attention_heads
57
self.hidden_size = config.hidden_size
58
self.head_size = self.hidden_size // self.total_num_heads
59
53
- self.qkv_proj = ColumnParallelLinear(
60
+ self.qkv_proj = QKVParallelLinear(
61
config.hidden_size,
55
- 3 * config.hidden_size,
62
+ self.head_size,
63
+ self.total_num_heads,
64
bias=False,
57
- gather_output=False,
65
+ linear_method=linear_method,
66
)
67
self.out_proj = RowParallelLinear(
68
config.hidden_size,
69
config.hidden_size,
70
bias=False,
63
- input_is_parallel=True,
71
+ linear_method=linear_method,
72
)
73
74
tp_world_size = get_tensor_model_parallel_world_size()
@@ -102,18 +110,23 @@ class GPTJAttention(nn.Module):
110
111
class GPTJMLP(nn.Module):
112
105
- def __init__(self, intermediate_size: int, config: GPTJConfig):
113
+ def __init__(
114
+ self,
115
+ intermediate_size: int,
116
+ config: GPTJConfig,
117
+ linear_method: Optional[LinearMethodBase] = None,
118
+ ):
119
super().__init__()
120
hidden_size = config.n_embd
121
self.fc_in = ColumnParallelLinear(
122
hidden_size,
123
intermediate_size,
111
- gather_output=False,
124
+ linear_method=linear_method,
125
)
126
self.fc_out = RowParallelLinear(
127
intermediate_size,
128
hidden_size,
116
- input_is_parallel=True,
129
+ linear_method=linear_method,
130
)
131
self.act = get_act_fn(config.activation_function)
132
@@ -126,15 +139,19 @@ class GPTJMLP(nn.Module):
139
140
class GPTJBlock(nn.Module):
141
129
- def __init__(self, config: GPTJConfig):
142
+ def __init__(
143
+ self,
144
+ config: GPTJConfig,
145
+ linear_method: Optional[LinearMethodBase] = None,
146
+ ):
147
super().__init__()
148
if config.n_inner is None:
149
inner_dim = 4 * config.n_embd
150
else:
151
inner_dim = config.n_inner
152
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
136
- self.attn = GPTJAttention(config)
137
- self.mlp = GPTJMLP(inner_dim, config)
153
+ self.attn = GPTJAttention(config, linear_method)
154
+ self.mlp = GPTJMLP(inner_dim, config, linear_method)
155
156
def forward(
157
self,
@@ -160,7 +177,11 @@ class GPTJBlock(nn.Module):
177
178
class GPTJModel(nn.Module):
179
163
- def __init__(self, config: GPTJConfig):
180
+ def __init__(
181
+ self,
182
+ config: GPTJConfig,
183
+ linear_method: Optional[LinearMethodBase] = None,
184
+ ):
185
super().__init__()
186
self.config = config
187
self.embed_dim = config.n_embd
@@ -169,7 +190,7 @@ class GPTJModel(nn.Module):
190
self.embed_dim,
191
)
192
self.h = nn.ModuleList(
172
- [GPTJBlock(config) for _ in range(config.n_layer)])
193
+ [GPTJBlock(config, linear_method) for _ in range(config.n_layer)])
194
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
195
196
def forward(
@@ -200,15 +221,20 @@ class GPTJModel(nn.Module):
221
222
class GPTJForCausalLM(nn.Module):
223
203
- def __init__(self, config: GPTJConfig):
224
+ def __init__(
225
+ self,
226
+ config: GPTJConfig,
227
+ linear_method: Optional[LinearMethodBase] = None,
228
+ ):
229
super().__init__()
230
self.config = config
231
+ self.linear_method = linear_method
232
assert not config.tie_word_embeddings
207
- self.transformer = GPTJModel(config)
208
- self.lm_head = ColumnParallelLinear(
209
- config.n_embd,
233
+ self.transformer = GPTJModel(config, linear_method)
234
+ self.lm_head = ParallelLMHead(
235
config.vocab_size,
211
- gather_output=False,
236
+ config.n_embd,
237
+ bias=True,
238
)
239
self.sampler = Sampler(config.vocab_size)
240
@@ -226,43 +252,33 @@ class GPTJForCausalLM(nn.Module):
252
input_metadata, self.lm_head.bias)
253
return next_tokens
254
229
- _column_parallel_weights = [
230
- "wte.weight", "fc_in.weight", "fc_in.bias", "lm_head.weight",
231
- "lm_head.bias"
232
- ]
233
- _row_parallel_weights = ["out_proj.weight", "fc_out.weight"]
234
-
255
def load_weights(self,
256
model_name_or_path: str,
257
cache_dir: Optional[str] = None,
258
load_format: str = "auto",
259
revision: Optional[str] = None):
240
- tp_rank = get_tensor_model_parallel_rank()
241
- state_dict = self.state_dict()
260
+ stacked_params_mapping = [
261
+ # (param_name, shard_name, shard_id)
262
+ ("qkv_proj", "q_proj", "q"),
263
+ ("qkv_proj", "k_proj", "k"),
264
+ ("qkv_proj", "v_proj", "v"),
265
+ ("gate_up_proj", "gate_proj", 0),
266
+ ("gate_up_proj", "up_proj", 1),
267
+ ]
268
+ params_dict = dict(self.named_parameters())
269
for name, loaded_weight in hf_model_weights_iterator(
270
model_name_or_path, cache_dir, load_format, revision):
271
if "attn.bias" in name or "attn.masked_bias" in name:
272
continue
246
-
247
- is_attention_weight = False
248
- for stride_id, att_weight_name in enumerate(
249
- ["q_proj", "k_proj", "v_proj"]):
250
- if att_weight_name not in name:
273
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
274
+ if weight_name not in name:
275
continue
252
- param = state_dict[name.replace(att_weight_name, "qkv_proj")]
253
- shard_size = param.shape[0] // 3
254
- loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
255
- (tp_rank + 1)]
256
- param_slice = param.data[shard_size * stride_id:shard_size *
257
- (stride_id + 1)]
258
- assert param_slice.shape == loaded_weight.shape
259
- param_slice.copy_(loaded_weight)
260
- is_attention_weight = True
276
+ param = params_dict[name.replace(weight_name, param_name)]
277
+ weight_loader = param.weight_loader
278
+ weight_loader(param, loaded_weight, shard_id)
279
break
262
- if is_attention_weight:
263
- continue
264
-
265
- param = state_dict[name]
266
- load_tensor_parallel_weights(param, loaded_weight, name,
267
- self._column_parallel_weights,
268
- self._row_parallel_weights, tp_rank)
280
+ else:
281
+ param = params_dict[name]
282
+ weight_loader = getattr(param, "weight_loader",
283
+ default_weight_loader)
284
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_neox.py
+70
-60
@@ -29,14 +29,17 @@ from transformers import GPTNeoXConfig
29
from vllm.model_executor.input_metadata import InputMetadata
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
+ LinearMethodBase,
34
+ QKVParallelLinear,
35
+ RowParallelLinear)
36
from vllm.model_executor.layers.sampler import Sampler
33
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34
- load_tensor_parallel_weights)
37
+from vllm.model_executor.layers.vocab_parallel_embedding import (
38
+ VocabParallelEmbedding, ParallelLMHead)
39
from vllm.model_executor.parallel_utils.parallel_state import (
36
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38
- ColumnParallelLinear,
39
- RowParallelLinear)
40
+ get_tensor_model_parallel_world_size)
41
+from vllm.model_executor.weight_utils import (default_weight_loader,
42
+ hf_model_weights_iterator)
43
from vllm.sequence import SamplerOutput
44
45
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -44,7 +47,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
47
48
class GPTNeoXAttention(nn.Module):
49
47
- def __init__(self, config: GPTNeoXConfig):
50
+ def __init__(
51
+ self,
52
+ config: GPTNeoXConfig,
53
+ linear_method: Optional[LinearMethodBase] = None,
54
+ ):
55
super().__init__()
56
self.total_num_heads = config.num_attention_heads
57
self.hidden_size = config.hidden_size
@@ -56,15 +63,16 @@ class GPTNeoXAttention(nn.Module):
63
self.num_heads = (self.total_num_heads //
64
tensor_model_parallel_world_size)
65
59
- self.query_key_value = ColumnParallelLinear(
66
+ self.query_key_value = QKVParallelLinear(
67
config.hidden_size,
61
- 3 * config.hidden_size,
62
- gather_output=False,
68
+ self.head_size,
69
+ self.total_num_heads,
70
+ linear_method=linear_method,
71
)
72
self.dense = RowParallelLinear(
73
config.hidden_size,
74
config.hidden_size,
67
- input_is_parallel=True,
75
+ linear_method=linear_method,
76
)
77
78
scaling = self.head_size**-0.5
@@ -100,17 +108,21 @@ class GPTNeoXAttention(nn.Module):
108
109
class GPTNeoXMLP(nn.Module):
110
103
- def __init__(self, config: GPTNeoXConfig):
111
+ def __init__(
112
+ self,
113
+ config: GPTNeoXConfig,
114
+ linear_method: Optional[LinearMethodBase] = None,
115
+ ):
116
super().__init__()
117
self.dense_h_to_4h = ColumnParallelLinear(
118
config.hidden_size,
119
config.intermediate_size,
108
- gather_output=False,
120
+ linear_method=linear_method,
121
)
122
self.dense_4h_to_h = RowParallelLinear(
123
config.intermediate_size,
124
config.hidden_size,
113
- input_is_parallel=True,
125
+ linear_method=linear_method,
126
)
127
self.act = get_act_fn(config.hidden_act)
128
@@ -123,15 +135,19 @@ class GPTNeoXMLP(nn.Module):
135
136
class GPTNeoXLayer(nn.Module):
137
126
- def __init__(self, config: GPTNeoXConfig):
138
+ def __init__(
139
+ self,
140
+ config: GPTNeoXConfig,
141
+ linear_method: Optional[LinearMethodBase] = None,
142
+ ):
143
super().__init__()
144
self.use_parallel_residual = config.use_parallel_residual
145
self.input_layernorm = nn.LayerNorm(config.hidden_size,
146
eps=config.layer_norm_eps)
147
self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
148
eps=config.layer_norm_eps)
133
- self.attention = GPTNeoXAttention(config)
134
- self.mlp = GPTNeoXMLP(config)
149
+ self.attention = GPTNeoXAttention(config, linear_method)
150
+ self.mlp = GPTNeoXMLP(config, linear_method)
151
152
def forward(
153
self,
@@ -169,7 +185,11 @@ class GPTNeoXLayer(nn.Module):
185
186
class GPTNeoXModel(nn.Module):
187
172
- def __init__(self, config: GPTNeoXConfig):
188
+ def __init__(
189
+ self,
190
+ config: GPTNeoXConfig,
191
+ linear_method: Optional[LinearMethodBase] = None,
192
+ ):
193
super().__init__()
194
self.config = config
195
@@ -177,8 +197,10 @@ class GPTNeoXModel(nn.Module):
197
config.vocab_size,
198
config.hidden_size,
199
)
180
- self.layers = nn.ModuleList(
181
- [GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
200
+ self.layers = nn.ModuleList([
201
+ GPTNeoXLayer(config, linear_method)
202
+ for _ in range(config.num_hidden_layers)
203
+ ])
204
self.final_layer_norm = nn.LayerNorm(config.hidden_size,
205
eps=config.layer_norm_eps)
206
@@ -210,15 +232,18 @@ class GPTNeoXModel(nn.Module):
232
233
class GPTNeoXForCausalLM(nn.Module):
234
213
- def __init__(self, config):
235
+ def __init__(
236
+ self,
237
+ config,
238
+ linear_method: Optional[LinearMethodBase] = None,
239
+ ):
240
super().__init__()
241
self.config = config
216
- self.gpt_neox = GPTNeoXModel(config)
217
- self.embed_out = ColumnParallelLinear(
218
- config.hidden_size,
242
+ self.linear_method = linear_method
243
+ self.gpt_neox = GPTNeoXModel(config, linear_method)
244
+ self.embed_out = ParallelLMHead(
245
config.vocab_size,
220
- bias=False,
221
- gather_output=False,
246
+ config.hidden_size,
247
)
248
self.sampler = Sampler(config.vocab_size)
249
@@ -236,50 +261,35 @@ class GPTNeoXForCausalLM(nn.Module):
261
input_metadata)
262
return next_tokens
263
239
- _column_parallel_weights = [
240
- "embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight",
241
- "dense_h_to_4h.bias"
242
- ]
243
- _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
244
-
264
def load_weights(self,
265
model_name_or_path: str,
266
cache_dir: Optional[str] = None,
267
load_format: str = "auto",
268
revision: Optional[str] = None):
250
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
251
- state_dict = self.state_dict()
269
+ params_dict = dict(self.named_parameters())
270
for name, loaded_weight in hf_model_weights_iterator(
271
model_name_or_path, cache_dir, load_format, revision):
272
if ("attention.bias" in name or "attention.masked_bias" in name
273
or "rotary_emb.inv_freq" in name):
274
continue
257
- param = state_dict[name]
275
+ param = params_dict[name]
276
+
277
if "query_key_value" in name:
259
- # NOTE(woosuk): GPT-NeoX's fused QKV has the shape of
260
- # [num_heads * 3 * head_size, hidden_size], while the
261
- # required shape is [3 * num_heads * head_size, hidden_size].
278
+ # NOTE: GPT-NeoX's fused QKV's output_dim has the shape of
279
+ # (num_heads * 3 * head_size), while the
280
+ # required shape is (3 * num_heads * head_size).
281
# Thus, we need weight conversion.
263
- shard_size = param.shape[0]
264
- loaded_weight = loaded_weight[
265
- shard_size * tensor_model_parallel_rank:shard_size *
266
- (tensor_model_parallel_rank + 1)]
267
-
282
+ output_dim = getattr(param, "output_dim", None)
283
num_heads = self.config.num_attention_heads
269
- hidden_size = self.config.hidden_size
270
- head_size = hidden_size // num_heads
271
- if "query_key_value.weight" in name:
272
- loaded_weight = loaded_weight.view(-1, 3, head_size,
273
- hidden_size)
274
- loaded_weight = loaded_weight.transpose(0, 1)
275
- loaded_weight = loaded_weight.reshape(-1, hidden_size)
276
- elif "query_key_value.bias" in name:
277
- loaded_weight = loaded_weight.view(-1, 3, head_size)
278
- loaded_weight = loaded_weight.transpose(0, 1)
279
- loaded_weight = loaded_weight.reshape(-1)
280
- else:
281
- raise ValueError(f"Unexpected weight name: {name}")
282
- load_tensor_parallel_weights(param, loaded_weight, name,
283
- self._column_parallel_weights,
284
- self._row_parallel_weights,
285
- tensor_model_parallel_rank)
284
+ if output_dim is not None:
285
+ loaded_weight_shape = loaded_weight.shape
286
+ loaded_weight = loaded_weight.view(
287
+ loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
288
+ loaded_weight_shape[output_dim + 1:])
289
+ loaded_weight = loaded_weight.transpose(
290
+ output_dim, output_dim + 1)
291
+ loaded_weight = loaded_weight.reshape(loaded_weight_shape)
292
+
293
+ weight_loader = getattr(param, "weight_loader",
294
+ default_weight_loader)
295
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/internlm.py
+62
-88
@@ -9,15 +9,17 @@ from vllm.model_executor.input_metadata import InputMetadata
9
from vllm.model_executor.layers.activation import SiluAndMul
10
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
+from vllm.model_executor.layers.linear import (LinearMethodBase,
13
+ MergedColumnParallelLinear,
14
+ QKVParallelLinear,
15
+ RowParallelLinear)
16
from vllm.model_executor.layers.sampler import Sampler
17
+from vllm.model_executor.layers.vocab_parallel_embedding import (
18
+ VocabParallelEmbedding, ParallelLMHead)
19
from vllm.model_executor.parallel_utils.parallel_state import (
14
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15
-from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
16
- RowParallelLinear,
17
- VocabParallelEmbedding)
18
-from vllm.model_executor.weight_utils import (
19
- hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
20
- load_tensor_parallel_weights)
20
+ get_tensor_model_parallel_world_size)
21
+from vllm.model_executor.weight_utils import (default_weight_loader,
22
+ hf_model_weights_iterator)
23
from vllm.sequence import SamplerOutput
24
25
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -30,20 +32,17 @@ class InternLMMLP(nn.Module):
32
hidden_size: int,
33
intermediate_size: int,
34
hidden_act: str,
35
+ linear_method: Optional[LinearMethodBase] = None,
36
):
37
super().__init__()
35
- self.gate_up_proj = ColumnParallelLinear(
36
- hidden_size,
37
- 2 * intermediate_size,
38
- bias=False,
39
- gather_output=False,
40
- )
41
- self.down_proj = RowParallelLinear(
42
- intermediate_size,
43
- hidden_size,
38
+ self.gate_up_proj = MergedColumnParallelLinear(
39
+ hidden_size, [intermediate_size] * 2,
40
bias=False,
45
- input_is_parallel=True,
46
- )
41
+ linear_method=linear_method)
42
+ self.down_proj = RowParallelLinear(intermediate_size,
43
+ hidden_size,
44
+ bias=False,
45
+ linear_method=linear_method)
46
if hidden_act != "silu":
47
raise ValueError(f"Unsupported activation: {hidden_act}. "
48
"Only silu is supported for now.")
@@ -65,6 +64,7 @@ class InternLMAttention(nn.Module):
64
bias: bool,
65
rope_theta: float = 10000,
66
max_position_embeddings: int = 8192,
67
+ linear_method: Optional[LinearMethodBase] = None,
68
):
69
super().__init__()
70
self.hidden_size = hidden_size
@@ -79,17 +79,18 @@ class InternLMAttention(nn.Module):
79
self.rope_theta = rope_theta
80
self.max_position_embeddings = max_position_embeddings
81
82
- self.qkv_proj = ColumnParallelLinear(
82
+ self.qkv_proj = QKVParallelLinear(
83
hidden_size,
84
- 3 * self.total_num_heads * self.head_dim,
84
+ self.head_dim,
85
+ self.total_num_heads,
86
bias=bias,
86
- gather_output=False,
87
+ linear_method=linear_method,
88
)
89
self.o_proj = RowParallelLinear(
90
self.total_num_heads * self.head_dim,
91
hidden_size,
92
bias=bias,
92
- input_is_parallel=True,
93
+ linear_method=linear_method,
94
)
95
self.attn = PagedAttentionWithRoPE(
96
self.num_heads,
@@ -118,7 +119,11 @@ class InternLMAttention(nn.Module):
119
120
class InternLMDecoderLayer(nn.Module):
121
121
- def __init__(self, config: LlamaConfig):
122
+ def __init__(
123
+ self,
124
+ config: LlamaConfig,
125
+ linear_method: Optional[LinearMethodBase] = None,
126
+ ):
127
super().__init__()
128
self.hidden_size = config.hidden_size
129
rope_theta = getattr(config, "rope_theta", 10000)
@@ -130,11 +135,13 @@ class InternLMDecoderLayer(nn.Module):
135
bias=config.bias,
136
rope_theta=rope_theta,
137
max_position_embeddings=max_position_embeddings,
138
+ linear_method=linear_method,
139
)
140
self.mlp = InternLMMLP(
141
hidden_size=self.hidden_size,
142
intermediate_size=config.intermediate_size,
143
hidden_act=config.hidden_act,
144
+ linear_method=linear_method,
145
)
146
self.input_layernorm = RMSNorm(config.hidden_size,
147
eps=config.rms_norm_eps)
@@ -171,7 +178,11 @@ class InternLMDecoderLayer(nn.Module):
178
179
class InternLMModel(nn.Module):
180
174
- def __init__(self, config: LlamaConfig):
181
+ def __init__(
182
+ self,
183
+ config: LlamaConfig,
184
+ linear_method: Optional[LinearMethodBase] = None,
185
+ ):
186
super().__init__()
187
self.config = config
188
self.padding_idx = config.pad_token_id
@@ -183,7 +194,7 @@ class InternLMModel(nn.Module):
194
config.hidden_size,
195
)
196
self.layers = nn.ModuleList([
186
- InternLMDecoderLayer(config)
197
+ InternLMDecoderLayer(config, linear_method)
198
for _ in range(config.num_hidden_layers)
199
])
200
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -216,17 +227,16 @@ class InternLMModel(nn.Module):
227
228
class InternLMForCausalLM(nn.Module):
229
219
- def __init__(self, config):
230
+ def __init__(
231
+ self,
232
+ config,
233
+ linear_method: Optional[LinearMethodBase] = None,
234
+ ):
235
super().__init__()
236
self.config = config
222
- self.model = InternLMModel(config)
223
- vocab_size = ((config.vocab_size + 63) // 64) * 64
224
- self.lm_head = ColumnParallelLinear(
225
- config.hidden_size,
226
- vocab_size,
227
- bias=False,
228
- gather_output=False,
229
- )
237
+ self.linear_method = linear_method
238
+ self.model = InternLMModel(config, linear_method)
239
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
240
self.sampler = Sampler(config.vocab_size)
241
242
def forward(
@@ -243,69 +253,33 @@ class InternLMForCausalLM(nn.Module):
253
input_metadata)
254
return next_tokens
255
246
- _column_parallel_weights = [
247
- "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
248
- ]
249
- _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
250
-
256
def load_weights(self,
257
model_name_or_path: str,
258
cache_dir: Optional[str] = None,
259
load_format: str = "auto",
260
revision: Optional[str] = None):
256
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
257
- state_dict = self.state_dict()
258
-
261
+ stacked_params_mapping = [
262
+ # (param_name, shard_name, shard_id)
263
+ ("qkv_proj", "q_proj", "q"),
264
+ ("qkv_proj", "k_proj", "k"),
265
+ ("qkv_proj", "v_proj", "v"),
266
+ ("gate_up_proj", "gate_proj", 0),
267
+ ("gate_up_proj", "up_proj", 1),
268
+ ]
269
+ params_dict = dict(self.named_parameters())
270
for name, loaded_weight in hf_model_weights_iterator(
271
model_name_or_path, cache_dir, load_format, revision):
272
if "rotary_emb.inv_freq" in name:
273
continue
263
-
264
- if "embed_tokens" in name or "lm_head" in name:
265
- param = state_dict[name]
266
- load_padded_tensor_parallel_vocab(param, loaded_weight,
267
- tensor_model_parallel_rank)
268
- continue
269
-
270
- is_attention_weight = False
271
- for stride_id, att_weight_name in enumerate(
272
- ["q_proj", "k_proj", "v_proj"]):
273
- if att_weight_name not in name:
274
- continue
275
- param = state_dict[name.replace(att_weight_name, "qkv_proj")]
276
- shard_size = param.shape[0] // 3
277
- loaded_weight = loaded_weight[
278
- shard_size * tensor_model_parallel_rank:shard_size *
279
- (tensor_model_parallel_rank + 1)]
280
- param_slice = param.data[shard_size * stride_id:shard_size *
281
- (stride_id + 1)]
282
- assert param_slice.shape == loaded_weight.shape
283
- param_slice.copy_(loaded_weight)
284
- is_attention_weight = True
285
- break
286
- if is_attention_weight:
287
- continue
288
-
289
- is_gate_up_weight = False
290
- for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
274
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
275
if weight_name not in name:
276
continue
293
- param = state_dict[name.replace(weight_name, "gate_up_proj")]
294
- shard_size = param.shape[0] // 2
295
- loaded_weight = loaded_weight[
296
- shard_size * tensor_model_parallel_rank:shard_size *
297
- (tensor_model_parallel_rank + 1)]
298
- param_slice = param.data[shard_size * stride_id:shard_size *
299
- (stride_id + 1)]
300
- assert param_slice.shape == loaded_weight.shape
301
- param_slice.copy_(loaded_weight)
302
- is_gate_up_weight = True
277
+ param = params_dict[name.replace(weight_name, param_name)]
278
+ weight_loader = param.weight_loader
279
+ weight_loader(param, loaded_weight, shard_id)
280
break
304
- if is_gate_up_weight:
305
- continue
306
-
307
- param = state_dict[name]
308
- load_tensor_parallel_weights(param, loaded_weight, name,
309
- self._column_parallel_weights,
310
- self._row_parallel_weights,
311
- tensor_model_parallel_rank)
281
+ else:
282
+ param = params_dict[name]
283
+ weight_loader = getattr(param, "weight_loader",
284
+ default_weight_loader)
285
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/llama.py
+53
-156
@@ -33,17 +33,19 @@ from transformers import LlamaConfig
33
34
from vllm.model_executor.input_metadata import InputMetadata
35
from vllm.model_executor.layers.activation import SiluAndMul
36
-from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
37
+from vllm.model_executor.layers.layernorm import RMSNorm
38
+from vllm.model_executor.layers.linear import (LinearMethodBase,
39
+ MergedColumnParallelLinear,
40
+ QKVParallelLinear,
41
+ RowParallelLinear)
42
from vllm.model_executor.layers.sampler import Sampler
39
-from vllm.model_executor.layers.quantized_linear import ParallelLinear
43
+from vllm.model_executor.layers.vocab_parallel_embedding import (
44
+ VocabParallelEmbedding, ParallelLMHead)
45
from vllm.model_executor.parallel_utils.parallel_state import (
41
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43
-from vllm.model_executor.quantization_utils import QuantizationConfig
44
-from vllm.model_executor.weight_utils import (
45
- convert_pyslice_to_tensor, hf_model_weights_iterator,
46
- load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
46
+ get_tensor_model_parallel_world_size)
47
+from vllm.model_executor.weight_utils import (default_weight_loader,
48
+ hf_model_weights_iterator)
49
from vllm.sequence import SamplerOutput
50
51
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,19 +58,17 @@ class LlamaMLP(nn.Module):
58
hidden_size: int,
59
intermediate_size: int,
60
hidden_act: str,
59
- quant_config: Optional[QuantizationConfig] = None,
61
+ linear_method: Optional[LinearMethodBase] = None,
62
) -> None:
63
super().__init__()
62
- self.gate_up_proj = ParallelLinear.column(hidden_size,
63
- 2 * intermediate_size,
64
- bias=False,
65
- gather_output=False,
66
- quant_config=quant_config)
67
- self.down_proj = ParallelLinear.row(intermediate_size,
68
- hidden_size,
69
- bias=False,
70
- input_is_parallel=True,
71
- quant_config=quant_config)
64
+ self.gate_up_proj = MergedColumnParallelLinear(
65
+ hidden_size, [intermediate_size] * 2,
66
+ bias=False,
67
+ linear_method=linear_method)
68
+ self.down_proj = RowParallelLinear(intermediate_size,
69
+ hidden_size,
70
+ bias=False,
71
+ linear_method=linear_method)
72
if hidden_act != "silu":
73
raise ValueError(f"Unsupported activation: {hidden_act}. "
74
"Only silu is supported for now.")
@@ -91,7 +91,7 @@ class LlamaAttention(nn.Module):
91
rope_theta: float = 10000,
92
rope_scaling: Optional[Dict[str, Any]] = None,
93
max_position_embeddings: int = 8192,
94
- quant_config: Optional[QuantizationConfig] = None,
94
+ linear_method: Optional[LinearMethodBase] = None,
95
) -> None:
96
super().__init__()
97
self.hidden_size = hidden_size
@@ -109,7 +109,6 @@ class LlamaAttention(nn.Module):
109
# the KV heads across multiple tensor parallel GPUs.
110
assert tp_size % self.total_num_kv_heads == 0
111
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112
- num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
112
self.head_dim = hidden_size // self.total_num_heads
113
self.q_size = self.num_heads * self.head_dim
114
self.kv_size = self.num_kv_heads * self.head_dim
@@ -117,21 +116,19 @@ class LlamaAttention(nn.Module):
116
self.rope_theta = rope_theta
117
self.max_position_embeddings = max_position_embeddings
118
120
- self.qkv_proj = ParallelLinear.column(
119
+ self.qkv_proj = QKVParallelLinear(
120
hidden_size,
122
- (self.total_num_heads +
123
- 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
121
self.head_dim,
122
+ self.total_num_heads,
123
+ self.total_num_kv_heads,
124
bias=False,
126
- gather_output=False,
127
- quant_config=quant_config,
125
+ linear_method=linear_method,
126
)
129
- self.o_proj = ParallelLinear.row(
127
+ self.o_proj = RowParallelLinear(
128
self.total_num_heads * self.head_dim,
129
hidden_size,
130
bias=False,
133
- input_is_parallel=True,
134
- quant_config=quant_config,
131
+ linear_method=linear_method,
132
)
133
self.attn = PagedAttentionWithRoPE(
134
self.num_heads,
@@ -165,11 +162,10 @@ class LlamaDecoderLayer(nn.Module):
162
def __init__(
163
self,
164
config: LlamaConfig,
168
- quant_config: Optional[QuantizationConfig] = None,
165
+ linear_method: Optional[LinearMethodBase] = None,
166
) -> None:
167
super().__init__()
168
self.hidden_size = config.hidden_size
172
- # Requires transformers > 4.32.0
169
rope_theta = getattr(config, "rope_theta", 10000)
170
rope_scaling = getattr(config, "rope_scaling", None)
171
max_position_embeddings = getattr(config, "max_position_embeddings",
@@ -181,13 +177,13 @@ class LlamaDecoderLayer(nn.Module):
177
rope_theta=rope_theta,
178
rope_scaling=rope_scaling,
179
max_position_embeddings=max_position_embeddings,
184
- quant_config=quant_config,
180
+ linear_method=linear_method,
181
)
182
self.mlp = LlamaMLP(
183
hidden_size=self.hidden_size,
184
intermediate_size=config.intermediate_size,
185
hidden_act=config.hidden_act,
190
- quant_config=quant_config,
186
+ linear_method=linear_method,
187
)
188
self.input_layernorm = RMSNorm(config.hidden_size,
189
eps=config.rms_norm_eps)
@@ -227,20 +223,18 @@ class LlamaModel(nn.Module):
223
def __init__(
224
self,
225
config: LlamaConfig,
230
- quant_config: Optional[QuantizationConfig] = None,
226
+ linear_method: Optional[LinearMethodBase] = None,
227
) -> None:
228
super().__init__()
229
self.config = config
230
self.padding_idx = config.pad_token_id
231
self.vocab_size = config.vocab_size
236
-
237
- vocab_size = ((config.vocab_size + 63) // 64) * 64
232
self.embed_tokens = VocabParallelEmbedding(
239
- vocab_size,
233
+ config.vocab_size,
234
config.hidden_size,
235
)
236
self.layers = nn.ModuleList([
243
- LlamaDecoderLayer(config, quant_config)
237
+ LlamaDecoderLayer(config, linear_method)
238
for _ in range(config.num_hidden_layers)
239
])
240
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -276,19 +270,13 @@ class LlamaForCausalLM(nn.Module):
270
def __init__(
271
self,
272
config: LlamaConfig,
279
- quant_config: Optional[QuantizationConfig] = None,
273
+ linear_method: Optional[LinearMethodBase] = None,
274
) -> None:
275
super().__init__()
276
self.config = config
283
- self.quant_config = quant_config
284
- self.model = LlamaModel(config, quant_config)
285
- vocab_size = ((config.vocab_size + 63) // 64) * 64
286
- # NOTE: The LM head is not quantized.
287
- self.lm_head = ParallelLinear.column(config.hidden_size,
288
- vocab_size,
289
- bias=False,
290
- gather_output=False,
291
- quant_config=None)
277
+ self.linear_method = linear_method
278
+ self.model = LlamaModel(config, linear_method)
279
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
280
self.sampler = Sampler(config.vocab_size)
281
282
def forward(
@@ -305,124 +293,33 @@ class LlamaForCausalLM(nn.Module):
293
input_metadata)
294
return next_tokens
295
308
- _column_parallel_layers = []
309
- _row_parallel_layers = ["o_proj", "down_proj"]
310
-
296
def load_weights(self,
297
model_name_or_path: str,
298
cache_dir: Optional[str] = None,
299
load_format: str = "auto",
300
revision: Optional[str] = None):
316
- if self.quant_config is None:
317
- col_weight_suffixes = ["weight"]
318
- row_weight_suffixes = ["weight"]
319
- else:
320
- col_weight_suffixes = (
321
- self.quant_config.get_col_parallel_tensor_names())
322
- row_weight_suffixes = (
323
- self.quant_config.get_row_parallel_tensor_names())
324
-
325
- column_parallel_weights: List[str] = []
326
- for layer in self._column_parallel_layers:
327
- for suffix in col_weight_suffixes:
328
- column_parallel_weights.append(f"{layer}.{suffix}")
329
- row_parallel_weights: List[str] = []
330
- for layer in self._row_parallel_layers:
331
- for suffix in row_weight_suffixes:
332
- row_parallel_weights.append(f"{layer}.{suffix}")
333
-
334
- tp_size = get_tensor_model_parallel_world_size()
335
- tp_rank = get_tensor_model_parallel_rank()
336
- q_proj_shard_size = (self.config.hidden_size // tp_size)
337
- num_kv_heads_replicas = max(1,
338
- tp_size // self.config.num_key_value_heads)
339
- num_kv_heads_per_gpu = max(1,
340
- self.config.num_key_value_heads // tp_size)
341
- kv_proj_shard_size = (self.config.hidden_size //
342
- self.config.num_attention_heads *
343
- num_kv_heads_per_gpu)
344
- attention_weight_specs = [
345
- # (weight_name, shard_size, offset)
346
- ("q_proj", q_proj_shard_size, 0),
347
- ("k_proj", kv_proj_shard_size, q_proj_shard_size),
348
- ("v_proj", kv_proj_shard_size,
349
- q_proj_shard_size + kv_proj_shard_size),
301
+ stacked_params_mapping = [
302
+ # (param_name, shard_name, shard_id)
303
+ ("qkv_proj", "q_proj", "q"),
304
+ ("qkv_proj", "k_proj", "k"),
305
+ ("qkv_proj", "v_proj", "v"),
306
+ ("gate_up_proj", "gate_proj", 0),
307
+ ("gate_up_proj", "up_proj", 1),
308
]
351
- state_dict = self.state_dict()
352
-
309
+ params_dict = dict(self.named_parameters())
310
for name, loaded_weight in hf_model_weights_iterator(
311
model_name_or_path, cache_dir, load_format, revision):
312
if "rotary_emb.inv_freq" in name:
313
continue
357
-
358
- packed_dim = None
359
- is_transposed = False
360
- if self.quant_config is not None:
361
- packed_dim = self.quant_config.get_packed_dim(name)
362
- is_transposed = self.quant_config.is_transposed(name)
363
- if is_transposed:
364
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
365
- loaded_weight = loaded_weight.T
366
-
367
- is_attention_weight = False
368
- for weight_name, shard_size, offset in attention_weight_specs:
314
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
315
if weight_name not in name:
316
continue
371
- param = state_dict[name.replace(weight_name, "qkv_proj")]
372
- if is_transposed:
373
- param = param.T
374
-
375
- if packed_dim is not None:
376
- shard_dim = 0 if not is_transposed else 1
377
- if packed_dim == shard_dim:
378
- shard_size //= self.quant_config.pack_factor
379
- offset //= self.quant_config.pack_factor
380
-
381
- if weight_name in ["k_proj", "v_proj"]:
382
- shard_id = tp_rank // num_kv_heads_replicas
383
- else:
384
- shard_id = tp_rank
385
- loaded_weight = loaded_weight[shard_size *
386
- shard_id:shard_size *
387
- (shard_id + 1)]
388
- param_slice = param.data[offset:offset + shard_size]
389
- assert param_slice.shape == loaded_weight.shape
390
-
391
- param_slice.copy_(loaded_weight)
392
- is_attention_weight = True
393
- break
394
- if is_attention_weight:
395
- continue
396
-
397
- is_gate_up_weight = False
398
- for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
399
- if weight_name not in name:
400
- continue
401
- param = state_dict[name.replace(weight_name, "gate_up_proj")]
402
- if is_transposed:
403
- param = param.T
404
-
405
- shard_size = param.shape[0] // 2
406
- loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
407
- (tp_rank + 1)]
408
- param_slice = param.data[shard_size * stride_id:shard_size *
409
- (stride_id + 1)]
410
- assert param_slice.shape == loaded_weight.shape
411
- param_slice.copy_(loaded_weight)
412
- is_gate_up_weight = True
317
+ param = params_dict[name.replace(weight_name, param_name)]
318
+ weight_loader = param.weight_loader
319
+ weight_loader(param, loaded_weight, shard_id)
320
break
414
- if is_gate_up_weight:
415
- continue
416
-
417
- param = state_dict[name]
418
- if is_transposed:
419
- param = param.T
420
-
421
- if "embed_tokens" in name or "lm_head" in name:
422
- load_padded_tensor_parallel_vocab(param, loaded_weight,
423
- tp_rank)
424
- continue
425
-
426
- load_tensor_parallel_weights(param, loaded_weight, name,
427
- column_parallel_weights,
428
- row_parallel_weights, tp_rank)
321
+ else:
322
+ param = params_dict[name]
323
+ weight_loader = getattr(param, "weight_loader",
324
+ default_weight_loader)
325
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/mistral.py
+62
-148
@@ -33,17 +33,19 @@ from transformers import MistralConfig
33
34
from vllm.model_executor.input_metadata import InputMetadata
35
from vllm.model_executor.layers.activation import SiluAndMul
36
-from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
37
+from vllm.model_executor.layers.layernorm import RMSNorm
38
+from vllm.model_executor.layers.linear import (LinearMethodBase,
39
+ MergedColumnParallelLinear,
40
+ QKVParallelLinear,
41
+ RowParallelLinear)
42
from vllm.model_executor.layers.sampler import Sampler
39
-from vllm.model_executor.layers.quantized_linear import ParallelLinear
43
+from vllm.model_executor.layers.vocab_parallel_embedding import (
44
+ VocabParallelEmbedding, ParallelLMHead)
45
from vllm.model_executor.parallel_utils.parallel_state import (
41
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43
-from vllm.model_executor.quantization_utils import QuantizationConfig
44
-from vllm.model_executor.weight_utils import (
45
- convert_pyslice_to_tensor, hf_model_weights_iterator,
46
- load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
46
+ get_tensor_model_parallel_world_size)
47
+from vllm.model_executor.weight_utils import (default_weight_loader,
48
+ hf_model_weights_iterator)
49
from vllm.sequence import SamplerOutput
50
51
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,19 +58,17 @@ class MistralMLP(nn.Module):
58
hidden_size: int,
59
intermediate_size: int,
60
hidden_act: str,
59
- quant_config: Optional[QuantizationConfig] = None,
61
+ linear_method: Optional[LinearMethodBase] = None,
62
) -> None:
63
super().__init__()
62
- self.gate_up_proj = ParallelLinear.column(hidden_size,
63
- 2 * intermediate_size,
64
- bias=False,
65
- gather_output=False,
66
- quant_config=quant_config)
67
- self.down_proj = ParallelLinear.row(intermediate_size,
68
- hidden_size,
69
- bias=False,
70
- input_is_parallel=True,
71
- quant_config=quant_config)
64
+ self.gate_up_proj = MergedColumnParallelLinear(
65
+ hidden_size, [intermediate_size] * 2,
66
+ bias=False,
67
+ linear_method=linear_method)
68
+ self.down_proj = RowParallelLinear(intermediate_size,
69
+ hidden_size,
70
+ bias=False,
71
+ linear_method=linear_method)
72
if hidden_act != "silu":
73
raise ValueError(f"Unsupported activation: {hidden_act}. "
74
"Only silu is supported for now.")
@@ -89,7 +89,7 @@ class MistralAttention(nn.Module):
89
num_kv_heads: int,
90
max_position: int = 4096 * 32,
91
rope_theta: float = 10000,
92
- quant_config: Optional[QuantizationConfig] = None,
92
+ linear_method: Optional[LinearMethodBase] = None,
93
sliding_window: Optional[int] = None) -> None:
94
super().__init__()
95
self.hidden_size = hidden_size
@@ -98,8 +98,15 @@ class MistralAttention(nn.Module):
98
assert self.total_num_heads % tp_size == 0
99
self.num_heads = self.total_num_heads // tp_size
100
self.total_num_kv_heads = num_kv_heads
101
- assert self.total_num_kv_heads % tp_size == 0
102
- self.num_kv_heads = self.total_num_kv_heads // tp_size
101
+ if self.total_num_kv_heads >= tp_size:
102
+ # Number of KV heads is greater than TP size, so we partition
103
+ # the KV heads across multiple tensor parallel GPUs.
104
+ assert self.total_num_kv_heads % tp_size == 0
105
+ else:
106
+ # Number of KV heads is less than TP size, so we replicate
107
+ # the KV heads across multiple tensor parallel GPUs.
108
+ assert tp_size % self.total_num_kv_heads == 0
109
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110
self.head_dim = hidden_size // self.total_num_heads
111
self.q_size = self.num_heads * self.head_dim
112
self.kv_size = self.num_kv_heads * self.head_dim
@@ -107,20 +114,19 @@ class MistralAttention(nn.Module):
114
self.rope_theta = rope_theta
115
self.sliding_window = sliding_window
116
110
- self.qkv_proj = ParallelLinear.column(
117
+ self.qkv_proj = QKVParallelLinear(
118
hidden_size,
112
- (self.total_num_heads + 2 * self.total_num_kv_heads) *
119
self.head_dim,
120
+ self.total_num_heads,
121
+ self.total_num_kv_heads,
122
bias=False,
115
- gather_output=False,
116
- quant_config=quant_config,
123
+ linear_method=linear_method,
124
)
118
- self.o_proj = ParallelLinear.row(
125
+ self.o_proj = RowParallelLinear(
126
self.total_num_heads * self.head_dim,
127
hidden_size,
128
bias=False,
122
- input_is_parallel=True,
123
- quant_config=quant_config,
129
+ linear_method=linear_method,
130
)
131
self.attn = PagedAttentionWithRoPE(self.num_heads,
132
self.head_dim,
@@ -153,7 +159,7 @@ class MistralDecoderLayer(nn.Module):
159
def __init__(
160
self,
161
config: MistralConfig,
156
- quant_config: Optional[QuantizationConfig] = None,
162
+ linear_method: Optional[LinearMethodBase] = None,
163
) -> None:
164
super().__init__()
165
self.hidden_size = config.hidden_size
@@ -165,13 +171,13 @@ class MistralDecoderLayer(nn.Module):
171
max_position=config.max_position_embeddings,
172
num_kv_heads=config.num_key_value_heads,
173
rope_theta=rope_theta,
168
- quant_config=quant_config,
174
+ linear_method=linear_method,
175
sliding_window=config.sliding_window)
176
self.mlp = MistralMLP(
177
hidden_size=self.hidden_size,
178
intermediate_size=config.intermediate_size,
179
hidden_act=config.hidden_act,
174
- quant_config=quant_config,
180
+ linear_method=linear_method,
181
)
182
self.input_layernorm = RMSNorm(config.hidden_size,
183
eps=config.rms_norm_eps)
@@ -211,20 +217,19 @@ class MistralModel(nn.Module):
217
def __init__(
218
self,
219
config: MistralConfig,
214
- quant_config: Optional[QuantizationConfig] = None,
220
+ linear_method: Optional[LinearMethodBase] = None,
221
) -> None:
222
super().__init__()
223
self.config = config
224
self.padding_idx = config.pad_token_id
225
self.vocab_size = config.vocab_size
226
221
- vocab_size = ((config.vocab_size + 63) // 64) * 64
227
self.embed_tokens = VocabParallelEmbedding(
223
- vocab_size,
228
+ config.vocab_size,
229
config.hidden_size,
230
)
231
self.layers = nn.ModuleList([
227
- MistralDecoderLayer(config, quant_config)
232
+ MistralDecoderLayer(config, linear_method)
233
for _ in range(config.num_hidden_layers)
234
])
235
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -260,19 +265,13 @@ class MistralForCausalLM(nn.Module):
265
def __init__(
266
self,
267
config: MistralConfig,
263
- quant_config: Optional[QuantizationConfig] = None,
268
+ linear_method: Optional[LinearMethodBase] = None,
269
) -> None:
270
super().__init__()
271
self.config = config
267
- self.quant_config = quant_config
268
- self.model = MistralModel(config, quant_config)
269
- vocab_size = ((config.vocab_size + 63) // 64) * 64
270
- # NOTE: The LM head is not quantized.
271
- self.lm_head = ParallelLinear.column(config.hidden_size,
272
- vocab_size,
273
- bias=False,
274
- gather_output=False,
275
- quant_config=None)
272
+ self.linear_method = linear_method
273
+ self.model = MistralModel(config, linear_method)
274
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
275
self.sampler = Sampler(config.vocab_size)
276
277
def forward(
@@ -289,118 +288,33 @@ class MistralForCausalLM(nn.Module):
288
input_metadata)
289
return next_tokens
290
292
- _column_parallel_layers = []
293
- _row_parallel_layers = ["o_proj", "down_proj"]
294
-
291
def load_weights(self,
292
model_name_or_path: str,
293
cache_dir: Optional[str] = None,
294
load_format: str = "auto",
295
revision: Optional[str] = None):
300
- if self.quant_config is None:
301
- col_weight_suffixes = ["weight"]
302
- row_weight_suffixes = ["weight"]
303
- else:
304
- col_weight_suffixes = (
305
- self.quant_config.get_col_parallel_tensor_names())
306
- row_weight_suffixes = (
307
- self.quant_config.get_row_parallel_tensor_names())
308
-
309
- column_parallel_weights: List[str] = []
310
- for layer in self._column_parallel_layers:
311
- for suffix in col_weight_suffixes:
312
- column_parallel_weights.append(f"{layer}.{suffix}")
313
- row_parallel_weights: List[str] = []
314
- for layer in self._row_parallel_layers:
315
- for suffix in row_weight_suffixes:
316
- row_parallel_weights.append(f"{layer}.{suffix}")
317
-
318
- tp_size = get_tensor_model_parallel_world_size()
319
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
320
- q_proj_shard_size = (self.config.hidden_size // tp_size)
321
- kv_proj_shard_size = (self.config.hidden_size //
322
- self.config.num_attention_heads *
323
- self.config.num_key_value_heads // tp_size)
324
- attention_weight_specs = [
325
- # (weight_name, shard_size, offset)
326
- ("q_proj", q_proj_shard_size, 0),
327
- ("k_proj", kv_proj_shard_size, q_proj_shard_size),
328
- ("v_proj", kv_proj_shard_size,
329
- q_proj_shard_size + kv_proj_shard_size),
296
+ stacked_params_mapping = [
297
+ # (param_name, shard_name, shard_id)
298
+ ("qkv_proj", "q_proj", "q"),
299
+ ("qkv_proj", "k_proj", "k"),
300
+ ("qkv_proj", "v_proj", "v"),
301
+ ("gate_up_proj", "gate_proj", 0),
302
+ ("gate_up_proj", "up_proj", 1),
303
]
331
- state_dict = self.state_dict()
332
-
304
+ params_dict = dict(self.named_parameters())
305
for name, loaded_weight in hf_model_weights_iterator(
306
model_name_or_path, cache_dir, load_format, revision):
307
if "rotary_emb.inv_freq" in name:
308
continue
337
-
338
- packed_dim = None
339
- is_transposed = False
340
- if self.quant_config is not None:
341
- packed_dim = self.quant_config.get_packed_dim(name)
342
- is_transposed = self.quant_config.is_transposed(name)
343
- if is_transposed:
344
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
345
- loaded_weight = loaded_weight.T
346
-
347
- is_attention_weight = False
348
- for weight_name, shard_size, offset in attention_weight_specs:
309
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
310
if weight_name not in name:
311
continue
351
- param = state_dict[name.replace(weight_name, "qkv_proj")]
352
- if is_transposed:
353
- param = param.T
354
-
355
- if packed_dim is not None:
356
- shard_dim = 0 if not is_transposed else 1
357
- if packed_dim == shard_dim:
358
- shard_size //= self.quant_config.pack_factor
359
- offset //= self.quant_config.pack_factor
360
-
361
- loaded_weight = loaded_weight[
362
- shard_size * tensor_model_parallel_rank:shard_size *
363
- (tensor_model_parallel_rank + 1)]
364
- param_slice = param.data[offset:offset + shard_size]
365
- assert param_slice.shape == loaded_weight.shape
366
-
367
- param_slice.copy_(loaded_weight)
368
- is_attention_weight = True
369
- break
370
- if is_attention_weight:
371
- continue
372
-
373
- is_gate_up_weight = False
374
- for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
375
- if weight_name not in name:
376
- continue
377
- param = state_dict[name.replace(weight_name, "gate_up_proj")]
378
- if is_transposed:
379
- param = param.T
380
-
381
- shard_size = param.shape[0] // 2
382
- loaded_weight = loaded_weight[
383
- shard_size * tensor_model_parallel_rank:shard_size *
384
- (tensor_model_parallel_rank + 1)]
385
- param_slice = param.data[shard_size * stride_id:shard_size *
386
- (stride_id + 1)]
387
- assert param_slice.shape == loaded_weight.shape
388
- param_slice.copy_(loaded_weight)
389
- is_gate_up_weight = True
312
+ param = params_dict[name.replace(weight_name, param_name)]
313
+ weight_loader = param.weight_loader
314
+ weight_loader(param, loaded_weight, shard_id)
315
break
391
- if is_gate_up_weight:
392
- continue
393
-
394
- param = state_dict[name]
395
- if is_transposed:
396
- param = param.T
397
-
398
- if "embed_tokens" in name or "lm_head" in name:
399
- load_padded_tensor_parallel_vocab(param, loaded_weight,
400
- tensor_model_parallel_rank)
401
- continue
402
-
403
- load_tensor_parallel_weights(param, loaded_weight, name,
404
- column_parallel_weights,
405
- row_parallel_weights,
406
- tensor_model_parallel_rank)
316
+ else:
317
+ param = params_dict[name]
318
+ weight_loader = getattr(param, "weight_loader",
319
+ default_weight_loader)
320
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/mpt.py
+52
-59
@@ -10,15 +10,17 @@ from transformers import MptConfig
10
from vllm.model_executor.input_metadata import InputMetadata
11
from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
13
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
+ LinearMethodBase,
15
+ QKVParallelLinear,
16
+ RowParallelLinear)
17
from vllm.model_executor.layers.sampler import Sampler
14
-from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
15
- hf_model_weights_iterator,
16
- load_tensor_parallel_weights)
18
+from vllm.model_executor.layers.vocab_parallel_embedding import (
19
+ VocabParallelEmbedding)
20
from vllm.model_executor.parallel_utils.parallel_state import (
21
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
19
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
20
- ColumnParallelLinear,
21
- RowParallelLinear)
22
+from vllm.model_executor.weight_utils import (default_weight_loader,
23
+ hf_model_weights_iterator)
24
from vllm.sequence import SamplerOutput
25
26
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -39,7 +41,11 @@ def _get_alibi_slopes(
41
42
class MptAttention(nn.Module):
43
42
- def __init__(self, config: MptConfig):
44
+ def __init__(
45
+ self,
46
+ config: MptConfig,
47
+ linear_method: Optional[LinearMethodBase] = None,
48
+ ):
49
super().__init__()
50
self.d_model = config.d_model
51
self.total_num_heads = config.n_heads
@@ -49,11 +55,13 @@ class MptAttention(nn.Module):
55
assert not config.attn_config.prefix_lm
56
assert config.attn_config.alibi
57
52
- self.qkv_proj = ColumnParallelLinear(
58
+ # pylint: disable=invalid-name
59
+ self.Wqkv = QKVParallelLinear(
60
self.d_model,
54
- 3 * self.d_model,
61
+ self.d_model // self.total_num_heads,
62
+ self.total_num_heads,
63
bias=not config.no_bias,
56
- gather_output=False,
64
+ linear_method=linear_method,
65
)
66
if self.qk_ln:
67
self.q_ln = nn.LayerNorm(self.d_model)
@@ -62,7 +70,7 @@ class MptAttention(nn.Module):
70
self.d_model,
71
self.d_model,
72
bias=not config.no_bias,
65
- input_is_parallel=True,
73
+ linear_method=linear_method,
74
)
75
76
tp_world_size = get_tensor_model_parallel_world_size()
@@ -91,7 +99,7 @@ class MptAttention(nn.Module):
99
cache_event: Optional[torch.cuda.Event],
100
) -> torch.Tensor:
101
del position_ids # unused.
94
- qkv, _ = self.qkv_proj(hidden_states)
102
+ qkv, _ = self.Wqkv(hidden_states)
103
if self.clip_qkv is not None:
104
qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
105
q, k, v = qkv.chunk(chunks=3, dim=-1)
@@ -107,7 +115,11 @@ class MptAttention(nn.Module):
115
116
class MptMLP(nn.Module):
117
110
- def __init__(self, config: MptConfig):
118
+ def __init__(
119
+ self,
120
+ config: MptConfig,
121
+ linear_method: Optional[LinearMethodBase] = None,
122
+ ):
123
super().__init__()
124
hidden_size = config.d_model
125
expansion_ratio = config.expansion_ratio
@@ -116,14 +128,14 @@ class MptMLP(nn.Module):
128
hidden_size,
129
intermediate_size,
130
bias=not config.no_bias,
119
- gather_output=False,
131
+ linear_method=linear_method,
132
)
133
self.act = get_act_fn("gelu")
134
self.down_proj = RowParallelLinear(
135
intermediate_size,
136
hidden_size,
137
bias=not config.no_bias,
126
- input_is_parallel=True,
138
+ linear_method=linear_method,
139
)
140
141
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -135,13 +147,17 @@ class MptMLP(nn.Module):
147
148
class MptBlock(nn.Module):
149
138
- def __init__(self, config: MptConfig):
150
+ def __init__(
151
+ self,
152
+ config: MptConfig,
153
+ linear_method: Optional[LinearMethodBase] = None,
154
+ ):
155
super().__init__()
156
hidden_size = config.d_model
157
self.norm_1 = nn.LayerNorm(hidden_size)
142
- self.attn = MptAttention(config)
158
+ self.attn = MptAttention(config, linear_method)
159
self.norm_2 = nn.LayerNorm(hidden_size)
144
- self.ffn = MptMLP(config)
160
+ self.ffn = MptMLP(config, linear_method)
161
162
def forward(
163
self,
@@ -168,7 +184,11 @@ class MptBlock(nn.Module):
184
185
class MptModel(nn.Module):
186
171
- def __init__(self, config: MptConfig):
187
+ def __init__(
188
+ self,
189
+ config: MptConfig,
190
+ linear_method: Optional[LinearMethodBase] = None,
191
+ ):
192
super().__init__()
193
assert config.embedding_fraction == 1.0
194
assert config.norm_type == "low_precision_layernorm"
@@ -178,7 +198,7 @@ class MptModel(nn.Module):
198
config.d_model,
199
)
200
self.blocks = nn.ModuleList(
181
- [MptBlock(config) for _ in range(config.n_layers)])
201
+ [MptBlock(config, linear_method) for _ in range(config.n_layers)])
202
self.norm_f = nn.LayerNorm(config.d_model)
203
if config.no_bias:
204
for module in self.modules():
@@ -215,14 +235,17 @@ class MptModel(nn.Module):
235
236
class MptForCausalLM(nn.Module):
237
218
- def __init__(self, config: MptConfig):
238
+ def __init__(
239
+ self,
240
+ config: MptConfig,
241
+ linear_method: Optional[LinearMethodBase] = None,
242
+ ):
243
super().__init__()
244
self.config = config
245
assert config.tie_word_embeddings
246
+ self.linear_method = linear_method
247
223
- self.transformer = MptModel(config)
224
- # TODO(zhuohan): create a new weight after implementing pipeline
225
- # parallelism
248
+ self.transformer = MptModel(config, linear_method)
249
self.lm_head_weight = self.transformer.wte.weight
250
self.sampler = Sampler(config.vocab_size)
251
@@ -240,45 +263,15 @@ class MptForCausalLM(nn.Module):
263
input_metadata)
264
return next_tokens
265
243
- _column_parallel_weights = ["wte.weight", "up_proj.weight", "up_proj.bias"]
244
- _row_parallel_weights = ["out_proj.weight", "down_proj.weight"]
245
-
266
def load_weights(self,
267
model_name_or_path: str,
268
cache_dir: Optional[str] = None,
269
load_format: str = "auto",
270
revision: Optional[str] = None):
251
- tp_world_size = get_tensor_model_parallel_world_size()
252
- tp_rank = get_tensor_model_parallel_rank()
253
- state_dict = self.state_dict()
271
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
272
for name, loaded_weight in hf_model_weights_iterator(
273
model_name_or_path, cache_dir, load_format, revision):
256
- if "Wqkv" in name:
257
- # NOTE(woosuk): MPT's fused QKV has the shape of
258
- # [3 * num_heads * head_size, hidden_size].
259
- # When tensor model parallelism is used, we need to shard
260
- # the weight along the hidden dimension.
261
- total_num_heads = self.config.num_attention_heads
262
- hidden_size = self.config.hidden_size
263
- head_size = hidden_size // total_num_heads
264
- num_heads = total_num_heads // tp_world_size
265
- head_start = tp_rank * num_heads
266
- head_end = (tp_rank + 1) * num_heads
267
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
268
- if name.endswith(".weight"):
269
- loaded_weight = loaded_weight.view(3, total_num_heads,
270
- head_size, hidden_size)
271
- loaded_weight = loaded_weight[:, head_start:head_end, :, :]
272
- loaded_weight = loaded_weight.reshape(-1, hidden_size)
273
- elif name.endswith(".bias"):
274
- loaded_weight = loaded_weight.view(3, total_num_heads,
275
- head_size)
276
- loaded_weight = loaded_weight[:, head_start:head_end, :]
277
- loaded_weight = loaded_weight.reshape(-1)
278
- else:
279
- raise ValueError(f"Unexpected parameter name {name}")
280
- name = name.replace("Wqkv", "qkv_proj")
281
- param = state_dict[name]
282
- load_tensor_parallel_weights(param, loaded_weight, name,
283
- self._column_parallel_weights,
284
- self._row_parallel_weights, tp_rank)
274
+ param = params_dict[name]
275
+ weight_loader = getattr(param, "weight_loader",
276
+ default_weight_loader)
277
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/opt.py
+71
-62
@@ -30,14 +30,18 @@ from transformers import OPTConfig
30
from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import get_act_fn
32
from vllm.model_executor.layers.attention import PagedAttention
33
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
+ LinearMethodBase,
35
+ QKVParallelLinear,
36
+ ReplicatedLinear,
37
+ RowParallelLinear)
38
from vllm.model_executor.layers.sampler import Sampler
34
-from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35
- load_tensor_parallel_weights)
39
+from vllm.model_executor.layers.vocab_parallel_embedding import (
40
+ VocabParallelEmbedding)
41
from vllm.model_executor.parallel_utils.parallel_state import (
37
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
-from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39
- ColumnParallelLinear,
40
- RowParallelLinear)
42
+ get_tensor_model_parallel_world_size)
43
+from vllm.model_executor.weight_utils import (default_weight_loader,
44
+ hf_model_weights_iterator)
45
from vllm.sequence import SamplerOutput
46
47
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -63,6 +67,7 @@ class OPTAttention(nn.Module):
67
embed_dim: int,
68
num_heads: int,
69
bias: bool = True,
70
+ linear_method: Optional[LinearMethodBase] = None,
71
) -> None:
72
super().__init__()
73
self.embed_dim = embed_dim
@@ -74,17 +79,18 @@ class OPTAttention(nn.Module):
79
self.head_dim = embed_dim // total_num_heads
80
self.scaling = self.head_dim**-0.5
81
77
- self.qkv_proj = ColumnParallelLinear(
82
+ self.qkv_proj = QKVParallelLinear(
83
embed_dim,
79
- 3 * embed_dim,
84
+ self.head_dim,
85
+ total_num_heads,
86
bias=bias,
81
- gather_output=False,
87
+ linear_method=linear_method,
88
)
89
self.out_proj = RowParallelLinear(
90
embed_dim,
91
embed_dim,
92
bias=bias,
87
- input_is_parallel=True,
93
+ linear_method=linear_method,
94
)
95
self.attn = PagedAttention(self.num_heads,
96
self.head_dim,
@@ -108,7 +114,11 @@ class OPTAttention(nn.Module):
114
115
class OPTDecoderLayer(nn.Module):
116
111
- def __init__(self, config: OPTConfig):
117
+ def __init__(
118
+ self,
119
+ config: OPTConfig,
120
+ linear_method: Optional[LinearMethodBase] = None,
121
+ ):
122
super().__init__()
123
self.config = config
124
self.embed_dim = config.hidden_size
@@ -116,6 +126,7 @@ class OPTDecoderLayer(nn.Module):
126
embed_dim=self.embed_dim,
127
num_heads=config.num_attention_heads,
128
bias=config.enable_bias,
129
+ linear_method=linear_method,
130
)
131
self.do_layer_norm_before = config.do_layer_norm_before
132
self.activation_fn = get_act_fn(config.activation_function)
@@ -127,13 +138,13 @@ class OPTDecoderLayer(nn.Module):
138
self.embed_dim,
139
config.ffn_dim,
140
bias=config.enable_bias,
130
- gather_output=False,
141
+ linear_method=linear_method,
142
)
143
self.fc2 = RowParallelLinear(
144
config.ffn_dim,
145
self.embed_dim,
146
bias=config.enable_bias,
136
- input_is_parallel=True,
147
+ linear_method=linear_method,
148
)
149
self.final_layer_norm = nn.LayerNorm(
150
self.embed_dim,
@@ -177,7 +188,11 @@ class OPTDecoderLayer(nn.Module):
188
189
class OPTDecoder(nn.Module):
190
180
- def __init__(self, config: OPTConfig):
191
+ def __init__(
192
+ self,
193
+ config: OPTConfig,
194
+ linear_method: Optional[LinearMethodBase] = None,
195
+ ):
196
super().__init__()
197
self.config = config
198
self.padding_idx = config.pad_token_id
@@ -194,16 +209,18 @@ class OPTDecoder(nn.Module):
209
210
# Project out & in will be replicated if they exist.
211
if config.word_embed_proj_dim != config.hidden_size:
197
- self.project_out = nn.Linear(config.hidden_size,
198
- config.word_embed_proj_dim,
199
- bias=False)
212
+ self.project_out = ReplicatedLinear(config.hidden_size,
213
+ config.word_embed_proj_dim,
214
+ bias=False,
215
+ linear_method=linear_method)
216
else:
217
self.project_out = None
218
219
if config.word_embed_proj_dim != config.hidden_size:
204
- self.project_in = nn.Linear(config.word_embed_proj_dim,
205
- config.hidden_size,
206
- bias=False)
220
+ self.project_in = ReplicatedLinear(config.word_embed_proj_dim,
221
+ config.hidden_size,
222
+ bias=False,
223
+ linear_method=linear_method)
224
else:
225
self.project_in = None
226
@@ -218,8 +235,10 @@ class OPTDecoder(nn.Module):
235
else:
236
self.final_layer_norm = None
237
221
- self.layers = nn.ModuleList(
222
- [OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
238
+ self.layers = nn.ModuleList([
239
+ OPTDecoderLayer(config, linear_method)
240
+ for _ in range(config.num_hidden_layers)
241
+ ])
242
243
def forward(
244
self,
@@ -253,9 +272,13 @@ class OPTDecoder(nn.Module):
272
273
class OPTModel(nn.Module):
274
256
- def __init__(self, config: OPTConfig):
275
+ def __init__(
276
+ self,
277
+ config: OPTConfig,
278
+ linear_method: Optional[LinearMethodBase] = None,
279
+ ):
280
super().__init__()
258
- self.decoder = OPTDecoder(config)
281
+ self.decoder = OPTDecoder(config, linear_method)
282
283
def forward(
284
self,
@@ -271,12 +294,15 @@ class OPTModel(nn.Module):
294
295
class OPTForCausalLM(nn.Module):
296
274
- def __init__(self, config):
297
+ def __init__(
298
+ self,
299
+ config,
300
+ linear_method: Optional[LinearMethodBase] = None,
301
+ ):
302
super().__init__()
303
self.config = config
277
- self.model = OPTModel(config)
278
- # TODO(zhuohan): create a new weight after implementing pipeline
279
- # parallelism
304
+ self.linear_method = linear_method
305
+ self.model = OPTModel(config, linear_method)
306
self.lm_head_weight = self.model.decoder.embed_tokens.weight
307
self.sampler = Sampler(config.vocab_size)
308
@@ -294,48 +320,31 @@ class OPTForCausalLM(nn.Module):
320
input_metadata)
321
return next_tokens
322
297
- _column_parallel_weights = [
298
- "embed_tokens.weight", "fc1.weight", "fc1.bias"
299
- ]
300
- _row_parallel_weights = ["out_proj.weight", "fc2.weight"]
301
-
323
def load_weights(self,
324
model_name_or_path: str,
325
cache_dir: Optional[str] = None,
326
load_format: str = "auto",
327
revision: Optional[str] = None):
307
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
308
- state_dict = self.state_dict()
309
-
328
+ stacked_params_mapping = [
329
+ # (param_name, shard_name, shard_id)
330
+ ("qkv_proj", "q_proj", "q"),
331
+ ("qkv_proj", "k_proj", "k"),
332
+ ("qkv_proj", "v_proj", "v"),
333
+ ]
334
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
335
for name, loaded_weight in hf_model_weights_iterator(
336
model_name_or_path, cache_dir, load_format, revision):
337
if "lm_head.weight" in name:
338
continue
314
-
315
- if name.startswith("decoder."):
316
- name = "model." + name
317
-
318
- is_attention_weight = False
319
- for stride_id, att_weight_name in enumerate(
320
- ["q_proj", "k_proj", "v_proj"]):
321
- if att_weight_name not in name:
339
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
340
+ if weight_name not in name:
341
continue
323
- param = state_dict[name.replace(att_weight_name, "qkv_proj")]
324
- shard_size = param.shape[0] // 3
325
- loaded_weight = loaded_weight[
326
- shard_size * tensor_model_parallel_rank:shard_size *
327
- (tensor_model_parallel_rank + 1)]
328
- param_slice = param.data[shard_size * stride_id:shard_size *
329
- (stride_id + 1)]
330
- assert param_slice.shape == loaded_weight.shape
331
- param_slice.copy_(loaded_weight)
332
- is_attention_weight = True
342
+ param = params_dict[name.replace(weight_name, param_name)]
343
+ weight_loader = param.weight_loader
344
+ weight_loader(param, loaded_weight, shard_id)
345
break
334
- if is_attention_weight:
335
- continue
336
-
337
- param = state_dict[name]
338
- load_tensor_parallel_weights(param, loaded_weight, name,
339
- self._column_parallel_weights,
340
- self._row_parallel_weights,
341
- tensor_model_parallel_rank)
346
+ else:
347
+ param = params_dict[name]
348
+ weight_loader = getattr(param, "weight_loader",
349
+ default_weight_loader)
350
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen.py
+80
-119
@@ -15,24 +15,19 @@ from torch import nn
15
16
from vllm.model_executor.input_metadata import InputMetadata
17
from vllm.model_executor.layers.activation import SiluAndMul
18
-from vllm.model_executor.layers.layernorm import RMSNorm
18
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
19
+from vllm.model_executor.layers.layernorm import RMSNorm
20
+from vllm.model_executor.layers.linear import (LinearMethodBase,
21
+ MergedColumnParallelLinear,
22
+ QKVParallelLinear,
23
+ RowParallelLinear)
24
from vllm.model_executor.layers.sampler import Sampler
21
-from vllm.model_executor.weight_utils import (
22
- convert_pyslice_to_tensor,
23
- hf_model_weights_iterator,
24
- load_padded_tensor_parallel_vocab,
25
- load_tensor_parallel_weights,
26
-)
25
+from vllm.model_executor.layers.vocab_parallel_embedding import (
26
+ VocabParallelEmbedding, ParallelLMHead)
27
from vllm.model_executor.parallel_utils.parallel_state import (
28
- get_tensor_model_parallel_rank,
29
- get_tensor_model_parallel_world_size,
30
-)
31
-from vllm.model_executor.parallel_utils.layers import (
32
- VocabParallelEmbedding,
33
- ColumnParallelLinear,
34
- RowParallelLinear,
35
-)
28
+ get_tensor_model_parallel_world_size)
29
+from vllm.model_executor.weight_utils import (default_weight_loader,
30
+ hf_model_weights_iterator)
31
from vllm.sequence import SamplerOutput
32
from vllm.transformers_utils.configs.qwen import QWenConfig
33
@@ -46,20 +41,17 @@ class QWenMLP(nn.Module):
41
hidden_size: int,
42
intermediate_size: int,
43
hidden_act: str = "silu",
44
+ linear_method: Optional[LinearMethodBase] = None,
45
):
46
super().__init__()
51
- self.gate_up_proj = ColumnParallelLinear(
52
- hidden_size,
53
- 2 * intermediate_size,
54
- bias=False,
55
- gather_output=False,
56
- )
57
- self.c_proj = RowParallelLinear(
58
- intermediate_size,
59
- hidden_size,
47
+ self.gate_up_proj = MergedColumnParallelLinear(
48
+ hidden_size, [intermediate_size] * 2,
49
bias=False,
61
- input_is_parallel=True,
62
- )
50
+ linear_method=linear_method)
51
+ self.c_proj = RowParallelLinear(intermediate_size,
52
+ hidden_size,
53
+ bias=False,
54
+ linear_method=linear_method)
55
if hidden_act != "silu":
56
raise ValueError(f"Unsupported activation: {hidden_act}. "
57
"Only silu is supported for now.")
@@ -74,12 +66,15 @@ class QWenMLP(nn.Module):
66
67
class QWenAttention(nn.Module):
68
77
- def __init__(self,
78
- hidden_size: int,
79
- num_heads: int,
80
- max_position_embeddings: int,
81
- rope_theta: float = 10000,
82
- rope_scaling: Optional[Dict[str, Any]] = None):
69
+ def __init__(
70
+ self,
71
+ hidden_size: int,
72
+ num_heads: int,
73
+ max_position_embeddings: int,
74
+ rope_theta: float = 10000,
75
+ rope_scaling: Optional[Dict[str, Any]] = None,
76
+ linear_method: Optional[LinearMethodBase] = None,
77
+ ):
78
super().__init__()
79
self.hidden_size = hidden_size
80
tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
@@ -90,18 +85,18 @@ class QWenAttention(nn.Module):
85
tensor_model_parallel_world_size)
86
self.head_dim = hidden_size // self.total_num_heads
87
93
- # pylint: disable=invalid-name
94
- self.c_attn = ColumnParallelLinear(
88
+ self.c_attn = QKVParallelLinear(
89
hidden_size,
96
- 3 * hidden_size,
90
+ self.head_dim,
91
+ self.total_num_heads,
92
bias=True,
98
- gather_output=False,
93
+ linear_method=linear_method,
94
)
95
self.c_proj = RowParallelLinear(
96
self.total_num_heads * self.head_dim,
97
hidden_size,
98
bias=False,
104
- input_is_parallel=True,
99
+ linear_method=linear_method,
100
)
101
self.scaling = self.head_dim**-0.5
102
self.attn = PagedAttentionWithRoPE(
@@ -134,7 +129,11 @@ class QWenAttention(nn.Module):
129
130
class QWenBlock(nn.Module):
131
137
- def __init__(self, config: QWenConfig):
132
+ def __init__(
133
+ self,
134
+ config: QWenConfig,
135
+ linear_method: Optional[LinearMethodBase] = None,
136
+ ):
137
super().__init__()
138
self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
139
@@ -144,11 +143,14 @@ class QWenBlock(nn.Module):
143
config.num_attention_heads,
144
config.max_position_embeddings,
145
rope_theta=rope_theta,
147
- rope_scaling=rope_scaling)
146
+ rope_scaling=rope_scaling,
147
+ linear_method=linear_method)
148
149
self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
150
151
- self.mlp = QWenMLP(config.hidden_size, config.intermediate_size // 2)
151
+ self.mlp = QWenMLP(config.hidden_size,
152
+ config.intermediate_size // 2,
153
+ linear_method=linear_method)
154
155
def forward(
156
self,
@@ -180,18 +182,23 @@ class QWenBlock(nn.Module):
182
183
class QWenModel(nn.Module):
184
183
- def __init__(self, config: QWenConfig):
185
+ def __init__(
186
+ self,
187
+ config: QWenConfig,
188
+ linear_method: Optional[LinearMethodBase] = None,
189
+ ):
190
super().__init__()
191
self.config = config
192
self.vocab_size = config.vocab_size
193
188
- vocab_size = ((config.vocab_size + 63) // 64) * 64
194
self.wte = VocabParallelEmbedding(
190
- vocab_size,
195
+ config.vocab_size,
196
config.hidden_size,
197
)
193
- self.h = nn.ModuleList(
194
- [QWenBlock(config) for _ in range(config.num_hidden_layers)])
198
+ self.h = nn.ModuleList([
199
+ QWenBlock(config, linear_method)
200
+ for _ in range(config.num_hidden_layers)
201
+ ])
202
self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
203
204
def forward(
@@ -222,17 +229,16 @@ class QWenModel(nn.Module):
229
230
class QWenLMHeadModel(nn.Module):
231
225
- def __init__(self, config: QWenConfig):
232
+ def __init__(
233
+ self,
234
+ config: QWenConfig,
235
+ linear_method: Optional[LinearMethodBase] = None,
236
+ ):
237
super().__init__()
238
self.config = config
228
- self.transformer = QWenModel(config)
229
- vocab_size = ((config.vocab_size + 63) // 64) * 64
230
- self.lm_head = ColumnParallelLinear(
231
- config.hidden_size,
232
- vocab_size,
233
- bias=False,
234
- gather_output=False,
235
- )
239
+ self.linear_method = linear_method
240
+ self.transformer = QWenModel(config, linear_method)
241
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
242
self.sampler = Sampler(config.vocab_size)
243
244
def forward(
@@ -249,75 +255,30 @@ class QWenLMHeadModel(nn.Module):
255
input_metadata)
256
return next_tokens
257
252
- _column_parallel_weights = []
253
- _row_parallel_weights = ["c_proj.weight"]
254
-
255
- def load_weights(
256
- self,
257
- model_name_or_path: str,
258
- cache_dir: Optional[str] = None,
259
- load_format: str = "auto",
260
- revision: Optional[str] = None,
261
- ):
262
- tp_world_size = get_tensor_model_parallel_world_size()
263
- tp_rank = get_tensor_model_parallel_rank()
264
- state_dict = self.state_dict()
265
-
258
+ def load_weights(self,
259
+ model_name_or_path: str,
260
+ cache_dir: Optional[str] = None,
261
+ load_format: str = "auto",
262
+ revision: Optional[str] = None):
263
+ stacked_params_mapping = [
264
+ # (param_name, shard_name, shard_id)
265
+ ("gate_up_proj", "w2", 0),
266
+ ("gate_up_proj", "w1", 1),
267
+ ]
268
+ params_dict = dict(self.named_parameters())
269
for name, loaded_weight in hf_model_weights_iterator(
270
model_name_or_path, cache_dir, load_format, revision):
271
if "rotary_emb.inv_freq" in name:
272
continue
270
-
271
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
272
-
273
- if "c_attn" in name:
274
- total_num_heads = self.config.num_attention_heads
275
- hidden_size = self.config.hidden_size
276
- head_size = hidden_size // total_num_heads
277
- num_heads = total_num_heads // tp_world_size
278
- head_start = tp_rank * num_heads
279
- head_end = (tp_rank + 1) * num_heads
280
-
281
- if "weight" in name:
282
- loaded_weight = loaded_weight.view(3, total_num_heads,
283
- head_size, hidden_size)
284
- loaded_weight = loaded_weight[:, head_start:head_end, :, :]
285
- loaded_weight = loaded_weight.reshape(-1, hidden_size)
286
- elif "bias" in name:
287
- loaded_weight = loaded_weight.view(3, total_num_heads,
288
- head_size)
289
- loaded_weight = loaded_weight[:, head_start:head_end, :]
290
- loaded_weight = loaded_weight.reshape(-1)
291
-
292
- is_gate_up_weight = False
293
- for stride_id, weight_name in enumerate(["w2", "w1"]):
273
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
274
if weight_name not in name:
275
continue
296
- param = state_dict[name.replace(weight_name, "gate_up_proj")]
297
- shard_size = param.shape[0] // 2
298
- loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
299
- (tp_rank + 1)]
300
- param_slice = param.data[shard_size * stride_id:shard_size *
301
- (stride_id + 1)]
302
- assert param_slice.shape == loaded_weight.shape
303
- param_slice.copy_(loaded_weight)
304
- is_gate_up_weight = True
276
+ param = params_dict[name.replace(weight_name, param_name)]
277
+ weight_loader = param.weight_loader
278
+ weight_loader(param, loaded_weight, shard_id)
279
break
306
- if is_gate_up_weight:
307
- continue
308
-
309
- param = state_dict[name]
310
-
311
- if "wte" in name or "lm_head" in name:
312
- load_padded_tensor_parallel_vocab(param, loaded_weight,
313
- tp_rank)
314
- continue
315
-
316
- load_tensor_parallel_weights(
317
- param,
318
- loaded_weight,
319
- name,
320
- self._column_parallel_weights,
321
- self._row_parallel_weights,
322
- tp_rank,
323
- )
280
+ else:
281
+ param = params_dict[name]
282
+ weight_loader = getattr(param, "weight_loader",
283
+ default_weight_loader)
284
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/yi.py
+53
-156
@@ -33,17 +33,19 @@ from vllm.transformers_utils.configs.yi import YiConfig
33
34
from vllm.model_executor.input_metadata import InputMetadata
35
from vllm.model_executor.layers.activation import SiluAndMul
36
-from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
37
+from vllm.model_executor.layers.layernorm import RMSNorm
38
+from vllm.model_executor.layers.linear import (LinearMethodBase,
39
+ MergedColumnParallelLinear,
40
+ QKVParallelLinear,
41
+ RowParallelLinear)
42
from vllm.model_executor.layers.sampler import Sampler
39
-from vllm.model_executor.layers.quantized_linear import ParallelLinear
43
+from vllm.model_executor.layers.vocab_parallel_embedding import (
44
+ VocabParallelEmbedding, ParallelLMHead)
45
from vllm.model_executor.parallel_utils.parallel_state import (
41
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43
-from vllm.model_executor.quantization_utils import QuantizationConfig
44
-from vllm.model_executor.weight_utils import (
45
- convert_pyslice_to_tensor, hf_model_weights_iterator,
46
- load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
46
+ get_tensor_model_parallel_world_size)
47
+from vllm.model_executor.weight_utils import (default_weight_loader,
48
+ hf_model_weights_iterator)
49
from vllm.sequence import SamplerOutput
50
51
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,19 +58,17 @@ class YiMLP(nn.Module):
58
hidden_size: int,
59
intermediate_size: int,
60
hidden_act: str,
59
- quant_config: Optional[QuantizationConfig] = None,
61
+ linear_method: Optional[LinearMethodBase] = None,
62
) -> None:
63
super().__init__()
62
- self.gate_up_proj = ParallelLinear.column(hidden_size,
63
- 2 * intermediate_size,
64
- bias=False,
65
- gather_output=False,
66
- quant_config=quant_config)
67
- self.down_proj = ParallelLinear.row(intermediate_size,
68
- hidden_size,
69
- bias=False,
70
- input_is_parallel=True,
71
- quant_config=quant_config)
64
+ self.gate_up_proj = MergedColumnParallelLinear(
65
+ hidden_size, [intermediate_size] * 2,
66
+ bias=False,
67
+ linear_method=linear_method)
68
+ self.down_proj = RowParallelLinear(intermediate_size,
69
+ hidden_size,
70
+ bias=False,
71
+ linear_method=linear_method)
72
if hidden_act != "silu":
73
raise ValueError(f"Unsupported activation: {hidden_act}. "
74
"Only silu is supported for now.")
@@ -91,7 +91,7 @@ class YiAttention(nn.Module):
91
rope_theta: float = 10000,
92
rope_scaling: Optional[Dict[str, Any]] = None,
93
max_position_embeddings: int = 8192,
94
- quant_config: Optional[QuantizationConfig] = None,
94
+ linear_method: Optional[LinearMethodBase] = None,
95
) -> None:
96
super().__init__()
97
self.hidden_size = hidden_size
@@ -109,7 +109,6 @@ class YiAttention(nn.Module):
109
# the KV heads across multiple tensor parallel GPUs.
110
assert tp_size % self.total_num_kv_heads == 0
111
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112
- num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
112
self.head_dim = hidden_size // self.total_num_heads
113
self.q_size = self.num_heads * self.head_dim
114
self.kv_size = self.num_kv_heads * self.head_dim
@@ -117,21 +116,19 @@ class YiAttention(nn.Module):
116
self.rope_theta = rope_theta
117
self.max_position_embeddings = max_position_embeddings
118
120
- self.qkv_proj = ParallelLinear.column(
119
+ self.qkv_proj = QKVParallelLinear(
120
hidden_size,
122
- (self.total_num_heads +
123
- 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
121
self.head_dim,
122
+ self.total_num_heads,
123
+ self.total_num_kv_heads,
124
bias=False,
126
- gather_output=False,
127
- quant_config=quant_config,
125
+ linear_method=linear_method,
126
)
129
- self.o_proj = ParallelLinear.row(
127
+ self.o_proj = RowParallelLinear(
128
self.total_num_heads * self.head_dim,
129
hidden_size,
130
bias=False,
133
- input_is_parallel=True,
134
- quant_config=quant_config,
131
+ linear_method=linear_method,
132
)
133
self.attn = PagedAttentionWithRoPE(
134
self.num_heads,
@@ -165,11 +162,10 @@ class YiDecoderLayer(nn.Module):
162
def __init__(
163
self,
164
config: YiConfig,
168
- quant_config: Optional[QuantizationConfig] = None,
165
+ linear_method: Optional[LinearMethodBase] = None,
166
) -> None:
167
super().__init__()
168
self.hidden_size = config.hidden_size
172
- # Requires transformers > 4.32.0
169
rope_theta = getattr(config, "rope_theta", 10000)
170
rope_scaling = getattr(config, "rope_scaling", None)
171
max_position_embeddings = getattr(config, "max_position_embeddings",
@@ -181,13 +177,13 @@ class YiDecoderLayer(nn.Module):
177
rope_theta=rope_theta,
178
rope_scaling=rope_scaling,
179
max_position_embeddings=max_position_embeddings,
184
- quant_config=quant_config,
180
+ linear_method=linear_method,
181
)
182
self.mlp = YiMLP(
183
hidden_size=self.hidden_size,
184
intermediate_size=config.intermediate_size,
185
hidden_act=config.hidden_act,
190
- quant_config=quant_config,
186
+ linear_method=linear_method,
187
)
188
self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
189
self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -225,20 +221,18 @@ class YiModel(nn.Module):
221
def __init__(
222
self,
223
config: YiConfig,
228
- quant_config: Optional[QuantizationConfig] = None,
224
+ linear_method: Optional[LinearMethodBase] = None,
225
) -> None:
226
super().__init__()
227
self.config = config
228
self.padding_idx = config.pad_token_id
229
self.vocab_size = config.vocab_size
234
-
235
- vocab_size = ((config.vocab_size + 63) // 64) * 64
230
self.embed_tokens = VocabParallelEmbedding(
237
- vocab_size,
231
+ config.vocab_size,
232
config.hidden_size,
233
)
234
self.layers = nn.ModuleList([
241
- YiDecoderLayer(config, quant_config)
235
+ YiDecoderLayer(config, linear_method)
236
for _ in range(config.num_hidden_layers)
237
])
238
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -274,19 +268,13 @@ class YiForCausalLM(nn.Module):
268
def __init__(
269
self,
270
config: YiConfig,
277
- quant_config: Optional[QuantizationConfig] = None,
271
+ linear_method: Optional[LinearMethodBase] = None,
272
) -> None:
273
super().__init__()
274
self.config = config
281
- self.quant_config = quant_config
282
- self.model = YiModel(config, quant_config)
283
- vocab_size = ((config.vocab_size + 63) // 64) * 64
284
- # NOTE: The LM head is not quantized.
285
- self.lm_head = ParallelLinear.column(config.hidden_size,
286
- vocab_size,
287
- bias=False,
288
- gather_output=False,
289
- quant_config=None)
275
+ self.linear_method = linear_method
276
+ self.model = YiModel(config, linear_method)
277
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
278
self.sampler = Sampler(config.vocab_size)
279
280
def forward(
@@ -303,124 +291,33 @@ class YiForCausalLM(nn.Module):
291
input_metadata)
292
return next_tokens
293
306
- _column_parallel_layers = []
307
- _row_parallel_layers = ["o_proj", "down_proj"]
308
-
294
def load_weights(self,
295
model_name_or_path: str,
296
cache_dir: Optional[str] = None,
297
load_format: str = "auto",
298
revision: Optional[str] = None):
314
- if self.quant_config is None:
315
- col_weight_suffixes = ["weight"]
316
- row_weight_suffixes = ["weight"]
317
- else:
318
- col_weight_suffixes = (
319
- self.quant_config.get_col_parallel_tensor_names())
320
- row_weight_suffixes = (
321
- self.quant_config.get_row_parallel_tensor_names())
322
-
323
- column_parallel_weights: List[str] = []
324
- for layer in self._column_parallel_layers:
325
- for suffix in col_weight_suffixes:
326
- column_parallel_weights.append(f"{layer}.{suffix}")
327
- row_parallel_weights: List[str] = []
328
- for layer in self._row_parallel_layers:
329
- for suffix in row_weight_suffixes:
330
- row_parallel_weights.append(f"{layer}.{suffix}")
331
-
332
- tp_size = get_tensor_model_parallel_world_size()
333
- tp_rank = get_tensor_model_parallel_rank()
334
- q_proj_shard_size = (self.config.hidden_size // tp_size)
335
- num_kv_heads_replicas = max(1,
336
- tp_size // self.config.num_key_value_heads)
337
- num_kv_heads_per_gpu = max(1,
338
- self.config.num_key_value_heads // tp_size)
339
- kv_proj_shard_size = (self.config.hidden_size //
340
- self.config.num_attention_heads *
341
- num_kv_heads_per_gpu)
342
- attention_weight_specs = [
343
- # (weight_name, shard_size, offset)
344
- ("q_proj", q_proj_shard_size, 0),
345
- ("k_proj", kv_proj_shard_size, q_proj_shard_size),
346
- ("v_proj", kv_proj_shard_size,
347
- q_proj_shard_size + kv_proj_shard_size),
299
+ stacked_params_mapping = [
300
+ # (param_name, shard_name, shard_id)
301
+ ("qkv_proj", "q_proj", "q"),
302
+ ("qkv_proj", "k_proj", "k"),
303
+ ("qkv_proj", "v_proj", "v"),
304
+ ("gate_up_proj", "gate_proj", 0),
305
+ ("gate_up_proj", "up_proj", 1),
306
]
349
- state_dict = self.state_dict()
350
-
307
+ params_dict = dict(self.named_parameters())
308
for name, loaded_weight in hf_model_weights_iterator(
309
model_name_or_path, cache_dir, load_format, revision):
310
if "rotary_emb.inv_freq" in name:
311
continue
355
-
356
- packed_dim = None
357
- is_transposed = False
358
- if self.quant_config is not None:
359
- packed_dim = self.quant_config.get_packed_dim(name)
360
- is_transposed = self.quant_config.is_transposed(name)
361
- if is_transposed:
362
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
363
- loaded_weight = loaded_weight.T
364
-
365
- is_attention_weight = False
366
- for weight_name, shard_size, offset in attention_weight_specs:
312
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
313
if weight_name not in name:
314
continue
369
- param = state_dict[name.replace(weight_name, "qkv_proj")]
370
- if is_transposed:
371
- param = param.T
372
-
373
- if packed_dim is not None:
374
- shard_dim = 0 if not is_transposed else 1
375
- if packed_dim == shard_dim:
376
- shard_size //= self.quant_config.pack_factor
377
- offset //= self.quant_config.pack_factor
378
-
379
- if weight_name in ["k_proj", "v_proj"]:
380
- shard_id = tp_rank // num_kv_heads_replicas
381
- else:
382
- shard_id = tp_rank
383
- loaded_weight = loaded_weight[shard_size *
384
- shard_id:shard_size *
385
- (shard_id + 1)]
386
- param_slice = param.data[offset:offset + shard_size]
387
- assert param_slice.shape == loaded_weight.shape
388
-
389
- param_slice.copy_(loaded_weight)
390
- is_attention_weight = True
391
- break
392
- if is_attention_weight:
393
- continue
394
-
395
- is_gate_up_weight = False
396
- for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
397
- if weight_name not in name:
398
- continue
399
- param = state_dict[name.replace(weight_name, "gate_up_proj")]
400
- if is_transposed:
401
- param = param.T
402
-
403
- shard_size = param.shape[0] // 2
404
- loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
405
- (tp_rank + 1)]
406
- param_slice = param.data[shard_size * stride_id:shard_size *
407
- (stride_id + 1)]
408
- assert param_slice.shape == loaded_weight.shape
409
- param_slice.copy_(loaded_weight)
410
- is_gate_up_weight = True
315
+ param = params_dict[name.replace(weight_name, param_name)]
316
+ weight_loader = param.weight_loader
317
+ weight_loader(param, loaded_weight, shard_id)
318
break
412
- if is_gate_up_weight:
413
- continue
414
-
415
- param = state_dict[name]
416
- if is_transposed:
417
- param = param.T
418
-
419
- if "embed_tokens" in name or "lm_head" in name:
420
- load_padded_tensor_parallel_vocab(param, loaded_weight,
421
- tp_rank)
422
- continue
423
-
424
- load_tensor_parallel_weights(param, loaded_weight, name,
425
- column_parallel_weights,
426
- row_parallel_weights, tp_rank)
319
+ else:
320
+ param = params_dict[name]
321
+ weight_loader = getattr(param, "weight_loader",
322
+ default_weight_loader)
323
+ weight_loader(param, loaded_weight)
vllm/model_executor/parallel_utils/layers.py
deleted
-303
@@ -1,303 +0,0 @@
1
-# Copyright 2023 The vLLM team.
2
-# Adapted from
3
-# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
4
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
-
6
-# Parts of the code here are adapted from PyTorch
7
-# repo: https://github.com/pytorch/pytorch
8
-from typing import Optional
9
-
10
-import torch
11
-import torch.nn.functional as F
12
-from torch.nn.parameter import Parameter
13
-
14
-from vllm.model_executor.parallel_utils.parallel_state import (
15
- get_tensor_model_parallel_rank,
16
- get_tensor_model_parallel_world_size,
17
-)
18
-from vllm.model_executor.quantization_utils import QuantizationConfig
19
-from vllm.model_executor.parallel_utils.communication_op import (
20
- tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
21
-
22
-from vllm.model_executor.parallel_utils.utils import (
23
- divide,
24
- VocabUtility,
25
- split_tensor_along_last_dim,
26
-)
27
-
28
-
29
-class VocabParallelEmbedding(torch.nn.Module):
30
- """Embedding parallelized in the vocabulary dimension.
31
-
32
- This is mainly adapted from torch.nn.Embedding and all the default
33
- values are kept.
34
- Arguments:
35
- num_embeddings: vocabulary size.
36
- embedding_dim: size of hidden state.
37
- params_dtype: type of the parameters.
38
- """
39
-
40
- def __init__(self,
41
- num_embeddings: int,
42
- embedding_dim: int,
43
- params_dtype: Optional[torch.dtype] = None):
44
- super().__init__()
45
-
46
- # Keep the input dimensions.
47
- self.num_embeddings = num_embeddings
48
- self.embedding_dim = embedding_dim
49
- if params_dtype is None:
50
- params_dtype = torch.get_default_dtype()
51
-
52
- self.tp_size = get_tensor_model_parallel_world_size()
53
- # TODO: Handle vocab padding here.
54
- # Divide the weight matrix along the vocaburaly dimension.
55
- self.vocab_start_index, self.vocab_end_index = (
56
- VocabUtility.vocab_range_from_global_vocab_size(
57
- self.num_embeddings, get_tensor_model_parallel_rank(),
58
- self.tp_size))
59
- self.num_embeddings_per_partition = (self.vocab_end_index -
60
- self.vocab_start_index)
61
-
62
- self.weight = Parameter(
63
- torch.empty(self.num_embeddings_per_partition,
64
- self.embedding_dim,
65
- device=torch.cuda.current_device(),
66
- dtype=params_dtype))
67
-
68
- def forward(self, input_):
69
- if self.tp_size > 1:
70
- # Build the mask.
71
- input_mask = ((input_ < self.vocab_start_index) |
72
- (input_ >= self.vocab_end_index))
73
- # Mask the input.
74
- masked_input = input_.clone() - self.vocab_start_index
75
- masked_input[input_mask] = 0
76
- else:
77
- masked_input = input_
78
- # Get the embeddings.
79
- output_parallel = F.embedding(masked_input, self.weight)
80
- # Mask the output embedding.
81
- if self.tp_size > 1:
82
- output_parallel[input_mask, :] = 0.0
83
- # Reduce across all the model parallel GPUs.
84
- output = tensor_model_parallel_all_reduce(output_parallel)
85
- return output
86
-
87
-
88
-class ColumnParallelLinear(torch.nn.Module):
89
- """Linear layer with column parallelism.
90
-
91
- The linear layer is defined as Y = XA + b. A is parallelized along
92
- its second dimension as A = [A_1, ..., A_p].
93
-
94
- Arguments:
95
- input_size: first dimension of matrix A.
96
- output_size: second dimension of matrix A.
97
-
98
- Keyword Arguments
99
- bias: If true, add bias
100
- gather_output: If true, call all-gather on output and make Y available
101
- to all GPUs, otherwise, every GPU will have its output
102
- which is Y_i = XA_i
103
- skip_bias_add: This was added to enable performance optimizations where
104
- bias can be fused with other element-wise operations. we
105
- skip adding bias but instead return it.
106
- params_dtype: Data type for the parameters.
107
- quant_config: Quantization configuration.
108
- """
109
-
110
- def __init__(
111
- self,
112
- input_size: int,
113
- output_size: int,
114
- bias: bool = True,
115
- gather_output: bool = True,
116
- skip_bias_add: bool = False,
117
- params_dtype: Optional[torch.dtype] = None,
118
- quant_config: Optional[QuantizationConfig] = None,
119
- ):
120
- super().__init__()
121
-
122
- # Keep input parameters
123
- self.input_size = input_size
124
- self.output_size = output_size
125
- self.gather_output = gather_output
126
- # Divide the weight matrix along the last dimension.
127
- self.tp_size = get_tensor_model_parallel_world_size()
128
- self.output_size_per_partition = divide(output_size, self.tp_size)
129
- self.skip_bias_add = skip_bias_add
130
- self.quant_config = quant_config
131
-
132
- if params_dtype is None:
133
- params_dtype = torch.get_default_dtype()
134
-
135
- # Parameters.
136
- # NOTE: torch.nn.functional.linear performs XA^T + b and as a result
137
- # we allocate the transpose.
138
- self.create_weights(params_dtype)
139
-
140
- if bias:
141
- self.bias = Parameter(
142
- torch.empty(self.output_size_per_partition,
143
- device=torch.cuda.current_device(),
144
- dtype=params_dtype))
145
- else:
146
- self.register_parameter('bias', None)
147
-
148
- def create_weights(self, dtype: torch.dtype) -> None:
149
- self.weight = Parameter(
150
- torch.empty(self.output_size_per_partition,
151
- self.input_size,
152
- device=torch.cuda.current_device(),
153
- dtype=dtype))
154
-
155
- def apply_weights(
156
- self,
157
- x: torch.Tensor,
158
- bias: Optional[torch.Tensor],
159
- ) -> torch.Tensor:
160
- return F.linear(x, self.weight, bias)
161
-
162
- def forward(self, input_):
163
- """Forward of ColumnParallelLinear
164
-
165
- Args:
166
- input_: Tensor whose last dimension is `input_size`.
167
-
168
- Returns:
169
- - output
170
- - bias
171
- """
172
- bias = self.bias if not self.skip_bias_add else None
173
-
174
- input_parallel = input_
175
- # Matrix multiply.
176
- output_parallel = self.apply_weights(input_parallel, bias)
177
- if self.gather_output:
178
- # All-gather across the partitions.
179
- output = tensor_model_parallel_all_gather(output_parallel)
180
- else:
181
- output = output_parallel
182
- output_bias = self.bias if self.skip_bias_add else None
183
- return output, output_bias
184
-
185
-
186
-class RowParallelLinear(torch.nn.Module):
187
- """Linear layer with row parallelism.
188
-
189
- The linear layer is defined as Y = XA + b. A is parallelized along
190
- its first dimension and X along its second dimension as:
191
- - -
192
- | A_1 |
193
- | . |
194
- A = | . | X = [X_1, ..., X_p]
195
- | . |
196
- | A_p |
197
- - -
198
- Arguments:
199
- input_size: first dimension of matrix A.
200
- output_size: second dimension of matrix A.
201
-
202
- Keyword Arguments:
203
- bias: If true, add bias. Note that bias is not parallelized.
204
- input_is_parallel: If true, we assume that the input is already
205
- split across the GPUs and we do not split
206
- again.
207
- skip_bias_add: This was added to enable performance optimization where
208
- bias can be fused with other element-wise operations.
209
- We skip adding bias but instead return it.
210
- params_dtype: Data type for the parameters.
211
- quant_config: Quantization configuration.
212
- """
213
-
214
- def __init__(
215
- self,
216
- input_size: int,
217
- output_size: int,
218
- bias: bool = True,
219
- input_is_parallel: bool = False,
220
- skip_bias_add: bool = False,
221
- params_dtype: Optional[torch.dtype] = None,
222
- reduce_results: bool = True,
223
- quant_config: Optional[QuantizationConfig] = None,
224
- ):
225
- super().__init__()
226
- # Keep input parameters
227
- self.input_size = input_size
228
- self.output_size = output_size
229
- self.input_is_parallel = input_is_parallel
230
- self.reduce_results = reduce_results
231
- if params_dtype is None:
232
- params_dtype = torch.get_default_dtype()
233
-
234
- # Divide the weight matrix along the last dimension.
235
- self.tp_size = get_tensor_model_parallel_world_size()
236
- self.input_size_per_partition = divide(input_size, self.tp_size)
237
- self.skip_bias_add = skip_bias_add
238
- self.quant_config = quant_config
239
-
240
- self.create_weights(params_dtype)
241
-
242
- if not reduce_results and (bias and not skip_bias_add):
243
- raise ValueError('When not reduce the results, adding bias to the '
244
- 'results can lead to incorrect results')
245
-
246
- if bias:
247
- self.bias = Parameter(
248
- torch.empty(self.output_size,
249
- device=torch.cuda.current_device(),
250
- dtype=params_dtype))
251
-
252
- # Always initialize bias to zero.
253
- with torch.no_grad():
254
- self.bias.zero_()
255
- else:
256
- self.register_parameter('bias', None)
257
-
258
- def create_weights(self, dtype: torch.dtype) -> None:
259
- self.weight = Parameter(
260
- torch.empty(self.output_size,
261
- self.input_size_per_partition,
262
- device=torch.cuda.current_device(),
263
- dtype=dtype))
264
-
265
- def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
266
- return F.linear(x, self.weight)
267
-
268
- def forward(self, input_):
269
- """Forward of RowParallelLinear
270
-
271
- Args:
272
- input_: tensor whose last dimension is `input_size`. If
273
- `input_is_parallel` is set, then the last dimension
274
- is `input_size // tp_size`.
275
-
276
- Returns:
277
- - output
278
- - bias
279
- """
280
- # Set up backprop all-reduce.
281
- if self.input_is_parallel:
282
- input_parallel = input_
283
- else:
284
- # TODO: simplify code below
285
- tp_rank = get_tensor_model_parallel_rank()
286
- splitted_input = split_tensor_along_last_dim(
287
- input_, num_partitions=self.tp_size)
288
- input_parallel = splitted_input[tp_rank].contiguous()
289
-
290
- # Matrix multiply.
291
- output_parallel = self.apply_weights(input_parallel)
292
- if self.reduce_results and self.tp_size > 1:
293
- output_ = tensor_model_parallel_all_reduce(output_parallel)
294
- else:
295
- output_ = output_parallel
296
-
297
- if not self.skip_bias_add:
298
- output = output_ + self.bias if self.bias is not None else output_
299
- output_bias = None
300
- else:
301
- output = output_
302
- output_bias = self.bias
303
- return output, output_bias
vllm/model_executor/parallel_utils/utils.py
+2
-24
@@ -2,7 +2,7 @@
2
# Adapted from
3
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
-from typing import List, Sequence
5
+from typing import Sequence
6
7
import torch
8
@@ -24,7 +24,7 @@ def split_tensor_along_last_dim(
24
tensor: torch.Tensor,
25
num_partitions: int,
26
contiguous_split_chunks: bool = False,
27
-) -> List[torch.Tensor]:
27
+) -> Sequence[torch.Tensor]:
28
""" Split a tensor along its last dimension.
29
30
Arguments:
@@ -46,25 +46,3 @@ def split_tensor_along_last_dim(
46
return tuple(chunk.contiguous() for chunk in tensor_list)
47
48
return tensor_list
49
-
50
-
51
-class VocabUtility:
52
- """ Split the vocabulary into `world_size` chunks and return the first
53
- and last index of the vocabulary belonging to the `rank`
54
- partition: Note that indices in [fist, last)
55
-
56
- """
57
-
58
- @staticmethod
59
- def vocab_range_from_per_partition_vocab_size(
60
- per_partition_vocab_size: int, rank: int) -> Sequence[int]:
61
- index_f = rank * per_partition_vocab_size
62
- index_l = index_f + per_partition_vocab_size
63
- return index_f, index_l
64
-
65
- @staticmethod
66
- def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
67
- world_size: int) -> Sequence[int]:
68
- per_partition_vocab_size = divide(global_vocab_size, world_size)
69
- return VocabUtility.vocab_range_from_per_partition_vocab_size(
70
- per_partition_vocab_size, rank)
vllm/model_executor/quantization_utils/__init__.py
deleted
-22
@@ -1,22 +0,0 @@
1
-from typing import Type
2
-
3
-from vllm.model_executor.quantization_utils.awq import AWQConfig
4
-from vllm.model_executor.quantization_utils.base import QuantizationConfig
5
-from vllm.model_executor.quantization_utils.squeezellm import SqueezeLLMConfig
6
-
7
-_QUANTIZATION_REGISTRY = {
8
- "awq": AWQConfig,
9
- "squeezellm": SqueezeLLMConfig,
10
-}
11
-
12
-
13
-def get_quant_class(quantization: str) -> Type[QuantizationConfig]:
14
- if quantization not in _QUANTIZATION_REGISTRY:
15
- raise ValueError(f"Invalid quantization method: {quantization}")
16
- return _QUANTIZATION_REGISTRY[quantization]
17
-
18
-
19
-__all__ = [
20
- "QuantizationConfig",
21
- "get_quant_class",
22
-]
vllm/model_executor/quantization_utils/awq.py
deleted
-76
@@ -1,76 +0,0 @@
1
-from typing import Any, Dict, List
2
-
3
-import torch
4
-
5
-from vllm.model_executor.quantization_utils.base import QuantizationConfig
6
-
7
-
8
-class AWQConfig(QuantizationConfig):
9
- """Config class for AWQ.
10
-
11
- Reference: https://arxiv.org/abs/2306.00978
12
- """
13
-
14
- def __init__(
15
- self,
16
- weight_bits: int,
17
- group_size: int,
18
- zero_point: bool,
19
- ) -> None:
20
- self.weight_bits = weight_bits
21
- self.group_size = group_size
22
- self.zero_point = zero_point
23
-
24
- if self.weight_bits != 4:
25
- raise ValueError(
26
- "Currently, only 4-bit weight quantization is supported for "
27
- f"AWQ, but got {self.weight_bits} bits.")
28
- self.pack_factor = 32 // self.weight_bits
29
-
30
- def __repr__(self) -> str:
31
- return (f"AWQConfig(weight_bits={self.weight_bits}, "
32
- f"group_size={self.group_size}, "
33
- f"zero_point={self.zero_point})")
34
-
35
- @classmethod
36
- def get_name(cls) -> str:
37
- return "awq"
38
-
39
- @classmethod
40
- def get_supported_act_dtypes(cls) -> List[torch.dtype]:
41
- return [torch.half]
42
-
43
- @classmethod
44
- def get_min_capability(cls) -> int:
45
- # The AWQ kernel only supports Turing or newer GPUs.
46
- return 75
47
-
48
- @classmethod
49
- def get_config_filenames(cls) -> List[str]:
50
- return [
51
- "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
52
- "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq # pylint: disable=line-too-long
53
- ]
54
-
55
- @classmethod
56
- def from_config(cls, config: Dict[str, Any]) -> "AWQConfig":
57
- weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
58
- group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
59
- zero_point = cls.get_from_keys(config, ["zero_point"])
60
- return cls(weight_bits, group_size, zero_point)
61
-
62
- @classmethod
63
- def get_packed_tensors(cls) -> Dict[str, int]:
64
- return {"qweight": 1, "qzeros": 1}
65
-
66
- @classmethod
67
- def get_transposed_tensor_names(cls) -> List[str]:
68
- return ["qweight", "qzeros", "scales"]
69
-
70
- @classmethod
71
- def get_col_parallel_tensor_names(cls) -> List[str]:
72
- return ["qweight", "qzeros", "scales"]
73
-
74
- @classmethod
75
- def get_row_parallel_tensor_names(cls) -> List[str]:
76
- return ["qweight", "qzeros", "scales"]
vllm/model_executor/quantization_utils/base.py
deleted
-85
@@ -1,85 +0,0 @@
1
-from typing import Any, Dict, List, Optional
2
-
3
-import torch
4
-
5
-
6
-class QuantizationConfig:
7
-
8
- @classmethod
9
- def get_name(cls) -> str:
10
- """Name of the quantization method."""
11
- raise NotImplementedError
12
-
13
- @classmethod
14
- def get_supported_act_dtypes(cls) -> List[torch.dtype]:
15
- """List of supported activation dtypes."""
16
- raise NotImplementedError
17
-
18
- @classmethod
19
- def get_min_capability(cls) -> int:
20
- """Minimum GPU capability to support the quantization method.
21
-
22
- E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
23
- This requirement is due to the custom CUDA kernels used by the
24
- quantization method.
25
- """
26
- raise NotImplementedError
27
-
28
- @classmethod
29
- def get_config_filenames(cls) -> List[str]:
30
- """List of filenames to search for in the model directory."""
31
- raise NotImplementedError
32
-
33
- @classmethod
34
- def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig":
35
- """Create a config class from the model's quantization config."""
36
- raise NotImplementedError
37
-
38
- @staticmethod
39
- def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
40
- """Get a value from the model's quantization config."""
41
- for key in keys:
42
- if key in config:
43
- return config[key]
44
- raise ValueError(f"Cannot find any of {keys} in the model's "
45
- "quantization config.")
46
-
47
- @classmethod
48
- def get_packed_tensors(cls) -> Dict[str, int]:
49
- """Returns a dictionary of packed tensor names and their pack dims."""
50
- raise NotImplementedError
51
-
52
- @classmethod
53
- def get_packed_dim(cls, tensor_name: str) -> Optional[int]:
54
- """Returns the pack dim of a tensor if it is packed.
55
-
56
- A tensor is considered packed if each element in the tensor is a
57
- packed representation of multiple elements in the original tensor.
58
- For example, an INT32 element in the tensor may represent 8 INT4
59
- elements in the original tensor.
60
- If the tensor is not packed, returns None.
61
- """
62
- packed_tensors = cls.get_packed_tensors()
63
- for packed_tensor_name, pack_dim in packed_tensors.items():
64
- if packed_tensor_name in tensor_name:
65
- return pack_dim
66
- return None
67
-
68
- @classmethod
69
- def get_transposed_tensor_names(cls) -> List[str]:
70
- raise NotImplementedError
71
-
72
- @classmethod
73
- def is_transposed(cls, tensor_name: str) -> bool:
74
- """Returns True if a tensor is transposed relative to nn.Linear.weight.
75
- """
76
- return any(tag in tensor_name
77
- for tag in cls.get_transposed_tensor_names())
78
-
79
- @classmethod
80
- def get_col_parallel_tensor_names(cls) -> List[str]:
81
- raise NotImplementedError
82
-
83
- @classmethod
84
- def get_row_parallel_tensor_names(cls) -> List[str]:
85
- raise NotImplementedError
vllm/model_executor/quantization_utils/squeezellm.py
deleted
-65
@@ -1,65 +0,0 @@
1
-from typing import Any, Dict, List
2
-
3
-import torch
4
-
5
-from vllm.model_executor.quantization_utils.base import QuantizationConfig
6
-
7
-
8
-class SqueezeLLMConfig(QuantizationConfig):
9
- """Config class for SqueezeLLM.
10
-
11
- Reference: https://arxiv.org/pdf/2306.07629
12
- """
13
-
14
- def __init__(
15
- self,
16
- weight_bits: int,
17
- ) -> None:
18
- self.weight_bits = weight_bits
19
-
20
- if self.weight_bits != 4:
21
- raise ValueError(
22
- "Currently, only 4-bit weight quantization is supported for "
23
- f"SqueezeLLM, but got {self.weight_bits} bits.")
24
-
25
- self.pack_factor = 32 // self.weight_bits
26
-
27
- def __repr__(self) -> str:
28
- return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
29
-
30
- @classmethod
31
- def get_name(cls) -> str:
32
- return "squeezellm"
33
-
34
- @classmethod
35
- def get_supported_act_dtypes(cls) -> List[torch.dtype]:
36
- return [torch.half]
37
-
38
- @classmethod
39
- def get_min_capability(cls) -> int:
40
- return 70
41
-
42
- @classmethod
43
- def get_config_filenames(cls) -> List[str]:
44
- return ["quant_config.json"]
45
-
46
- @classmethod
47
- def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
48
- weight_bits = cls.get_from_keys(config, ["wbits"])
49
- return cls(weight_bits)
50
-
51
- @classmethod
52
- def get_packed_tensors(cls) -> Dict[str, int]:
53
- return {"qweight": 0}
54
-
55
- @classmethod
56
- def get_transposed_tensor_names(cls) -> List[str]:
57
- return ["qweight"]
58
-
59
- @classmethod
60
- def get_col_parallel_tensor_names(cls) -> List[str]:
61
- return ["qweight", "lookup_table"]
62
-
63
- @classmethod
64
- def get_row_parallel_tensor_names(cls) -> List[str]:
65
- return ["qweight"]
vllm/model_executor/utils.py
+22
@@ -1,5 +1,6 @@
1
"""Utils for model executor."""
2
import random
3
+from typing import Any, Dict, Optional
4
5
import numpy as np
6
import torch
@@ -11,3 +12,24 @@ def set_random_seed(seed: int) -> None:
12
torch.manual_seed(seed)
13
if torch.cuda.is_available():
14
torch.cuda.manual_seed_all(seed)
15
+
16
+
17
+def set_weight_attrs(
18
+ weight: torch.Tensor,
19
+ weight_attrs: Optional[Dict[str, Any]],
20
+):
21
+ """Set attributes on a weight tensor.
22
+
23
+ This method is used to set attributes on a weight tensor. This method
24
+ will not overwrite existing attributes.
25
+
26
+ Args:
27
+ weight: The weight tensor.
28
+ weight_attrs: A dictionary of attributes to set on the weight tensor.
29
+ """
30
+ if weight_attrs is None:
31
+ return
32
+ for key, value in weight_attrs.items():
33
+ assert not hasattr(
34
+ weight, key), (f"Overwriting existing tensor attribute: {key}")
35
+ setattr(weight, key, value)
vllm/model_executor/weight_utils.py
+8
-44
@@ -13,8 +13,8 @@ import torch
13
from tqdm.auto import tqdm
14
15
from vllm.logger import init_logger
16
-from vllm.model_executor.quantization_utils import get_quant_class
17
-from vllm.model_executor.quantization_utils.base import QuantizationConfig
16
+from vllm.model_executor.layers.quantization import (get_quantization_config,
17
+ QuantizationConfig)
18
19
logger = init_logger(__name__)
20
@@ -98,7 +98,7 @@ def get_quant_config(
98
hf_folder = model_name_or_path
99
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
100
101
- quant_cls = get_quant_class(quantization)
101
+ quant_cls = get_quantization_config(quantization)
102
quant_config_files = [
103
f for f in config_files if any(
104
f.endswith(x) for x in quant_cls.get_config_filenames())
@@ -237,7 +237,7 @@ def hf_model_weights_iterator(
237
with safe_open(st_file, framework="pt") as f:
238
for name in f.keys():
239
param = f.get_slice(name)
240
- yield name, param
240
+ yield name, convert_pyslice_to_tensor(param)
241
else:
242
for bin_file in hf_weights_files:
243
state = torch.load(bin_file, map_location="cpu")
@@ -262,46 +262,10 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
262
return x
263
264
265
-def load_padded_tensor_parallel_vocab(
266
- param: torch.Tensor,
267
- loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
268
- tensor_model_parallel_rank: int,
269
-) -> None:
270
- shard_size = param.shape[0]
271
- start_idx = tensor_model_parallel_rank * shard_size
272
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
273
- loaded_weight = loaded_weight[start_idx:end_idx]
274
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
275
- param[:loaded_weight.shape[0]].copy_(loaded_weight)
276
-
277
-
278
-def load_tensor_parallel_weights(
279
- param: torch.Tensor,
280
- loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
281
- param_name: str,
282
- column_parallel_weight_names: List[str],
283
- row_parallel_weight_names: List[str],
284
- tensor_model_parallel_rank: int,
285
-) -> None:
286
- for p in column_parallel_weight_names:
287
- if p in param_name:
288
- shard_size = param.shape[0]
289
- start_idx = tensor_model_parallel_rank * shard_size
290
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
291
- loaded_weight = loaded_weight[start_idx:end_idx]
292
- break
293
- for p in row_parallel_weight_names:
294
- if p in param_name:
295
- shard_size = param.shape[1]
296
- start_idx = tensor_model_parallel_rank * shard_size
297
- end_idx = (tensor_model_parallel_rank + 1) * shard_size
298
- loaded_weight = loaded_weight[:, start_idx:end_idx]
299
- break
300
-
301
- loaded_weight = convert_pyslice_to_tensor(loaded_weight)
302
- assert param.shape == loaded_weight.shape, (
303
- f"{param_name} shape mismatch between model and checkpoint: "
304
- f"{param.shape} != {loaded_weight.shape}")
265
+def default_weight_loader(param: torch.Tensor,
266
+ loaded_weight: torch.Tensor) -> None:
267
+ """Default weight loader."""
268
+ assert param.size() == loaded_weight.size()
269
param.data.copy_(loaded_weight)
270
271