@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7076fa1c9f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

TP/quantization/weight loading refactor part 2 - Refactor quantized linear logic and extend quantization support to all models (#1622)

Refactor the tensor parallelism, quantization, and weight-loading codes. Summary of the new features enabled by this PR: - **All models** are able to be quantized with AWQ and SqueezeLLM, and [soon GPTQ](https://github.com/vllm-project/vllm/pull/1580). - Model loading code became much simpler. - Support model parallelism for all MQA/GQA models when the number of key/value heads is smaller than the tensor parallel size.

Zhuohan Li committed Nov 15, 2023 at 22:50 UTC 7076fa1c9f5769469bc2671afaca5af604a9bed3
36 files changed +2159 -2508
vllm/config.py
+30 -19
@@ -140,8 +140,8 @@ class ModelConfig:
140 # FIXME(woosuk): This may not be true for all models.
141 return self.hf_config.hidden_size // self.hf_config.num_attention_heads
142
143 - def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
144 - """Returns the number of KV heads per GPU worker."""
143 + def get_total_num_kv_heads(self) -> int:
144 + """Returns the total number of KV heads."""
145 # For GPTBigCode & Falcon:
146 # NOTE: for falcon, when new_decoder_architecture is True, the
147 # multi_query flag is ignored and we use n_head_kv for the number of
@@ -155,23 +155,34 @@ class ModelConfig:
155 # Multi-query attention, only one KV head.
156 # Currently, tensor parallelism is not supported in this case.
157 return 1
158 - # For Falcon:
159 - if getattr(self.hf_config, "n_head_kv", None) is not None:
160 - return (self.hf_config.n_head_kv //
161 - parallel_config.tensor_parallel_size)
162 - if getattr(self.hf_config, "num_kv_heads", None) is not None:
163 - return (self.hf_config.num_kv_heads //
164 - parallel_config.tensor_parallel_size)
165 - # For LLaMA-2:
166 - if getattr(self.hf_config, "num_key_value_heads", None) is not None:
167 - return (self.hf_config.num_key_value_heads //
168 - parallel_config.tensor_parallel_size)
169 - # For ChatGLM-2:
170 - if getattr(self.hf_config, "multi_query_group_num", None) is not None:
171 - return (self.hf_config.multi_query_group_num //
172 - parallel_config.tensor_parallel_size)
173 - total_num_attention_heads = self.hf_config.num_attention_heads
174 - return total_num_attention_heads // parallel_config.tensor_parallel_size
158 +
159 + attributes = [
160 + # For Falcon:
161 + "n_head_kv",
162 + "num_kv_heads",
163 + # For LLaMA-2:
164 + "num_key_value_heads",
165 + # For ChatGLM:
166 + "multi_query_group_num",
167 + ]
168 + for attr in attributes:
169 + num_kv_heads = getattr(self.hf_config, attr, None)
170 + if num_kv_heads is not None:
171 + return num_kv_heads
172 +
173 + # For non-grouped-query attention models, the number of KV heads is
174 + # equal to the number of attention heads.
175 + return self.hf_config.num_attention_heads
176 +
177 + def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
178 + """Returns the number of KV heads per GPU."""
179 + total_num_kv_heads = self.get_total_num_kv_heads()
180 + # If tensor parallelism is used, we divide the number of KV heads by
181 + # the tensor parallel size. We will replicate the KV heads in the
182 + # case where the number of KV heads is smaller than the tensor
183 + # parallel size so each GPU has at least one KV head.
184 + return max(1,
185 + total_num_kv_heads // parallel_config.tensor_parallel_size)
186
187 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
188 total_num_hidden_layers = self.hf_config.num_hidden_layers
vllm/engine/async_llm_engine.py
+2 -2
@@ -142,10 +142,10 @@ class RequestTracker:
142
143 self._request_streams[request_id].finish()
144
145 - def get_new_and_finished_requests(self) -> Tuple[List[dict], Set[str]]:
145 + def get_new_and_finished_requests(self) -> Tuple[List[Dict], Set[str]]:
146 """Get the new requests and finished requests to be
147 sent to the engine."""
148 - new_requests: List[dict] = []
148 + new_requests: List[Dict] = []
149 finished_requests: Set[str] = set()
150
151 while not self._finished_requests.empty():
vllm/model_executor/layers/linear.py new
+541
@@ -0,0 +1,541 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Dict, List, Optional
3 +
4 +import torch
5 +import torch.nn.functional as F
6 +from torch.nn.parameter import Parameter
7 +
8 +from vllm.model_executor.parallel_utils.parallel_state import (
9 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
10 +from vllm.model_executor.parallel_utils.communication_op import (
11 + tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
12 +from vllm.model_executor.parallel_utils.utils import (
13 + divide, split_tensor_along_last_dim)
14 +from vllm.model_executor.utils import set_weight_attrs
15 +from vllm.logger import init_logger
16 +
17 +logger = init_logger(__name__)
18 +
19 +
20 +class LinearMethodBase(ABC):
21 + """Base class for different (maybe quantized) linear methods."""
22 +
23 + @abstractmethod
24 + def create_weights(self, input_size: int, output_size: int,
25 + params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
26 + """Create weights for a linear layer."""
27 + raise NotImplementedError
28 +
29 + @abstractmethod
30 + def apply_weights(self,
31 + weights: Dict[str, torch.Tensor],
32 + x: torch.Tensor,
33 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
34 + """Apply the weights to the input tensor."""
35 + raise NotImplementedError
36 +
37 +
38 +class UnquantizedLinearMethod(LinearMethodBase):
39 + """Linear method without quantization.
40 +
41 + Args:
42 + separate_bias_add: If true, add bias separately after matrix
43 + multiplication.
44 + """
45 +
46 + def __init__(self, separate_bias_add: bool = False):
47 + self.separate_bias_add = separate_bias_add
48 +
49 + def create_weights(self, input_size: int, output_size: int,
50 + params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
51 + weight = Parameter(torch.empty(output_size,
52 + input_size,
53 + device=torch.cuda.current_device(),
54 + dtype=params_dtype),
55 + requires_grad=False)
56 + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
57 + return {"weight": weight}
58 +
59 + def apply_weights(self,
60 + weights: Dict[str, torch.Tensor],
61 + x: torch.Tensor,
62 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
63 + weight = weights["weight"]
64 + if self.separate_bias_add:
65 + if bias:
66 + return F.linear(x, weight) + bias
67 + return F.linear(x, weight)
68 + return F.linear(x, weight, bias)
69 +
70 +
71 +class ReplicatedLinear(torch.nn.Module):
72 + """Replicated linear layer.
73 +
74 + Args:
75 + input_size: input dimension of the linear layer.
76 + output_size: output dimension of the linear layer.
77 + bias: If true, add bias.
78 + skip_bias_add: If true, skip adding bias but instead return it.
79 + params_dtype: Data type for the parameters.
80 + linear_method: (Maybe quantized) linear method.
81 + """
82 +
83 + def __init__(
84 + self,
85 + input_size: int,
86 + output_size: int,
87 + bias: bool = True,
88 + skip_bias_add: bool = False,
89 + params_dtype: Optional[torch.dtype] = None,
90 + linear_method: Optional[LinearMethodBase] = None,
91 + ):
92 + super().__init__()
93 +
94 + # Keep input parameters
95 + self.input_size = input_size
96 + self.output_size = output_size
97 + self.skip_bias_add = skip_bias_add
98 + if params_dtype is None:
99 + params_dtype = torch.get_default_dtype()
100 + self.params_dtype = params_dtype
101 + if linear_method is None:
102 + linear_method = UnquantizedLinearMethod()
103 + self.linear_method = linear_method
104 + self.linear_weights = self.linear_method.create_weights(
105 + self.input_size, self.output_size, self.params_dtype)
106 + for name, weight in self.linear_weights.items():
107 + self.register_parameter(name, weight)
108 + if bias:
109 + self.bias = Parameter(
110 + torch.empty(self.output_size,
111 + device=torch.cuda.current_device(),
112 + dtype=self.params_dtype))
113 + set_weight_attrs(self.bias, {"output_dim": 0})
114 + else:
115 + self.register_parameter("bias", None)
116 +
117 + def forward(self, x: torch.Tensor) -> torch.Tensor:
118 + bias = self.bias if not self.skip_bias_add else None
119 + output = self.linear_method.apply_weights(self.linear_weights, x, bias)
120 + output_bias = self.bias if self.skip_bias_add else None
121 + return output, output_bias
122 +
123 +
124 +class ColumnParallelLinear(torch.nn.Module):
125 + """Linear layer with column parallelism.
126 +
127 + The linear layer is defined as Y = XA + b. A is parallelized along
128 + its second dimension as A = [A_1, ..., A_p].
129 +
130 + Args:
131 + input_size: first dimension of matrix A.
132 + output_size: second dimension of matrix A.
133 + bias: If true, add bias.
134 + gather_output: If true, call all-gather on output and make Y available
135 + to all GPUs, otherwise, every GPU will have its output
136 + which is Y_i = XA_i
137 + skip_bias_add: This was added to enable performance optimizations where
138 + bias can be fused with other element-wise operations. we
139 + skip adding bias but instead return it.
140 + params_dtype: Data type for the parameters.
141 + linear_method: (Maybe quantized) linear method.
142 + """
143 +
144 + def __init__(
145 + self,
146 + input_size: int,
147 + output_size: int,
148 + bias: bool = True,
149 + gather_output: bool = False,
150 + skip_bias_add: bool = False,
151 + params_dtype: Optional[torch.dtype] = None,
152 + linear_method: Optional[LinearMethodBase] = None,
153 + ):
154 + super().__init__()
155 +
156 + # Keep input parameters
157 + self.input_size = input_size
158 + self.output_size = output_size
159 + self.gather_output = gather_output
160 + # Divide the weight matrix along the last dimension.
161 + tp_size = get_tensor_model_parallel_world_size()
162 + self.output_size_per_partition = divide(output_size, tp_size)
163 + self.skip_bias_add = skip_bias_add
164 + if params_dtype is None:
165 + params_dtype = torch.get_default_dtype()
166 + self.params_dtype = params_dtype
167 + if linear_method is None:
168 + linear_method = UnquantizedLinearMethod()
169 + self.linear_method = linear_method
170 + self.linear_weights = self.linear_method.create_weights(
171 + self.input_size, self.output_size_per_partition, self.params_dtype)
172 + for name, weight in self.linear_weights.items():
173 + self.register_parameter(name, weight)
174 + set_weight_attrs(weight, {"weight_loader": self.weight_loader})
175 + if bias:
176 + self.bias = Parameter(
177 + torch.empty(self.output_size_per_partition,
178 + device=torch.cuda.current_device(),
179 + dtype=params_dtype))
180 + set_weight_attrs(self.bias, {
181 + "output_dim": 0,
182 + "weight_loader": self.weight_loader,
183 + })
184 + else:
185 + self.register_parameter("bias", None)
186 +
187 + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
188 + tp_rank = get_tensor_model_parallel_rank()
189 + output_dim = getattr(param, "output_dim", None)
190 + param_data = param.data
191 + if output_dim is not None:
192 + shard_size = param_data.shape[output_dim]
193 + start_idx = tp_rank * shard_size
194 + loaded_weight = loaded_weight.narrow(output_dim, start_idx,
195 + shard_size)
196 + assert param_data.shape == loaded_weight.shape
197 + param_data.copy_(loaded_weight)
198 +
199 + def forward(self, input_):
200 + bias = self.bias if not self.skip_bias_add else None
201 +
202 + # Matrix multiply.
203 + output_parallel = self.linear_method.apply_weights(
204 + self.linear_weights, input_, bias)
205 + if self.gather_output:
206 + # All-gather across the partitions.
207 + output = tensor_model_parallel_all_gather(output_parallel)
208 + else:
209 + output = output_parallel
210 + output_bias = self.bias if self.skip_bias_add else None
211 + return output, output_bias
212 +
213 +
214 +class MergedColumnParallelLinear(ColumnParallelLinear):
215 + """Packed linear layers with column parallelism.
216 +
217 + Similar to ColumnParallelLinear, but the weight matrix is concatenated
218 + along the output dimension. When the weight matrix is loaded, the
219 + different partitions are sharded separately.
220 +
221 + Args:
222 + input_size: input dimension of the linear layer.
223 + output_sizes: list of output dimensions of the linear layer.
224 + bias: If true, add bias.
225 + gather_output: If true, call all-gather on output and make the output
226 + available to all GPUs, otherwise, every GPU will have
227 + its own output.
228 + skip_bias_add: This was added to enable performance optimizations where
229 + bias can be fused with other element-wise operations. we
230 + skip adding bias but instead return it.
231 + params_dtype: Data type for the parameters.
232 + linear_method: (Maybe quantized) linear method.
233 + """
234 +
235 + def __init__(
236 + self,
237 + input_size: int,
238 + output_sizes: List[int],
239 + bias: bool = True,
240 + gather_output: bool = False,
241 + skip_bias_add: bool = False,
242 + params_dtype: Optional[torch.dtype] = None,
243 + linear_method: Optional[LinearMethodBase] = None,
244 + ):
245 + self.output_sizes = output_sizes
246 + tp_size = get_tensor_model_parallel_world_size()
247 + assert all(output_size % tp_size == 0 for output_size in output_sizes)
248 + super().__init__(input_size, sum(output_sizes), bias, gather_output,
249 + skip_bias_add, params_dtype, linear_method)
250 +
251 + def weight_loader(self,
252 + param: Parameter,
253 + loaded_weight: torch.Tensor,
254 + loaded_shard_id: Optional[int] = None):
255 + param_data = param.data
256 + output_dim = getattr(param, "output_dim", None)
257 + if loaded_shard_id is None:
258 + # Loaded weight is already packed.
259 + if output_dim is None:
260 + assert param_data.shape == loaded_weight.shape
261 + param_data.copy_(loaded_weight)
262 + return
263 + current_shard_offset = 0
264 + shard_offsets = []
265 + for i, output_size in enumerate(self.output_sizes):
266 + shard_offsets.append((i, current_shard_offset, output_size))
267 + current_shard_offset += output_size
268 + packed_dim = getattr(param, "packed_dim", None)
269 + for shard_id, shard_offset, shard_size in shard_offsets:
270 + # If quantized, we need to adjust the offset and size to account
271 + # for the packing.
272 + if packed_dim == output_dim:
273 + shard_size = shard_size // param.pack_factor
274 + shard_offset = shard_offset // param.pack_factor
275 + loaded_weight_shard = loaded_weight.narrow(
276 + output_dim, shard_offset, shard_size)
277 + self.weight_loader(param, loaded_weight_shard, shard_id)
278 + return
279 +
280 + assert loaded_shard_id < len(self.output_sizes)
281 + tp_rank = get_tensor_model_parallel_rank()
282 + tp_size = get_tensor_model_parallel_world_size()
283 + if output_dim is not None:
284 + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // tp_size
285 + shard_size = self.output_sizes[loaded_shard_id] // tp_size
286 + # If quantized, we need to adjust the offset and size to account
287 + # for the packing.
288 + packed_dim = getattr(param, "packed_dim", None)
289 + if packed_dim == output_dim:
290 + shard_size = shard_size // param.pack_factor
291 + shard_offset = shard_offset // param.pack_factor
292 + param_data = param_data.narrow(output_dim, shard_offset,
293 + shard_size)
294 + start_idx = tp_rank * shard_size
295 + loaded_weight = loaded_weight.narrow(output_dim, start_idx,
296 + shard_size)
297 + else:
298 + logger.warning(
299 + "Loading a weight without `output_dim` attribute in "
300 + "MergedColumnParallelLinear, assume the weight is "
301 + "the same for all partitions.")
302 + assert param_data.shape == loaded_weight.shape
303 + param_data.copy_(loaded_weight)
304 +
305 +
306 +class QKVParallelLinear(ColumnParallelLinear):
307 + """Linear layers for the attention's QKV transformation.
308 +
309 + Linear layers for the linear transformation of the query, key, and value
310 + vectors in the attention layer. The weight matrix is concatenated along
311 + the output dimension. The layer is parallelized along the head dimension.
312 + When the number of key/value heads is smaller than the number of query
313 + heads (e.g., multi-query/grouped-query attention), the key/value head may
314 + be replicated while the query heads are partitioned.
315 +
316 + Args:
317 + hidden_size: input hidden state size of the transformer.
318 + head_size: size of each attention head.
319 + total_num_heads: total number of attention query heads.
320 + total_num_kv_heads: total number of attention key/value heads. If
321 + None, assume total_num_kv_heads = total_num_heads.
322 + bias: If true, add bias.
323 + skip_bias_add: This was added to enable performance optimizations where
324 + bias can be fused with other element-wise operations. we
325 + skip adding bias but instead return it.
326 + params_dtype: Data type for the parameters.
327 + linear_method: (Maybe quantized) linear method.
328 + """
329 +
330 + def __init__(
331 + self,
332 + hidden_size: int,
333 + head_size: int,
334 + total_num_heads: int,
335 + total_num_kv_heads: Optional[int] = None,
336 + bias: bool = True,
337 + skip_bias_add: bool = False,
338 + params_dtype: Optional[torch.dtype] = None,
339 + linear_method: Optional[LinearMethodBase] = None,
340 + ):
341 + self.hidden_size = hidden_size
342 + self.head_size = head_size
343 + self.total_num_heads = total_num_heads
344 + if total_num_kv_heads is None:
345 + total_num_kv_heads = total_num_heads
346 + self.total_num_kv_heads = total_num_kv_heads
347 + # Divide the weight matrix along the last dimension.
348 + tp_size = get_tensor_model_parallel_world_size()
349 + self.num_heads = divide(self.total_num_heads, tp_size)
350 + if tp_size >= self.total_num_kv_heads:
351 + self.num_kv_heads = 1
352 + self.num_kv_head_replicas = divide(tp_size,
353 + self.total_num_kv_heads)
354 + else:
355 + self.num_kv_heads = divide(self.total_num_kv_heads, tp_size)
356 + self.num_kv_head_replicas = 1
357 + input_size = self.hidden_size
358 + output_size = (self.num_heads +
359 + 2 * self.num_kv_heads) * tp_size * self.head_size
360 + super().__init__(input_size, output_size, bias, False, skip_bias_add,
361 + params_dtype, linear_method)
362 +
363 + def weight_loader(self,
364 + param: Parameter,
365 + loaded_weight: torch.Tensor,
366 + loaded_shard_id: Optional[str] = None):
367 + param_data = param.data
368 + output_dim = getattr(param, "output_dim", None)
369 + if loaded_shard_id is None:
370 + # Loaded weight is already packed.
371 + if output_dim is None:
372 + assert param_data.shape == loaded_weight.shape
373 + param_data.copy_(loaded_weight)
374 + return
375 + shard_offsets = [
376 + # (shard_id, shard_offset, shard_size)
377 + ("q", 0, self.total_num_heads * self.head_size),
378 + ("k", self.total_num_heads * self.head_size,
379 + self.total_num_kv_heads * self.head_size),
380 + ("v", (self.total_num_heads + self.total_num_kv_heads) *
381 + self.head_size, self.total_num_kv_heads * self.head_size),
382 + ]
383 + packed_dim = getattr(param, "packed_dim", None)
384 + for shard_id, shard_offset, shard_size in shard_offsets:
385 + # If quantized, we need to adjust the offset and size to account
386 + # for the packing.
387 + if packed_dim == output_dim:
388 + shard_size = shard_size // param.pack_factor
389 + shard_offset = shard_offset // param.pack_factor
390 + loaded_weight_shard = loaded_weight.narrow(
391 + output_dim, shard_offset, shard_size)
392 + self.weight_loader(param, loaded_weight_shard, shard_id)
393 + return
394 +
395 + tp_rank = get_tensor_model_parallel_rank()
396 + assert loaded_shard_id in ["q", "k", "v"]
397 + if output_dim is not None:
398 + if loaded_shard_id == "q":
399 + shard_offset = 0
400 + shard_size = self.num_heads * self.head_size
401 + elif loaded_shard_id == "k":
402 + shard_offset = self.num_heads * self.head_size
403 + shard_size = self.num_kv_heads * self.head_size
404 + elif loaded_shard_id == "v":
405 + shard_offset = (self.num_heads +
406 + self.num_kv_heads) * self.head_size
407 + shard_size = self.num_kv_heads * self.head_size
408 + # If quantized, we need to adjust the offset and size to account
409 + # for the packing.
410 + packed_dim = getattr(param, "packed_dim", None)
411 + if packed_dim == output_dim:
412 + shard_size = shard_size // param.pack_factor
413 + shard_offset = shard_offset // param.pack_factor
414 + param_data = param_data.narrow(output_dim, shard_offset,
415 + shard_size)
416 + shard_id = tp_rank // self.num_kv_head_replicas
417 + start_idx = shard_id * shard_size
418 + loaded_weight = loaded_weight.narrow(output_dim, start_idx,
419 + shard_size)
420 + else:
421 + logger.warning(
422 + "Loading a weight without `output_dim` attribute in "
423 + "QKVParallelLinear, assume the weight is the same "
424 + "for all partitions.")
425 + assert param_data.shape == loaded_weight.shape
426 + param_data.copy_(loaded_weight)
427 +
428 +
429 +class RowParallelLinear(torch.nn.Module):
430 + """Linear layer with row parallelism.
431 +
432 + The linear layer is defined as Y = XA + b. A is parallelized along
433 + its first dimension and X along its second dimension as:
434 + - -
435 + | A_1 |
436 + | . |
437 + A = | . | X = [X_1, ..., X_p]
438 + | . |
439 + | A_p |
440 + - -
441 + Arguments:
442 + input_size: first dimension of matrix A.
443 + output_size: second dimension of matrix A.
444 + bias: If true, add bias. Note that bias is not parallelized.
445 + input_is_parallel: If true, we assume that the input is already
446 + split across the GPUs and we do not split
447 + again.
448 + skip_bias_add: This was added to enable performance optimization where
449 + bias can be fused with other element-wise operations.
450 + We skip adding bias but instead return it.
451 + params_dtype: Data type for the parameters.
452 + linear_method: (Maybe quantized) linear method.
453 + """
454 +
455 + def __init__(
456 + self,
457 + input_size: int,
458 + output_size: int,
459 + bias: bool = True,
460 + input_is_parallel: bool = True,
461 + skip_bias_add: bool = False,
462 + params_dtype: Optional[torch.dtype] = None,
463 + reduce_results: bool = True,
464 + linear_method: Optional[LinearMethodBase] = None,
465 + ):
466 + super().__init__()
467 + # Keep input parameters
468 + self.input_size = input_size
469 + self.output_size = output_size
470 + self.input_is_parallel = input_is_parallel
471 + self.reduce_results = reduce_results
472 + if params_dtype is None:
473 + params_dtype = torch.get_default_dtype()
474 + self.params_dtype = params_dtype
475 +
476 + # Divide the weight matrix along the last dimension.
477 + self.tp_size = get_tensor_model_parallel_world_size()
478 + self.input_size_per_partition = divide(input_size, self.tp_size)
479 + self.skip_bias_add = skip_bias_add
480 + if linear_method is None:
481 + linear_method = UnquantizedLinearMethod()
482 + self.linear_method = linear_method
483 + self.linear_weights = self.linear_method.create_weights(
484 + self.input_size_per_partition, self.output_size, self.params_dtype)
485 + for name, weight in self.linear_weights.items():
486 + self.register_parameter(name, weight)
487 + set_weight_attrs(weight, {"weight_loader": self.weight_loader})
488 +
489 + if not reduce_results and (bias and not skip_bias_add):
490 + raise ValueError("When not reduce the results, adding bias to the "
491 + "results can lead to incorrect results")
492 +
493 + if bias:
494 + self.bias = Parameter(
495 + torch.empty(self.output_size,
496 + device=torch.cuda.current_device(),
497 + dtype=params_dtype))
498 + set_weight_attrs(self.bias, {
499 + "output_dim": 0,
500 + "weight_loader": self.weight_loader,
501 + })
502 + else:
503 + self.register_parameter("bias", None)
504 +
505 + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
506 + tp_rank = get_tensor_model_parallel_rank()
507 + input_dim = getattr(param, "input_dim", None)
508 + param_data = param.data
509 + if input_dim is not None:
510 + shard_size = param_data.shape[input_dim]
511 + start_idx = tp_rank * shard_size
512 + loaded_weight = loaded_weight.narrow(input_dim, start_idx,
513 + shard_size)
514 + assert param_data.shape == loaded_weight.shape
515 + param_data.copy_(loaded_weight)
516 +
517 + def forward(self, input_):
518 + # Set up backprop all-reduce.
519 + if self.input_is_parallel:
520 + input_parallel = input_
521 + else:
522 + tp_rank = get_tensor_model_parallel_rank()
523 + splitted_input = split_tensor_along_last_dim(
524 + input_, num_partitions=self.tp_size)
525 + input_parallel = splitted_input[tp_rank].contiguous()
526 +
527 + # Matrix multiply.
528 + output_parallel = self.linear_method.apply_weights(
529 + self.linear_weights, input_parallel)
530 + if self.reduce_results and self.tp_size > 1:
531 + output_ = tensor_model_parallel_all_reduce(output_parallel)
532 + else:
533 + output_ = output_parallel
534 +
535 + if not self.skip_bias_add:
536 + output = output_ + self.bias if self.bias is not None else output_
537 + output_bias = None
538 + else:
539 + output = output_
540 + output_bias = self.bias
541 + return output, output_bias
vllm/model_executor/layers/quantization/__init__.py new
+22
@@ -0,0 +1,22 @@
1 +from typing import Type
2 +
3 +from vllm.model_executor.layers.quantization.awq import AWQConfig
4 +from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
5 +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
6 +
7 +_QUANTIZATION_CONFIG_REGISTRY = {
8 + "awq": AWQConfig,
9 + "squeezellm": SqueezeLLMConfig,
10 +}
11 +
12 +
13 +def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
14 + if quantization not in _QUANTIZATION_CONFIG_REGISTRY:
15 + raise ValueError(f"Invalid quantization method: {quantization}")
16 + return _QUANTIZATION_CONFIG_REGISTRY[quantization]
17 +
18 +
19 +__all__ = [
20 + "QuantizationConfig",
21 + "get_quantization_config",
22 +]
vllm/model_executor/layers/quantization/awq.py new
+155
@@ -0,0 +1,155 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +from torch.nn.parameter import Parameter
5 +
6 +from vllm import quantization_ops
7 +from vllm.model_executor.layers.linear import (LinearMethodBase,
8 + set_weight_attrs)
9 +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
10 +
11 +
12 +class AWQConfig(QuantizationConfig):
13 + """Config class for AWQ.
14 +
15 + Reference: https://arxiv.org/abs/2306.00978
16 + """
17 +
18 + def __init__(
19 + self,
20 + weight_bits: int,
21 + group_size: int,
22 + zero_point: bool,
23 + ) -> None:
24 + self.weight_bits = weight_bits
25 + self.group_size = group_size
26 + self.zero_point = zero_point
27 +
28 + if self.weight_bits != 4:
29 + raise ValueError(
30 + "Currently, only 4-bit weight quantization is supported for "
31 + f"AWQ, but got {self.weight_bits} bits.")
32 + self.pack_factor = 32 // self.weight_bits
33 +
34 + def __repr__(self) -> str:
35 + return (f"AWQConfig(weight_bits={self.weight_bits}, "
36 + f"group_size={self.group_size}, "
37 + f"zero_point={self.zero_point})")
38 +
39 + def get_name(self) -> str:
40 + return "awq"
41 +
42 + def get_supported_act_dtypes(self) -> List[torch.dtype]:
43 + return [torch.half]
44 +
45 + def get_min_capability(self) -> int:
46 + # The AWQ kernel only supports Turing or newer GPUs.
47 + return 75
48 +
49 + @staticmethod
50 + def get_config_filenames() -> List[str]:
51 + return [
52 + "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
53 + "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq # pylint: disable=line-too-long
54 + ]
55 +
56 + @classmethod
57 + def from_config(cls, config: Dict[str, Any]) -> "AWQConfig":
58 + weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
59 + group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
60 + zero_point = cls.get_from_keys(config, ["zero_point"])
61 + return cls(weight_bits, group_size, zero_point)
62 +
63 + def get_linear_method(self) -> "AWQLinearMethod":
64 + return AWQLinearMethod(self)
65 +
66 +
67 +class AWQLinearMethod(LinearMethodBase):
68 + """Linear method for AWQ.
69 +
70 + Args:
71 + quant_config: The AWQ quantization config.
72 + """
73 +
74 + def __init__(self, quant_config: AWQConfig):
75 + self.quant_config = quant_config
76 +
77 + def create_weights(self, input_size: int, output_size: int,
78 + params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
79 + if input_size % self.quant_config.group_size != 0:
80 + raise ValueError(
81 + "The input size is not aligned with the quantized "
82 + "weight shape. This can be caused by too large "
83 + "tensor parallel size.")
84 + if output_size % self.quant_config.pack_factor != 0:
85 + raise ValueError(
86 + "The output size is not aligned with the quantized "
87 + "weight shape. This can be caused by too large "
88 + "tensor parallel size.")
89 +
90 + qweight = Parameter(
91 + torch.empty(
92 + input_size,
93 + output_size // self.quant_config.pack_factor,
94 + device="cuda",
95 + dtype=torch.int32,
96 + ),
97 + requires_grad=False,
98 + )
99 + set_weight_attrs(
100 + qweight, {
101 + "input_dim": 0,
102 + "output_dim": 1,
103 + "packed_dim": 1,
104 + "pack_factor": self.quant_config.pack_factor,
105 + })
106 + qzeros = Parameter(
107 + torch.empty(
108 + input_size // self.quant_config.group_size,
109 + output_size // self.quant_config.pack_factor,
110 + device="cuda",
111 + dtype=torch.int32,
112 + ),
113 + requires_grad=False,
114 + )
115 + set_weight_attrs(
116 + qzeros, {
117 + "input_dim": 0,
118 + "output_dim": 1,
119 + "packed_dim": 1,
120 + "pack_factor": self.quant_config.pack_factor,
121 + })
122 + scales = Parameter(
123 + torch.empty(
124 + input_size // self.quant_config.group_size,
125 + output_size,
126 + device="cuda",
127 + dtype=params_dtype,
128 + ),
129 + requires_grad=False,
130 + )
131 + set_weight_attrs(scales, {
132 + "input_dim": 0,
133 + "output_dim": 1,
134 + })
135 + return {
136 + "qweight": qweight,
137 + "qzeros": qzeros,
138 + "scales": scales,
139 + }
140 +
141 + def apply_weights(self,
142 + weights: Dict[str, torch.Tensor],
143 + x: torch.Tensor,
144 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
145 + qweight = weights["qweight"]
146 + qzeros = weights["qzeros"]
147 + scales = weights["scales"]
148 + pack_factor = self.quant_config.pack_factor
149 + out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
150 + reshaped_x = x.reshape(-1, x.shape[-1])
151 + out = quantization_ops.awq_gemm(reshaped_x, qweight, scales, qzeros,
152 + pack_factor)
153 + if bias is not None:
154 + out = out + bias
155 + return out.reshape(out_shape)
vllm/model_executor/layers/quantization/base_config.py new
+56
@@ -0,0 +1,56 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Any, Dict, List
3 +
4 +import torch
5 +
6 +from vllm.model_executor.layers.linear import LinearMethodBase
7 +
8 +
9 +class QuantizationConfig(ABC):
10 + """Base class for quantization configs."""
11 +
12 + @abstractmethod
13 + def get_name(self) -> str:
14 + """Name of the quantization method."""
15 + raise NotImplementedError
16 +
17 + @abstractmethod
18 + def get_supported_act_dtypes(self) -> List[torch.dtype]:
19 + """List of supported activation dtypes."""
20 + raise NotImplementedError
21 +
22 + @abstractmethod
23 + def get_min_capability(self) -> int:
24 + """Minimum GPU capability to support the quantization method.
25 +
26 + E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
27 + This requirement is due to the custom CUDA kernels used by the
28 + quantization method.
29 + """
30 + raise NotImplementedError
31 +
32 + @staticmethod
33 + @abstractmethod
34 + def get_config_filenames() -> List[str]:
35 + """List of filenames to search for in the model directory."""
36 + raise NotImplementedError
37 +
38 + @classmethod
39 + @abstractmethod
40 + def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig":
41 + """Create a config class from the model's quantization config."""
42 + raise NotImplementedError
43 +
44 + @staticmethod
45 + def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
46 + """Get a value from the model's quantization config."""
47 + for key in keys:
48 + if key in config:
49 + return config[key]
50 + raise ValueError(f"Cannot find any of {keys} in the model's "
51 + "quantization config.")
52 +
53 + @abstractmethod
54 + def get_linear_method(self) -> LinearMethodBase:
55 + """Get the linear method to use for the quantized linear layer."""
56 + raise NotImplementedError
vllm/model_executor/layers/quantization/squeezellm.py new
+121
@@ -0,0 +1,121 @@
1 +from typing import Any, Dict, List, Optional
2 +
3 +import torch
4 +from torch.nn.parameter import Parameter
5 +
6 +from vllm import quantization_ops
7 +from vllm.model_executor.layers.linear import (LinearMethodBase,
8 + set_weight_attrs)
9 +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
10 +
11 +
12 +class SqueezeLLMConfig(QuantizationConfig):
13 + """Config class for SqueezeLLM.
14 +
15 + Reference: https://arxiv.org/pdf/2306.07629
16 + """
17 +
18 + def __init__(
19 + self,
20 + weight_bits: int,
21 + ) -> None:
22 + self.weight_bits = weight_bits
23 +
24 + if self.weight_bits != 4:
25 + raise ValueError(
26 + "Currently, only 4-bit weight quantization is supported for "
27 + f"SqueezeLLM, but got {self.weight_bits} bits.")
28 +
29 + self.pack_factor = 32 // self.weight_bits
30 +
31 + def __repr__(self) -> str:
32 + return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
33 +
34 + def get_name(self) -> str:
35 + return "squeezellm"
36 +
37 + def get_supported_act_dtypes(self) -> List[torch.dtype]:
38 + return [torch.half]
39 +
40 + def get_min_capability(self) -> int:
41 + return 70
42 +
43 + @staticmethod
44 + def get_config_filenames() -> List[str]:
45 + return ["quant_config.json"]
46 +
47 + @classmethod
48 + def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
49 + weight_bits = cls.get_from_keys(config, ["wbits"])
50 + return cls(weight_bits)
51 +
52 + def get_linear_method(self) -> "SqueezeLLMLinearMethod":
53 + return SqueezeLLMLinearMethod(self)
54 +
55 +
56 +class SqueezeLLMLinearMethod(LinearMethodBase):
57 + """Linear method for SqueezeLLM.
58 +
59 + Args:
60 + quant_config: The SqueezeLLM quantization config.
61 + """
62 +
63 + def __init__(self, quant_config: SqueezeLLMConfig):
64 + self.quant_config = quant_config
65 +
66 + def create_weights(self, input_size: int, output_size: int,
67 + params_dtype: torch.dtype) -> Dict[str, torch.Tensor]:
68 + if input_size % self.quant_config.pack_factor != 0:
69 + raise ValueError(
70 + "The input size is not aligned with the quantized "
71 + "weight shape. This can be caused by too large "
72 + "tensor parallel size.")
73 + qweight = Parameter(
74 + torch.empty(
75 + input_size // self.quant_config.pack_factor,
76 + output_size,
77 + device="cuda",
78 + dtype=torch.int32,
79 + ),
80 + requires_grad=False,
81 + )
82 + set_weight_attrs(
83 + qweight, {
84 + "input_dim": 0,
85 + "output_dim": 1,
86 + "packed_dim": 0,
87 + "pack_factor": self.quant_config.pack_factor,
88 + })
89 + lookup_table = Parameter(
90 + torch.empty(
91 + output_size,
92 + self.quant_config.weight_bits**2,
93 + device="cuda",
94 + dtype=params_dtype,
95 + ),
96 + requires_grad=False,
97 + )
98 + set_weight_attrs(lookup_table, {
99 + "output_dim": 0,
100 + })
101 + return {
102 + "qweight": qweight,
103 + "lookup_table": lookup_table,
104 + }
105 +
106 + def apply_weights(self,
107 + weights: Dict[str, torch.Tensor],
108 + x: torch.Tensor,
109 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
110 + qweight = weights["qweight"]
111 + lookup_table = weights["lookup_table"]
112 + out_shape = x.shape[:-1] + (qweight.shape[-1], )
113 + reshaped_x = x.reshape(-1, x.shape[-1])
114 + # NOTE: The output tensor should be zero-initialized.
115 + out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
116 + quantization_ops.squeezellm_gemm(reshaped_x, qweight, out,
117 + lookup_table)
118 +
119 + if bias is not None:
120 + out = out + bias
121 + return out.reshape(out_shape)
vllm/model_executor/layers/quantized_linear/__init__.py deleted
-41
@@ -1,41 +0,0 @@
1 -from vllm.model_executor.layers.quantized_linear.awq import (
2 - AWQColumnParallelLinear, AWQRowParallelLinear)
3 -from vllm.model_executor.layers.quantized_linear.squeezellm import (
4 - SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear)
5 -from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
6 - RowParallelLinear)
7 -
8 -_QUANTIZED_LINEAR_REGISTRY = {
9 - "awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
10 - "squeezellm":
11 - (SqueezeLLMColumnParallelLinear, SqueezeLLMRowParallelLinear),
12 -}
13 -
14 -
15 -class ParallelLinear:
16 -
17 - @classmethod
18 - def column(cls, *args, **kwargs) -> ColumnParallelLinear:
19 - quant_config = kwargs.get("quant_config", None)
20 - if quant_config is None:
21 - return ColumnParallelLinear(*args, **kwargs)
22 -
23 - name = quant_config.get_name()
24 - if name not in _QUANTIZED_LINEAR_REGISTRY:
25 - raise ValueError(f"No quantized linear is found for {name}")
26 -
27 - quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][0]
28 - return quant_linear_cls(*args, **kwargs)
29 -
30 - @classmethod
31 - def row(cls, *args, **kwargs) -> RowParallelLinear:
32 - quant_config = kwargs.get("quant_config", None)
33 - if quant_config is None:
34 - return RowParallelLinear(*args, **kwargs)
35 -
36 - name = quant_config.get_name()
37 - if name not in _QUANTIZED_LINEAR_REGISTRY:
38 - raise ValueError(f"No quantized linear is found for {name}")
39 -
40 - quant_linear_cls = _QUANTIZED_LINEAR_REGISTRY[name][1]
41 - return quant_linear_cls(*args, **kwargs)
vllm/model_executor/layers/quantized_linear/awq.py deleted
-106
@@ -1,106 +0,0 @@
1 -from typing import Optional
2 -
3 -import torch
4 -from torch.nn.parameter import Parameter
5 -
6 -from vllm import quantization_ops
7 -from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8 - RowParallelLinear)
9 -
10 -
11 -class AWQColumnParallelLinear(ColumnParallelLinear):
12 -
13 - def create_weights(self, dtype: torch.dtype) -> None:
14 - assert self.input_size % self.quant_config.group_size == 0
15 - if self.output_size_per_partition % self.quant_config.pack_factor != 0:
16 - raise ValueError(
17 - "The tensor parallel size is not aligned with the quantized "
18 - "weight shape. Please use a different tensor parallel size.")
19 - self.qweight = Parameter(
20 - torch.empty(
21 - self.input_size,
22 - self.output_size_per_partition //
23 - self.quant_config.pack_factor,
24 - device="cuda",
25 - dtype=torch.int32,
26 - ),
27 - requires_grad=False,
28 - )
29 - self.qzeros = Parameter(
30 - torch.empty(
31 - self.input_size // self.quant_config.group_size,
32 - self.output_size_per_partition //
33 - self.quant_config.pack_factor,
34 - device="cuda",
35 - dtype=torch.int32,
36 - ),
37 - requires_grad=False,
38 - )
39 - self.scales = Parameter(
40 - torch.empty(
41 - self.input_size // self.quant_config.group_size,
42 - self.output_size_per_partition,
43 - device="cuda",
44 - dtype=dtype,
45 - ),
46 - requires_grad=False,
47 - )
48 -
49 - def apply_weights(
50 - self,
51 - x: torch.Tensor,
52 - bias: Optional[torch.Tensor],
53 - ) -> torch.Tensor:
54 - pack_factor = self.quant_config.pack_factor
55 - out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
56 - reshaped_x = x.reshape(-1, x.shape[-1])
57 - out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
58 - self.qzeros, pack_factor)
59 - if bias is not None:
60 - out = out + bias
61 - return out.reshape(out_shape)
62 -
63 -
64 -class AWQRowParallelLinear(RowParallelLinear):
65 -
66 - def create_weights(self, dtype: torch.dtype) -> None:
67 - assert self.output_size % self.quant_config.pack_factor == 0
68 - if self.input_size_per_partition % self.quant_config.group_size != 0:
69 - raise ValueError(
70 - "The tensor parallel size is not aligned with the quantized "
71 - "weight shape. Please use a different tensor parallel size.")
72 - self.qweight = Parameter(
73 - torch.empty(
74 - self.input_size_per_partition,
75 - self.output_size // self.quant_config.pack_factor,
76 - device="cuda",
77 - dtype=torch.int32,
78 - ),
79 - requires_grad=False,
80 - )
81 - self.qzeros = Parameter(
82 - torch.empty(
83 - self.input_size_per_partition // self.quant_config.group_size,
84 - self.output_size // self.quant_config.pack_factor,
85 - device="cuda",
86 - dtype=torch.int32,
87 - ),
88 - requires_grad=False,
89 - )
90 - self.scales = Parameter(
91 - torch.empty(
92 - self.input_size_per_partition // self.quant_config.group_size,
93 - self.output_size,
94 - device="cuda",
95 - dtype=dtype,
96 - ),
97 - requires_grad=False,
98 - )
99 -
100 - def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
101 - pack_factor = self.quant_config.pack_factor
102 - out_shape = (x.shape[:-1] + (self.qweight.shape[-1] * pack_factor, ))
103 - reshaped_x = x.reshape(-1, x.shape[-1])
104 - out = quantization_ops.awq_gemm(reshaped_x, self.qweight, self.scales,
105 - self.qzeros, pack_factor)
106 - return out.reshape(out_shape)
vllm/model_executor/layers/quantized_linear/squeezellm.py deleted
-84
@@ -1,84 +0,0 @@
1 -from typing import Optional
2 -
3 -import torch
4 -from torch.nn.parameter import Parameter
5 -
6 -from vllm import quantization_ops
7 -from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8 - RowParallelLinear)
9 -
10 -
11 -class SqueezeLLMColumnParallelLinear(ColumnParallelLinear):
12 -
13 - def create_weights(self, dtype: torch.dtype) -> None:
14 - assert self.input_size % self.quant_config.pack_factor == 0
15 - self.qweight = Parameter(
16 - torch.empty(
17 - self.input_size // self.quant_config.pack_factor,
18 - self.output_size_per_partition,
19 - device="cuda",
20 - dtype=torch.int32,
21 - ),
22 - requires_grad=False,
23 - )
24 - self.lookup_table = Parameter(
25 - torch.empty(
26 - self.output_size_per_partition,
27 - self.quant_config.weight_bits**2,
28 - device="cuda",
29 - dtype=dtype,
30 - ),
31 - requires_grad=False,
32 - )
33 -
34 - def apply_weights(
35 - self,
36 - x: torch.Tensor,
37 - bias: Optional[torch.Tensor],
38 - ) -> torch.Tensor:
39 - out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
40 - reshaped_x = x.reshape(-1, x.shape[-1])
41 - # NOTE: The output tensor should be zero-initialized.
42 - out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
43 - quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
44 - self.lookup_table)
45 -
46 - if bias is not None:
47 - out = out + bias
48 - return out.reshape(out_shape)
49 -
50 -
51 -class SqueezeLLMRowParallelLinear(RowParallelLinear):
52 -
53 - def create_weights(self, dtype: torch.dtype) -> None:
54 - if self.input_size_per_partition % self.quant_config.pack_factor != 0:
55 - raise ValueError(
56 - "The tensor parallel size is not aligned with the quantized "
57 - "weight shape. Please use a different tensor parallel size.")
58 - self.qweight = Parameter(
59 - torch.empty(
60 - self.input_size_per_partition // self.quant_config.pack_factor,
61 - self.output_size,
62 - device="cuda",
63 - dtype=torch.int32,
64 - ),
65 - requires_grad=False,
66 - )
67 - self.lookup_table = Parameter(
68 - torch.empty(
69 - self.output_size,
70 - self.quant_config.weight_bits**2,
71 - device="cuda",
72 - dtype=dtype,
73 - ),
74 - requires_grad=False,
75 - )
76 -
77 - def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
78 - reshaped_x = x.reshape(-1, x.shape[-1])
79 - out_shape = x.shape[:-1] + (self.qweight.shape[-1], )
80 - # NOTE: The output tensor should be zero-initialized.
81 - out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
82 - quantization_ops.squeezellm_gemm(reshaped_x, self.qweight, out,
83 - self.lookup_table)
84 - return out.reshape(out_shape)
vllm/model_executor/layers/vocab_parallel_embedding.py new
+139
@@ -0,0 +1,139 @@
1 +from typing import Optional, Sequence
2 +
3 +import torch
4 +import torch.nn.functional as F
5 +from torch.nn.parameter import Parameter
6 +
7 +from vllm.model_executor.parallel_utils.parallel_state import (
8 + get_tensor_model_parallel_rank,
9 + get_tensor_model_parallel_world_size,
10 +)
11 +from vllm.model_executor.parallel_utils.utils import divide
12 +from vllm.model_executor.parallel_utils.communication_op import (
13 + tensor_model_parallel_all_reduce)
14 +from vllm.model_executor.utils import set_weight_attrs
15 +
16 +
17 +def pad_vocab_size(vocab_size: int, pad_to: int = 64) -> int:
18 + """Pad the vocab size to the given value."""
19 + return ((vocab_size + pad_to - 1) // pad_to) * pad_to
20 +
21 +
22 +def vocab_range_from_per_partition_vocab_size(per_partition_vocab_size: int,
23 + rank: int) -> Sequence[int]:
24 + index_f = rank * per_partition_vocab_size
25 + index_l = index_f + per_partition_vocab_size
26 + return index_f, index_l
27 +
28 +
29 +def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
30 + world_size: int) -> Sequence[int]:
31 + per_partition_vocab_size = divide(global_vocab_size, world_size)
32 + return vocab_range_from_per_partition_vocab_size(per_partition_vocab_size,
33 + rank)
34 +
35 +
36 +class VocabParallelEmbedding(torch.nn.Module):
37 + """Embedding parallelized in the vocabulary dimension.
38 +
39 + Adapted from torch.nn.Embedding, note that we pad the vocabulary size to
40 + make sure it is divisible by the number of model parallel GPUs.
41 +
42 + Args:
43 + num_embeddings: vocabulary size.
44 + embedding_dim: size of hidden state.
45 + params_dtype: type of the parameters.
46 + """
47 +
48 + def __init__(self,
49 + num_embeddings: int,
50 + embedding_dim: int,
51 + params_dtype: Optional[torch.dtype] = None):
52 + super().__init__()
53 +
54 + # Keep the input dimensions.
55 + self.num_embeddings = num_embeddings
56 + self.num_embeddings_padded = pad_vocab_size(num_embeddings)
57 + self.embedding_dim = embedding_dim
58 + if params_dtype is None:
59 + params_dtype = torch.get_default_dtype()
60 + self.tp_size = get_tensor_model_parallel_world_size()
61 + # Divide the weight matrix along the vocaburaly dimension.
62 + self.vocab_start_index, self.vocab_end_index = (
63 + vocab_range_from_global_vocab_size(
64 + self.num_embeddings_padded, get_tensor_model_parallel_rank(),
65 + self.tp_size))
66 + self.num_embeddings_per_partition = (self.vocab_end_index -
67 + self.vocab_start_index)
68 + self.weight = Parameter(
69 + torch.empty(self.num_embeddings_per_partition,
70 + self.embedding_dim,
71 + device=torch.cuda.current_device(),
72 + dtype=params_dtype))
73 + set_weight_attrs(self.weight, {
74 + "parallel_dim": 0,
75 + "weight_loader": self.weight_loader
76 + })
77 +
78 + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
79 + parallel_dim = param.parallel_dim
80 + assert loaded_weight.shape[parallel_dim] == self.num_embeddings
81 + loaded_weight = loaded_weight[self.vocab_start_index:self.
82 + vocab_end_index]
83 + param[:loaded_weight.shape[0]].data.copy_(loaded_weight)
84 +
85 + def forward(self, input_):
86 + if self.tp_size > 1:
87 + # Build the mask.
88 + input_mask = ((input_ < self.vocab_start_index) |
89 + (input_ >= self.vocab_end_index))
90 + # Mask the input.
91 + masked_input = input_.clone() - self.vocab_start_index
92 + masked_input[input_mask] = 0
93 + else:
94 + masked_input = input_
95 + # Get the embeddings.
96 + output_parallel = F.embedding(masked_input, self.weight)
97 + # Mask the output embedding.
98 + if self.tp_size > 1:
99 + output_parallel[input_mask, :] = 0.0
100 + # Reduce across all the model parallel GPUs.
101 + output = tensor_model_parallel_all_reduce(output_parallel)
102 + return output
103 +
104 +
105 +class ParallelLMHead(VocabParallelEmbedding):
106 + """Parallelized LM head.
107 +
108 + Output logits weight matrices used in the Sampler. The weight and bias
109 + tensors are padded to make sure they are divisible by the number of
110 + model parallel GPUs.
111 +
112 + Args:
113 + num_embeddings: vocabulary size.
114 + embedding_dim: size of hidden state.
115 + bias: whether to use bias.
116 + params_dtype: type of the parameters.
117 + """
118 +
119 + def __init__(self,
120 + num_embeddings: int,
121 + embedding_dim: int,
122 + bias: bool = False,
123 + params_dtype: Optional[torch.dtype] = None):
124 + super().__init__(num_embeddings, embedding_dim, params_dtype)
125 + if bias:
126 + self.bias = Parameter(
127 + torch.empty(self.num_embeddings_per_partition,
128 + device=torch.cuda.current_device(),
129 + dtype=params_dtype))
130 + set_weight_attrs(self.bias, {
131 + "parallel_dim": 0,
132 + "weight_loader": self.weight_loader
133 + })
134 + else:
135 + self.register_parameter("bias", None)
136 +
137 + def forward(self, input_):
138 + del input_
139 + raise RuntimeError("LMHead's weights should be used in the sampler.")
vllm/model_executor/model_loader.py
+4 -16
@@ -37,13 +37,6 @@ _MODEL_REGISTRY = {
37 "YiForCausalLM": YiForCausalLM,
38 }
39
40 -# FIXME(woosuk): Remove this once all models support quantization.
41 -_MODEL_CLASSES_SUPPORT_QUANTIZATION = [
42 - LlamaForCausalLM,
43 - MistralForCausalLM,
44 - YiForCausalLM,
45 -]
46 -
40
41 @contextlib.contextmanager
42 def _set_default_torch_dtype(dtype: torch.dtype):
@@ -67,12 +60,9 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
60 def get_model(model_config: ModelConfig) -> nn.Module:
61 model_class = _get_model_architecture(model_config.hf_config)
62
70 - # Get the quantization config.
71 - quant_config = None
63 + # Get the (maybe quantized) linear method.
64 + linear_method = None
65 if model_config.quantization is not None:
73 - if model_class not in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
74 - raise ValueError(
75 - f"Quantization is not supported for {model_class}.")
66 quant_config = get_quant_config(model_config.quantization,
67 model_config.model,
68 model_config.download_dir)
@@ -90,14 +80,12 @@ def get_model(model_config: ModelConfig) -> nn.Module:
80 f"{model_config.dtype} is not supported for quantization "
81 f"method {model_config.quantization}. Supported dtypes: "
82 f"{supported_dtypes}")
83 + linear_method = quant_config.get_linear_method()
84
85 with _set_default_torch_dtype(model_config.dtype):
86 # Create a model instance.
87 # The weights will be initialized as empty tensors.
97 - if model_class in _MODEL_CLASSES_SUPPORT_QUANTIZATION:
98 - model = model_class(model_config.hf_config, quant_config)
99 - else:
100 - model = model_class(model_config.hf_config)
88 + model = model_class(model_config.hf_config, linear_method)
89 if model_config.load_format == "dummy":
90 model = model.cuda()
91 # NOTE(woosuk): For accurate performance evaluation, we assign
vllm/model_executor/models/aquila.py
+64 -102
@@ -33,15 +33,17 @@ from torch import nn
33 from vllm.model_executor.input_metadata import InputMetadata
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
36 +from vllm.model_executor.layers.linear import (LinearMethodBase,
37 + MergedColumnParallelLinear,
38 + QKVParallelLinear,
39 + RowParallelLinear)
40 from vllm.model_executor.layers.sampler import Sampler
37 -from vllm.model_executor.weight_utils import (
38 - hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
39 - load_tensor_parallel_weights)
41 +from vllm.model_executor.layers.vocab_parallel_embedding import (
42 + VocabParallelEmbedding, ParallelLMHead)
43 from vllm.model_executor.parallel_utils.parallel_state import (
41 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43 - ColumnParallelLinear,
44 - RowParallelLinear)
44 + get_tensor_model_parallel_world_size)
45 +from vllm.model_executor.weight_utils import (default_weight_loader,
46 + hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
48 from vllm.transformers_utils.configs.aquila import AquilaConfig
49
@@ -55,20 +57,17 @@ class AquilaMLP(nn.Module):
57 hidden_size: int,
58 intermediate_size: int,
59 hidden_act: str,
60 + linear_method: Optional[LinearMethodBase] = None,
61 ):
62 super().__init__()
60 - self.gate_up_proj = ColumnParallelLinear(
61 - hidden_size,
62 - 2 * intermediate_size,
63 + self.gate_up_proj = MergedColumnParallelLinear(
64 + hidden_size, [intermediate_size] * 2,
65 bias=False,
64 - gather_output=False,
65 - )
66 - self.down_proj = RowParallelLinear(
67 - intermediate_size,
68 - hidden_size,
69 - bias=False,
70 - input_is_parallel=True,
71 - )
66 + linear_method=linear_method)
67 + self.down_proj = RowParallelLinear(intermediate_size,
68 + hidden_size,
69 + bias=False,
70 + linear_method=linear_method)
71 if hidden_act != "silu":
72 raise ValueError(f"Unsupported activation: {hidden_act}. "
73 "Only silu is supported for now.")
@@ -111,6 +110,7 @@ class AquilaAttention(nn.Module):
110 rope_theta: float = 10000,
111 max_position_embeddings: int = 8192,
112 rope_scaling: Optional[Dict[str, Any]] = None,
113 + linear_method: Optional[LinearMethodBase] = None,
114 ):
115 super().__init__()
116 self.hidden_size = hidden_size
@@ -128,29 +128,29 @@ class AquilaAttention(nn.Module):
128 self.rope_theta = rope_theta
129 self.max_position_embeddings = max_position_embeddings
130
131 - self.qkv_proj = ColumnParallelLinear(
131 + self.qkv_proj = QKVParallelLinear(
132 hidden_size,
133 - (self.total_num_heads + 2 * self.total_num_kv_heads) *
133 self.head_dim,
134 + self.total_num_heads,
135 + self.total_num_kv_heads,
136 bias=False,
136 - gather_output=False,
137 + linear_method=linear_method,
138 )
139 self.o_proj = RowParallelLinear(
140 self.total_num_heads * self.head_dim,
141 hidden_size,
142 bias=False,
142 - input_is_parallel=True,
143 + linear_method=linear_method,
144 )
145 self.attn = PagedAttentionWithRoPE(
146 self.num_heads,
147 self.head_dim,
148 self.scaling,
148 - rotary_dim=self.head_dim,
149 base=self.rope_theta,
150 max_position=self.max_position_embeddings,
151 + rotary_dim=self.head_dim,
152 num_kv_heads=self.num_kv_heads,
152 - rope_scaling=rope_scaling,
153 - )
153 + rope_scaling=rope_scaling)
154
155 def forward(
156 self,
@@ -171,7 +171,11 @@ class AquilaAttention(nn.Module):
171
172 class AquilaDecoderLayer(nn.Module):
173
174 - def __init__(self, config: AquilaConfig):
174 + def __init__(
175 + self,
176 + config: AquilaConfig,
177 + linear_method: Optional[LinearMethodBase] = None,
178 + ):
179 super().__init__()
180 self.hidden_size = config.hidden_size
181 rope_theta = getattr(config, "rope_theta", 10000)
@@ -185,11 +189,13 @@ class AquilaDecoderLayer(nn.Module):
189 rope_theta=rope_theta,
190 max_position_embeddings=max_position_embeddings,
191 rope_scaling=rope_scaling,
192 + linear_method=linear_method,
193 )
194 self.mlp = AquilaMLP(
195 hidden_size=self.hidden_size,
196 intermediate_size=config.intermediate_size,
197 hidden_act=config.hidden_act,
198 + linear_method=linear_method,
199 )
200 self.input_layernorm = AquilaRMSNorm(config.hidden_size,
201 eps=config.rms_norm_eps)
@@ -226,19 +232,22 @@ class AquilaDecoderLayer(nn.Module):
232
233 class AquilaModel(nn.Module):
234
229 - def __init__(self, config: AquilaConfig):
235 + def __init__(
236 + self,
237 + config: AquilaConfig,
238 + linear_method: Optional[LinearMethodBase] = None,
239 + ):
240 super().__init__()
241 self.config = config
242 self.padding_idx = config.pad_token_id
243 self.vocab_size = config.vocab_size
234 -
235 - #vocab_size = ((config.vocab_size + 63) // 64) * 64
244 self.embed_tokens = VocabParallelEmbedding(
245 config.vocab_size,
246 config.hidden_size,
247 )
248 self.layers = nn.ModuleList([
241 - AquilaDecoderLayer(config) for _ in range(config.num_hidden_layers)
249 + AquilaDecoderLayer(config, linear_method)
250 + for _ in range(config.num_hidden_layers)
251 ])
252 self.norm = AquilaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
253
@@ -271,17 +280,16 @@ class AquilaModel(nn.Module):
280
281 class AquilaForCausalLM(nn.Module):
282
274 - def __init__(self, config):
283 + def __init__(
284 + self,
285 + config,
286 + linear_method: Optional[LinearMethodBase] = None,
287 + ):
288 super().__init__()
289 self.config = config
277 - self.model = AquilaModel(config)
278 - vocab_size = ((config.vocab_size + 63) // 64) * 64
279 - self.lm_head = ColumnParallelLinear(
280 - config.hidden_size,
281 - vocab_size,
282 - bias=False,
283 - gather_output=False,
284 - )
290 + self.linear_method = linear_method
291 + self.model = AquilaModel(config, linear_method)
292 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
293 self.sampler = Sampler(config.vocab_size)
294
295 def forward(
@@ -298,79 +306,33 @@ class AquilaForCausalLM(nn.Module):
306 input_metadata)
307 return next_tokens
308
301 - _column_parallel_weights = [
302 - "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
303 - ]
304 - _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
305 -
309 def load_weights(self,
310 model_name_or_path: str,
311 cache_dir: Optional[str] = None,
312 load_format: str = "auto",
313 revision: Optional[str] = None):
311 - tp_size = get_tensor_model_parallel_world_size()
312 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
313 - q_proj_shard_size = (self.config.hidden_size // tp_size)
314 - kv_proj_shard_size = (self.config.hidden_size //
315 - self.config.num_attention_heads *
316 - self.config.num_key_value_heads // tp_size)
317 - attention_weight_specs = [
318 - # (weight_name, shard_size, offset)
319 - ("q_proj", q_proj_shard_size, 0),
320 - ("k_proj", kv_proj_shard_size, q_proj_shard_size),
321 - ("v_proj", kv_proj_shard_size,
322 - q_proj_shard_size + kv_proj_shard_size),
314 + stacked_params_mapping = [
315 + # (param_name, shard_name, shard_id)
316 + ("qkv_proj", "q_proj", "q"),
317 + ("qkv_proj", "k_proj", "k"),
318 + ("qkv_proj", "v_proj", "v"),
319 + ("gate_up_proj", "gate_proj", 0),
320 + ("gate_up_proj", "up_proj", 1),
321 ]
324 - state_dict = self.state_dict()
325 -
322 + params_dict = dict(self.named_parameters())
323 for name, loaded_weight in hf_model_weights_iterator(
324 model_name_or_path, cache_dir, load_format, revision):
325 if "rotary_emb.inv_freq" in name:
326 continue
330 -
331 - is_attention_weight = False
332 - for weight_name, shard_size, offset in attention_weight_specs:
327 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
328 if weight_name not in name:
329 continue
335 - param = state_dict[name.replace(weight_name, "qkv_proj")]
336 -
337 - loaded_weight = loaded_weight[
338 - shard_size * tensor_model_parallel_rank:shard_size *
339 - (tensor_model_parallel_rank + 1)]
340 - param_slice = param.data[offset:offset + shard_size]
341 - assert param_slice.shape == loaded_weight.shape
342 -
343 - param_slice.copy_(loaded_weight)
344 - is_attention_weight = True
330 + param = params_dict[name.replace(weight_name, param_name)]
331 + weight_loader = param.weight_loader
332 + weight_loader(param, loaded_weight, shard_id)
333 break
346 - if is_attention_weight:
347 - continue
348 -
349 - is_gate_up_weight = False
350 - for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
351 - if weight_name not in name:
352 - continue
353 - param = state_dict[name.replace(weight_name, "gate_up_proj")]
354 - shard_size = param.shape[0] // 2
355 - loaded_weight = loaded_weight[
356 - shard_size * tensor_model_parallel_rank:shard_size *
357 - (tensor_model_parallel_rank + 1)]
358 - param_slice = param.data[shard_size * stride_id:shard_size *
359 - (stride_id + 1)]
360 - assert param_slice.shape == loaded_weight.shape
361 - param_slice.copy_(loaded_weight)
362 - is_gate_up_weight = True
363 - break
364 - if is_gate_up_weight:
365 - continue
366 -
367 - param = state_dict[name]
368 - if "embed_tokens" in name or "lm_head" in name:
369 - load_padded_tensor_parallel_vocab(param, loaded_weight,
370 - tensor_model_parallel_rank)
371 - continue
372 -
373 - load_tensor_parallel_weights(param, loaded_weight, name,
374 - self._column_parallel_weights,
375 - self._row_parallel_weights,
376 - tensor_model_parallel_rank)
334 + else:
335 + param = params_dict[name]
336 + weight_loader = getattr(param, "weight_loader",
337 + default_weight_loader)
338 + weight_loader(param, loaded_weight)
vllm/model_executor/models/baichuan.py
+65 -89
@@ -30,18 +30,20 @@ from torch import nn
30
31 from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.activation import SiluAndMul
33 -from vllm.model_executor.layers.layernorm import RMSNorm
33 from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
34 PagedAttentionWithALiBi)
35 +from vllm.model_executor.layers.layernorm import RMSNorm
36 +from vllm.model_executor.layers.linear import (LinearMethodBase,
37 + MergedColumnParallelLinear,
38 + QKVParallelLinear,
39 + RowParallelLinear)
40 from vllm.model_executor.layers.sampler import Sampler
37 -from vllm.model_executor.weight_utils import (
38 - convert_pyslice_to_tensor, hf_model_weights_iterator,
39 - load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
41 +from vllm.model_executor.layers.vocab_parallel_embedding import (
42 + VocabParallelEmbedding, ParallelLMHead)
43 from vllm.model_executor.parallel_utils.parallel_state import (
44 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43 - ColumnParallelLinear,
44 - RowParallelLinear)
45 +from vllm.model_executor.weight_utils import (default_weight_loader,
46 + hf_model_weights_iterator)
47 from vllm.sequence import SamplerOutput
48 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
49
@@ -80,20 +82,17 @@ class BaiChuanMLP(nn.Module):
82 hidden_size: int,
83 intermediate_size: int,
84 hidden_act: str,
85 + linear_method: Optional[LinearMethodBase] = None,
86 ):
87 super().__init__()
85 - self.gate_up_proj = ColumnParallelLinear(
86 - hidden_size,
87 - 2 * intermediate_size,
88 - bias=False,
89 - gather_output=False,
90 - )
91 - self.down_proj = RowParallelLinear(
92 - intermediate_size,
93 - hidden_size,
88 + self.gate_up_proj = MergedColumnParallelLinear(
89 + hidden_size, [intermediate_size] * 2,
90 bias=False,
95 - input_is_parallel=True,
96 - )
91 + linear_method=linear_method)
92 + self.down_proj = RowParallelLinear(intermediate_size,
93 + hidden_size,
94 + bias=False,
95 + linear_method=linear_method)
96 if hidden_act != "silu":
97 raise ValueError(f"Unsupported activation: {hidden_act}. "
98 "Only silu is supported for now.")
@@ -116,6 +115,7 @@ class BaiChuanAttention(nn.Module):
115 position_embedding: str,
116 rope_theta: float = 10000,
117 max_position_embeddings: int = 8192,
118 + linear_method: Optional[LinearMethodBase] = None,
119 ):
120 super().__init__()
121 self.hidden_size = hidden_size
@@ -131,17 +131,19 @@ class BaiChuanAttention(nn.Module):
131 self.max_position_embeddings = max_position_embeddings
132
133 # pylint: disable=invalid-name
134 - self.W_pack = ColumnParallelLinear(
134 + self.W_pack = QKVParallelLinear(
135 hidden_size,
136 - 3 * hidden_size,
136 + self.head_dim,
137 + self.total_num_heads,
138 + self.total_num_heads,
139 bias=False,
138 - gather_output=False,
140 + linear_method=linear_method,
141 )
142 self.o_proj = RowParallelLinear(
143 self.total_num_heads * self.head_dim,
144 hidden_size,
145 bias=False,
144 - input_is_parallel=True,
146 + linear_method=linear_method,
147 )
148 # Create the alibi slopes and slice them.
149 if self.postion_embedding == "ALIBI":
@@ -188,7 +190,10 @@ class BaiChuanAttention(nn.Module):
190
191 class BaiChuanDecoderLayer(nn.Module):
192
191 - def __init__(self, config: BaiChuanConfig, position_embedding: str):
193 + def __init__(self,
194 + config: BaiChuanConfig,
195 + position_embedding: str,
196 + linear_method: Optional[LinearMethodBase] = None):
197 super().__init__()
198 self.hidden_size = config.hidden_size
199 rope_theta = getattr(config, "rope_theta", 10000)
@@ -200,11 +205,13 @@ class BaiChuanDecoderLayer(nn.Module):
205 position_embedding=position_embedding,
206 rope_theta=rope_theta,
207 max_position_embeddings=max_position_embeddings,
208 + linear_method=linear_method,
209 )
210 self.mlp = BaiChuanMLP(
211 hidden_size=self.hidden_size,
212 intermediate_size=config.intermediate_size,
213 hidden_act=config.hidden_act,
214 + linear_method=linear_method,
215 )
216 self.input_layernorm = RMSNorm(config.hidden_size,
217 eps=config.rms_norm_eps)
@@ -241,7 +248,10 @@ class BaiChuanDecoderLayer(nn.Module):
248
249 class BaiChuanModel(nn.Module):
250
244 - def __init__(self, config: BaiChuanConfig, position_embedding: str):
251 + def __init__(self,
252 + config: BaiChuanConfig,
253 + position_embedding: str,
254 + linear_method: Optional[LinearMethodBase] = None):
255 super().__init__()
256 self.config = config
257 self.padding_idx = config.pad_token_id
@@ -252,7 +262,7 @@ class BaiChuanModel(nn.Module):
262 config.hidden_size,
263 )
264 self.layers = nn.ModuleList([
255 - BaiChuanDecoderLayer(config, position_embedding)
265 + BaiChuanDecoderLayer(config, position_embedding, linear_method)
266 for _ in range(config.num_hidden_layers)
267 ])
268 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -285,16 +295,15 @@ class BaiChuanModel(nn.Module):
295
296 class BaiChuanBaseForCausalLM(nn.Module):
297
288 - def __init__(self, config, position_embedding: str):
298 + def __init__(self,
299 + config,
300 + position_embedding: str,
301 + linear_method: Optional[LinearMethodBase] = None):
302 super().__init__()
303 self.config = config
291 - self.model = BaiChuanModel(config, position_embedding)
292 - self.lm_head = ColumnParallelLinear(
293 - config.hidden_size,
294 - config.vocab_size,
295 - bias=False,
296 - gather_output=False,
297 - )
304 + self.linear_method = linear_method
305 + self.model = BaiChuanModel(config, position_embedding, linear_method)
306 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
307 self.sampler = Sampler(config.vocab_size)
308
309 def forward(
@@ -311,79 +320,46 @@ class BaiChuanBaseForCausalLM(nn.Module):
320 input_metadata)
321 return next_tokens
322
314 - _column_parallel_weights = []
315 - _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
316 -
323 def load_weights(self,
324 model_name_or_path: str,
325 cache_dir: Optional[str] = None,
326 load_format: str = "auto",
327 revision: Optional[str] = None):
322 - tp_world_size = get_tensor_model_parallel_world_size()
323 - tp_rank = get_tensor_model_parallel_rank()
324 - state_dict = self.state_dict()
325 -
328 + stacked_params_mapping = [
329 + # (param_name, shard_name, shard_id)
330 + ("gate_up_proj", "gate_proj", 0),
331 + ("gate_up_proj", "up_proj", 1),
332 + ]
333 + params_dict = dict(self.named_parameters())
334 for name, loaded_weight in hf_model_weights_iterator(
335 model_name_or_path, cache_dir, load_format, revision):
336 if "rotary_emb.inv_freq" in name:
337 continue
330 -
331 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
332 -
333 - if "W_pack" in name:
334 - total_num_heads = self.config.num_attention_heads
335 - hidden_size = self.config.hidden_size
336 - head_size = hidden_size // total_num_heads
337 - num_heads = total_num_heads // tp_world_size
338 - head_start = tp_rank * num_heads
339 - head_end = (tp_rank + 1) * num_heads
340 -
341 - loaded_weight = loaded_weight.view(3, total_num_heads,
342 - head_size, hidden_size)
343 - loaded_weight = loaded_weight[:, head_start:head_end, :, :]
344 - loaded_weight = loaded_weight.reshape(-1, hidden_size)
345 -
346 - is_gate_up_weight = False
347 - for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
338 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
339 if weight_name not in name:
340 continue
350 - param = state_dict[name.replace(weight_name, "gate_up_proj")]
351 - shard_size = param.shape[0] // 2
352 - loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
353 - (tp_rank + 1)]
354 - param_slice = param.data[shard_size * stride_id:shard_size *
355 - (stride_id + 1)]
356 - assert param_slice.shape == loaded_weight.shape
357 - param_slice.copy_(loaded_weight)
358 - is_gate_up_weight = True
341 + param = params_dict[name.replace(weight_name, param_name)]
342 + weight_loader = param.weight_loader
343 + weight_loader(param, loaded_weight, shard_id)
344 break
360 - if is_gate_up_weight:
361 - continue
362 -
363 - param = state_dict[name]
364 -
365 - if "embed_tokens" in name or "lm_head" in name:
366 - load_padded_tensor_parallel_vocab(param, loaded_weight,
367 - tp_rank)
368 - continue
369 -
370 - load_tensor_parallel_weights(
371 - param,
372 - loaded_weight,
373 - name,
374 - self._column_parallel_weights,
375 - self._row_parallel_weights,
376 - tp_rank,
377 - )
345 + else:
346 + param = params_dict[name]
347 + weight_loader = getattr(param, "weight_loader",
348 + default_weight_loader)
349 + weight_loader(param, loaded_weight)
350
351
352 class BaichuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 13b
353
382 - def __init__(self, config):
383 - super().__init__(config, "ALIBI")
354 + def __init__(self,
355 + config,
356 + linear_method: Optional[LinearMethodBase] = None):
357 + super().__init__(config, "ALIBI", linear_method)
358
359
360 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 7b
361
388 - def __init__(self, config):
389 - super().__init__(config, "ROPE")
362 + def __init__(self,
363 + config,
364 + linear_method: Optional[LinearMethodBase] = None):
365 + super().__init__(config, "ROPE", linear_method)
vllm/model_executor/models/bloom.py
+69 -63
@@ -30,14 +30,17 @@ from transformers import BloomConfig
30 from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import get_act_fn
32 from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
33 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 + LinearMethodBase,
35 + QKVParallelLinear,
36 + RowParallelLinear)
37 from vllm.model_executor.layers.sampler import Sampler
34 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35 - load_tensor_parallel_weights)
38 +from vllm.model_executor.layers.vocab_parallel_embedding import (
39 + VocabParallelEmbedding)
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39 - ColumnParallelLinear,
40 - RowParallelLinear)
42 +from vllm.model_executor.weight_utils import (default_weight_loader,
43 + hf_model_weights_iterator)
44 from vllm.sequence import SamplerOutput
45
46 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -70,7 +73,11 @@ def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
73
74 class BloomAttention(nn.Module):
75
73 - def __init__(self, config: BloomConfig):
76 + def __init__(
77 + self,
78 + config: BloomConfig,
79 + linear_method: Optional[LinearMethodBase] = None,
80 + ):
81 super().__init__()
82 self.hidden_size = config.hidden_size
83 self.total_num_heads = config.n_head
@@ -81,17 +88,18 @@ class BloomAttention(nn.Module):
88 assert self.total_num_heads % tp_world_size == 0
89 self.num_heads = self.total_num_heads // tp_world_size
90
84 - self.query_key_value = ColumnParallelLinear(
91 + self.query_key_value = QKVParallelLinear(
92 self.hidden_size,
86 - 3 * self.hidden_size,
93 + self.head_dim,
94 + self.total_num_heads,
95 bias=True,
88 - gather_output=False,
96 + linear_method=linear_method,
97 )
98 self.dense = RowParallelLinear(
99 self.hidden_size,
100 self.hidden_size,
101 bias=True,
94 - input_is_parallel=True,
102 + linear_method=linear_method,
103 )
104
105 # Create the alibi slopes and slice them.
@@ -125,19 +133,23 @@ class BloomAttention(nn.Module):
133
134 class BloomMLP(nn.Module):
135
128 - def __init__(self, config: BloomConfig):
136 + def __init__(
137 + self,
138 + config: BloomConfig,
139 + linear_method: Optional[LinearMethodBase] = None,
140 + ):
141 super().__init__()
142 hidden_size = config.hidden_size
143 self.dense_h_to_4h = ColumnParallelLinear(
144 hidden_size,
145 4 * hidden_size,
134 - gather_output=False,
146 + linear_method=linear_method,
147 )
148 self.act = get_act_fn("gelu")
149 self.dense_4h_to_h = RowParallelLinear(
150 4 * hidden_size,
151 hidden_size,
140 - input_is_parallel=True,
152 + linear_method=linear_method,
153 )
154
155 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -149,16 +161,20 @@ class BloomMLP(nn.Module):
161
162 class BloomBlock(nn.Module):
163
152 - def __init__(self, config: BloomConfig):
164 + def __init__(
165 + self,
166 + config: BloomConfig,
167 + linear_method: Optional[LinearMethodBase] = None,
168 + ):
169 super().__init__()
170 hidden_size = config.hidden_size
171
172 self.input_layernorm = nn.LayerNorm(hidden_size,
173 eps=config.layer_norm_epsilon)
158 - self.self_attention = BloomAttention(config)
174 + self.self_attention = BloomAttention(config, linear_method)
175 self.post_attention_layernorm = nn.LayerNorm(
176 hidden_size, eps=config.layer_norm_epsilon)
161 - self.mlp = BloomMLP(config)
177 + self.mlp = BloomMLP(config, linear_method)
178 self.apply_residual_connection_post_layernorm = (
179 config.apply_residual_connection_post_layernorm)
180
@@ -203,7 +219,11 @@ class BloomBlock(nn.Module):
219
220 class BloomModel(nn.Module):
221
206 - def __init__(self, config: BloomConfig):
222 + def __init__(
223 + self,
224 + config: BloomConfig,
225 + linear_method: Optional[LinearMethodBase] = None,
226 + ):
227 super().__init__()
228 self.embed_dim = config.hidden_size
229
@@ -216,8 +236,10 @@ class BloomModel(nn.Module):
236 self.embed_dim, eps=config.layer_norm_epsilon)
237
238 # Transformer blocks
219 - self.h = nn.ModuleList(
220 - [BloomBlock(config) for _ in range(config.num_hidden_layers)])
239 + self.h = nn.ModuleList([
240 + BloomBlock(config, linear_method)
241 + for _ in range(config.num_hidden_layers)
242 + ])
243
244 # Final Layer Norm
245 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -251,12 +273,15 @@ class BloomModel(nn.Module):
273
274 class BloomForCausalLM(nn.Module):
275
254 - def __init__(self, config: BloomConfig):
276 + def __init__(
277 + self,
278 + config: BloomConfig,
279 + linear_method: Optional[LinearMethodBase] = None,
280 + ):
281 super().__init__()
282 self.config = config
257 - self.transformer = BloomModel(config)
258 - # TODO(zhuohan): create a new weight after implementing pipeline
259 - # parallelism
283 + self.linear_method = linear_method
284 + self.transformer = BloomModel(config, linear_method)
285 self.lm_head_weight = self.transformer.word_embeddings.weight
286 self.sampler = Sampler(config.vocab_size)
287
@@ -274,55 +299,36 @@ class BloomForCausalLM(nn.Module):
299 input_metadata)
300 return next_tokens
301
277 - _column_parallel_weights = [
278 - "word_embeddings.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"
279 - ]
280 - _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
281 -
302 def load_weights(self,
303 model_name_or_path: str,
304 cache_dir: Optional[str] = None,
305 load_format: str = "auto",
306 revision: Optional[str] = None):
287 - tp_rank = get_tensor_model_parallel_rank()
288 - state_dict = self.state_dict()
307 + params_dict = dict(self.named_parameters(remove_duplicate=False))
308 for name, loaded_weight in hf_model_weights_iterator(
309 model_name_or_path, cache_dir, load_format, revision):
310 if name == "lm_head.weight":
292 - # Since hidden_states are parallelized, we need to
293 - # load lm_head.weight in parallel.
294 - self._column_parallel_weights.append(name)
295 - # If lm_head is provided, use it instead.
296 - param = self.lm_head_weight
297 - else:
298 - if not name.startswith("transformer."):
299 - name = "transformer." + name
300 - param = state_dict[name]
311 + continue
312 + if not name.startswith("transformer."):
313 + name = "transformer." + name
314 + param = params_dict[name]
315
316 if "query_key_value" in name:
303 - # NOTE(woosuk): BLOOM's fused QKV has the shape of
304 - # [num_heads * 3 * head_size, hidden_size], while the
305 - # required shape is [3 * num_heads * head_size, hidden_size].
317 + # NOTE: BLOOM's fused QKV's output_dim has the shape of
318 + # (num_heads * 3 * head_size), while the
319 + # required shape is (3 * num_heads * head_size).
320 # Thus, we need weight conversion.
307 - shard_size = param.shape[0]
308 - start = shard_size * tp_rank
309 - end = shard_size * (tp_rank + 1)
310 - loaded_weight = loaded_weight[start:end]
311 -
321 + output_dim = getattr(param, "output_dim", None)
322 num_heads = self.config.num_attention_heads
313 - hidden_size = self.config.hidden_size
314 - head_size = hidden_size // num_heads
315 - if "query_key_value.weight" in name:
316 - loaded_weight = loaded_weight.view(-1, 3, head_size,
317 - hidden_size)
318 - loaded_weight = loaded_weight.transpose(0, 1)
319 - loaded_weight = loaded_weight.reshape(-1, hidden_size)
320 - elif "query_key_value.bias" in name:
321 - loaded_weight = loaded_weight.view(-1, 3, head_size)
322 - loaded_weight = loaded_weight.transpose(0, 1)
323 - loaded_weight = loaded_weight.reshape(-1)
324 - else:
325 - raise ValueError(f"Unexpected weight name: {name}")
326 - load_tensor_parallel_weights(param, loaded_weight, name,
327 - self._column_parallel_weights,
328 - self._row_parallel_weights, tp_rank)
323 + if output_dim is not None:
324 + loaded_weight_shape = loaded_weight.shape
325 + loaded_weight = loaded_weight.view(
326 + loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
327 + loaded_weight_shape[output_dim + 1:])
328 + loaded_weight = loaded_weight.transpose(
329 + output_dim, output_dim + 1)
330 + loaded_weight = loaded_weight.reshape(loaded_weight_shape)
331 +
332 + weight_loader = getattr(param, "weight_loader",
333 + default_weight_loader)
334 + weight_loader(param, loaded_weight)
vllm/model_executor/models/chatglm.py
+79 -111
@@ -6,32 +6,28 @@
6 The input of the model is flattened to a 1D tensor of tokens. The model uses
7 InputMetadata to extract the original 2D shape of the input.
8 """
9 -from typing import Dict, List, Optional, Tuple
9 +from typing import List, Optional, Tuple
10
11 import torch
12 from torch import nn
13 from torch.nn import LayerNorm
14
15 from vllm.model_executor.input_metadata import InputMetadata
16 +from vllm.model_executor.layers.activation import SiluAndMul
17 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
18 from vllm.model_executor.layers.layernorm import RMSNorm
18 -from vllm.model_executor.layers.activation import SiluAndMul
19 +from vllm.model_executor.layers.linear import (LinearMethodBase,
20 + MergedColumnParallelLinear,
21 + QKVParallelLinear,
22 + RowParallelLinear)
23 from vllm.model_executor.layers.sampler import Sampler
20 -from vllm.model_executor.weight_utils import (
21 - hf_model_weights_iterator,
22 - load_tensor_parallel_weights,
23 -)
24 +from vllm.model_executor.layers.vocab_parallel_embedding import (
25 + VocabParallelEmbedding, ParallelLMHead)
26 from vllm.model_executor.parallel_utils.parallel_state import (
25 - get_tensor_model_parallel_rank,
26 - get_tensor_model_parallel_world_size,
27 -)
28 -from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
29 -from vllm.model_executor.parallel_utils.layers import (
30 - ColumnParallelLinear,
31 - RowParallelLinear,
32 -)
33 -from vllm.sequence import SequenceOutputs
34 -
27 + get_tensor_model_parallel_world_size)
28 +from vllm.model_executor.weight_utils import (default_weight_loader,
29 + hf_model_weights_iterator)
30 +from vllm.sequence import SamplerOutput
31 from vllm.transformers_utils.configs import ChatGLMConfig
32
33 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -39,7 +35,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
35
36 class GLMAttention(nn.Module):
37
42 - def __init__(self, config):
38 + def __init__(
39 + self,
40 + config,
41 + linear_method: Optional[LinearMethodBase] = None,
42 + ):
43 super().__init__()
44 self.hidden_size = config.hidden_size
45 tp_size = get_tensor_model_parallel_world_size()
@@ -50,25 +50,33 @@ class GLMAttention(nn.Module):
50 self.total_num_kv_heads = (config.multi_query_group_num
51 if config.multi_query_attention else
52 config.num_attention_heads)
53 - assert self.total_num_kv_heads % tp_size == 0
54 - self.num_kv_heads = self.total_num_kv_heads // tp_size
53 + if self.total_num_kv_heads >= tp_size:
54 + # Number of KV heads is greater than TP size, so we partition
55 + # the KV heads across multiple tensor parallel GPUs.
56 + assert self.total_num_kv_heads % tp_size == 0
57 + else:
58 + # Number of KV heads is less than TP size, so we replicate
59 + # the KV heads across multiple tensor parallel GPUs.
60 + assert tp_size % self.total_num_kv_heads == 0
61 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
62 self.head_dim = config.hidden_size // self.total_num_heads
63 self.q_size = self.num_heads * self.head_dim
64 self.kv_size = self.num_kv_heads * self.head_dim
65 self.scaling = self.head_dim**-0.5
66
60 - self.query_key_value = ColumnParallelLinear(
61 - config.hidden_size,
62 - (self.total_num_heads + 2 * self.total_num_kv_heads) *
67 + self.query_key_value = QKVParallelLinear(
68 + self.hidden_size,
69 self.head_dim,
64 - bias=config.add_qkv_bias,
65 - gather_output=False,
70 + self.total_num_heads,
71 + self.total_num_kv_heads,
72 + bias=config.add_bias_linear or config.add_qkv_bias,
73 + linear_method=linear_method,
74 )
75 self.dense = RowParallelLinear(
76 self.total_num_heads * self.head_dim,
77 config.hidden_size,
78 bias=config.add_bias_linear,
71 - input_is_parallel=True,
79 + linear_method=linear_method,
80 )
81
82 self.attn = PagedAttentionWithRoPE(
@@ -78,7 +86,6 @@ class GLMAttention(nn.Module):
86 rotary_dim=self.head_dim // 2,
87 num_kv_heads=self.num_kv_heads,
88 is_neox_style=False,
81 - # is_glm_style=True
89 )
90
91 def forward(
@@ -117,17 +124,21 @@ class GLMMLP(nn.Module):
124 state back into h hidden dimension.
125 """
126
120 - def __init__(self, config):
127 + def __init__(
128 + self,
129 + config,
130 + linear_method: Optional[LinearMethodBase] = None,
131 + ):
132 super().__init__()
133
134 self.add_bias = config.add_bias_linear
135
136 # Project to 4h.
126 - self.dense_h_to_4h = ColumnParallelLinear(
137 + self.dense_h_to_4h = MergedColumnParallelLinear(
138 config.hidden_size,
128 - config.ffn_hidden_size * 2,
139 + [config.ffn_hidden_size] * 2,
140 bias=config.add_bias_linear,
130 - gather_output=False,
141 + linear_method=linear_method,
142 )
143
144 self.activation_func = SiluAndMul()
@@ -137,7 +148,7 @@ class GLMMLP(nn.Module):
148 config.ffn_hidden_size,
149 config.hidden_size,
150 bias=config.add_bias_linear,
140 - input_is_parallel=True,
151 + linear_method=linear_method,
152 )
153
154 def forward(self, hidden_states):
@@ -159,6 +170,7 @@ class GLMBlock(nn.Module):
170 def __init__(
171 self,
172 config,
173 + linear_method: Optional[LinearMethodBase] = None,
174 ):
175 super().__init__()
176 self.apply_residual_connection_post_layernorm = (
@@ -172,7 +184,7 @@ class GLMBlock(nn.Module):
184 eps=config.layernorm_epsilon)
185
186 # Self attention.
175 - self.self_attention = GLMAttention(config)
187 + self.self_attention = GLMAttention(config, linear_method)
188 self.hidden_dropout = config.hidden_dropout
189
190 # Layernorm on the attention output
@@ -180,7 +192,7 @@ class GLMBlock(nn.Module):
192 config.hidden_size, eps=config.layernorm_epsilon)
193
194 # MLP
183 - self.mlp = GLMMLP(config)
195 + self.mlp = GLMMLP(config, linear_method)
196
197 def forward(
198 self,
@@ -227,7 +239,11 @@ class GLMBlock(nn.Module):
239 class GLMTransformer(nn.Module):
240 """Transformer class."""
241
230 - def __init__(self, config):
242 + def __init__(
243 + self,
244 + config,
245 + linear_method: Optional[LinearMethodBase] = None,
246 + ):
247 super().__init__()
248 self.post_layer_norm = config.post_layer_norm
249
@@ -236,7 +252,7 @@ class GLMTransformer(nn.Module):
252
253 # Transformer layers.
254 self.layers = nn.ModuleList(
239 - [GLMBlock(config) for i in range(self.num_layers)])
255 + [GLMBlock(config, linear_method) for i in range(self.num_layers)])
256
257 if self.post_layer_norm:
258 layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -274,7 +290,11 @@ class GLMTransformer(nn.Module):
290
291 class ChatGLMModel(nn.Module):
292
277 - def __init__(self, config):
293 + def __init__(
294 + self,
295 + config,
296 + linear_method: Optional[LinearMethodBase] = None,
297 + ):
298 super().__init__()
299
300 self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
@@ -283,15 +303,10 @@ class ChatGLMModel(nn.Module):
303 self.num_layers = config.num_layers
304 self.multi_query_group_num = config.multi_query_group_num
305 self.kv_channels = config.kv_channels
286 - self.encoder = GLMTransformer(config)
306 + self.encoder = GLMTransformer(config, linear_method)
307
288 - self.output_layer = ColumnParallelLinear(
289 - config.hidden_size,
290 - config.padded_vocab_size,
291 - bias=False,
292 - gather_output=False,
293 - params_dtype=config.torch_dtype,
294 - )
308 + self.output_layer = ParallelLMHead(config.padded_vocab_size,
309 + config.hidden_size)
310
311 def forward(
312 self,
@@ -317,10 +332,15 @@ class ChatGLMModel(nn.Module):
332
333 class ChatGLMForCausalLM(nn.Module):
334
320 - def __init__(self, config: ChatGLMConfig):
335 + def __init__(
336 + self,
337 + config: ChatGLMConfig,
338 + linear_method: Optional[LinearMethodBase] = None,
339 + ):
340 super().__init__()
341 self.config: ChatGLMConfig = config
323 - self.transformer = ChatGLMModel(config)
342 + self.linear_method = linear_method
343 + self.transformer = ChatGLMModel(config, linear_method)
344 self.lm_head_weight = self.transformer.output_layer.weight
345 self.sampler = Sampler(config.padded_vocab_size)
346
@@ -331,78 +351,26 @@ class ChatGLMForCausalLM(nn.Module):
351 kv_caches: List[KVCache],
352 input_metadata: InputMetadata,
353 cache_events: Optional[List[torch.cuda.Event]],
334 - ) -> Dict[int, SequenceOutputs]:
354 + ) -> SamplerOutput:
355 hidden_states = self.transformer(input_ids, positions, kv_caches,
356 input_metadata, cache_events)
357 next_tokens = self.sampler(self.lm_head_weight, hidden_states,
358 input_metadata)
359 return next_tokens
360
341 - _column_parallel_weights = [
342 - "output_layer.weight",
343 - "embedding.weight",
344 - ]
345 - _row_parallel_weights = ["dense_4h_to_h", "self_attention.dense"]
346 -
347 - def load_weights(
348 - self,
349 - model_name_or_path: str,
350 - cache_dir: Optional[str] = None,
351 - load_format: str = "auto",
352 - revision: Optional[str] = None,
353 - ):
354 - tp_rank = get_tensor_model_parallel_rank()
355 - tp_size = get_tensor_model_parallel_world_size()
356 -
357 - q_proj_shard_size = self.config.hidden_size // tp_size
358 - kv_proj_shard_size = (self.config.hidden_size //
359 - self.config.num_attention_heads *
360 - self.config.multi_query_group_num // tp_size)
361 -
362 - mlp_hidden_shard_size = self.config.ffn_hidden_size // tp_size
363 -
364 - state_dict = self.state_dict()
361 + def load_weights(self,
362 + model_name_or_path: str,
363 + cache_dir: Optional[str] = None,
364 + load_format: str = "auto",
365 + revision: Optional[str] = None):
366 + params_dict = dict(self.named_parameters(remove_duplicate=False))
367 for name, loaded_weight in hf_model_weights_iterator(
368 model_name_or_path, cache_dir, load_format, revision):
369 + if "rotary_pos_emb.inv_freq" in name:
370 + continue
371 if "word_embeddings" in name:
372 name = name.replace(".word_embeddings", "")
369 -
370 - if name in state_dict:
371 - param = state_dict[name]
372 - if "query_key_value" in name:
373 - q_offset = q_proj_shard_size * tp_rank
374 - k_offset = (q_proj_shard_size * tp_size +
375 - kv_proj_shard_size * tp_rank)
376 - v_offset = (q_proj_shard_size * tp_size +
377 - kv_proj_shard_size * (tp_size + tp_rank))
378 - wq = loaded_weight[q_offset:q_offset + q_proj_shard_size]
379 - wk = loaded_weight[k_offset:k_offset + kv_proj_shard_size]
380 - wv = loaded_weight[v_offset:v_offset + kv_proj_shard_size]
381 - loaded_weight = torch.cat([wq, wk, wv], dim=0)
382 - param.data.copy_(loaded_weight)
383 - continue
384 -
385 - if "dense_h_to_4h" in name:
386 - w_gate = loaded_weight[mlp_hidden_shard_size *
387 - tp_rank:mlp_hidden_shard_size *
388 - (tp_rank + 1)]
389 - w_proj = loaded_weight[mlp_hidden_shard_size *
390 - (tp_size +
391 - tp_rank):mlp_hidden_shard_size *
392 - (tp_size + tp_rank + 1)]
393 - loaded_weight = torch.cat([w_gate, w_proj], dim=0)
394 - param.data.copy_(loaded_weight)
395 - continue
396 -
397 - load_tensor_parallel_weights(
398 - param,
399 - loaded_weight,
400 - name,
401 - self._column_parallel_weights,
402 - self._row_parallel_weights,
403 - tp_rank,
404 - )
405 - elif name == "transformer.rotary_pos_emb.inv_freq":
406 - continue
407 - else:
408 - print("Warning never found tensor's name:", name)
373 + param = params_dict[name]
374 + weight_loader = getattr(param, "weight_loader",
375 + default_weight_loader)
376 + weight_loader(param, loaded_weight)
vllm/model_executor/models/falcon.py
+93 -150
@@ -30,17 +30,19 @@ from vllm.model_executor.input_metadata import InputMetadata
30 from vllm.model_executor.layers.attention import (PagedAttention,
31 PagedAttentionWithALiBi,
32 PagedAttentionWithRoPE)
33 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 + LinearMethodBase,
35 + QKVParallelLinear,
36 + RowParallelLinear)
37 from vllm.model_executor.layers.sampler import Sampler
34 -from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
35 - hf_model_weights_iterator,
36 - load_tensor_parallel_weights)
37 -from vllm.model_executor.parallel_utils.parallel_state import (
38 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40 - ColumnParallelLinear,
41 - RowParallelLinear)
38 +from vllm.model_executor.layers.vocab_parallel_embedding import (
39 + VocabParallelEmbedding, ParallelLMHead)
40 from vllm.model_executor.parallel_utils.communication_op import (
41 tensor_model_parallel_all_reduce)
42 +from vllm.model_executor.parallel_utils.parallel_state import (
43 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.weight_utils import (default_weight_loader,
45 + hf_model_weights_iterator)
46 from vllm.sequence import SamplerOutput
47 from vllm.transformers_utils.configs import RWConfig
48
@@ -48,19 +50,6 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
50 FalconConfig = Union[HF_FalconConfig, RWConfig]
51
52
51 -# NOTE(Hesslow): Unfortunately we did not fuse matmul and bias during
52 -# training, this means that there's one additional quantization to bfloat16
53 -# between the operations. In order not to degrade the quality of our HF-port,
54 -# we keep these characteristics in the final model.
55 -class FalconLinear(nn.Linear):
56 -
57 - def forward(self, x: torch.Tensor) -> torch.Tensor:
58 - hidden_states = x @ self.weight.T
59 - if self.bias is None:
60 - return hidden_states
61 - return hidden_states + self.bias
62 -
63 -
53 def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
54 closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
55 base = torch.tensor(2**(-(2**-(math.log2(closest_power_of_2) - 3))),
@@ -86,7 +75,11 @@ def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
75
76 class FalconAttention(nn.Module):
77
89 - def __init__(self, config: FalconConfig):
78 + def __init__(
79 + self,
80 + config: FalconConfig,
81 + linear_method: Optional[LinearMethodBase] = None,
82 + ):
83 super().__init__()
84
85 self.hidden_size = config.hidden_size
@@ -103,41 +96,29 @@ class FalconAttention(nn.Module):
96
97 if self.new_decoder_architecture:
98 self.total_num_kv_heads = config.num_kv_heads
106 - assert self.total_num_heads % tp_size == 0
107 - self.num_kv_heads = self.total_num_kv_heads // tp_size
108 - self.query_key_value = ColumnParallelLinear(
109 - self.hidden_size,
110 - (self.total_num_heads + 2 * self.total_num_kv_heads) *
111 - self.head_dim,
112 - bias=config.bias,
113 - gather_output=False,
114 - skip_bias_add=True,
115 - )
99 elif self.multi_query:
100 self.total_num_kv_heads = 1
118 - self.num_kv_heads = 1
119 - self.query = ColumnParallelLinear(
120 - self.hidden_size,
121 - self.total_num_heads * self.head_dim,
122 - bias=config.bias,
123 - gather_output=False,
124 - skip_bias_add=True,
125 - )
126 - self.key_value = FalconLinear(self.hidden_size,
127 - 2 * self.head_dim,
128 - bias=config.bias)
101 else:
102 self.total_num_kv_heads = self.total_num_heads
131 - self.num_kv_heads = self.num_heads
132 - self.query_key_value = ColumnParallelLinear(
133 - self.hidden_size,
134 - (self.total_num_heads + 2 * self.total_num_kv_heads) *
135 - self.head_dim,
136 - bias=config.bias,
137 - gather_output=False,
138 - skip_bias_add=True,
139 - )
103 + if self.total_num_kv_heads >= tp_size:
104 + # Number of KV heads is greater than TP size, so we partition
105 + # the KV heads across multiple tensor parallel GPUs.
106 + assert self.total_num_kv_heads % tp_size == 0
107 + else:
108 + # Number of KV heads is less than TP size, so we replicate
109 + # the KV heads across multiple tensor parallel GPUs.
110 + assert tp_size % self.total_num_kv_heads == 0
111 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112
113 + self.query_key_value = QKVParallelLinear(
114 + self.hidden_size,
115 + self.head_dim,
116 + self.total_num_heads,
117 + self.total_num_kv_heads,
118 + bias=config.bias,
119 + skip_bias_add=True,
120 + linear_method=linear_method,
121 + )
122 self.q_size = self.num_heads * self.head_dim
123 self.kv_size = self.num_kv_heads * self.head_dim
124
@@ -149,7 +130,6 @@ class FalconAttention(nn.Module):
130 self.hidden_size,
131 self.hidden_size,
132 bias=config.bias,
152 - input_is_parallel=True,
133 skip_bias_add=True,
134 reduce_results=self.reduce_row_parallel_results)
135
@@ -196,18 +176,10 @@ class FalconAttention(nn.Module):
176 input_metadata: InputMetadata,
177 cache_event: Optional[torch.cuda.Event],
178 ) -> torch.Tensor:
199 - if not self.new_decoder_architecture and self.multi_query:
200 - q, bias = self.query(hidden_states)
201 - if bias is not None:
202 - q += bias
203 - kv = self.key_value(hidden_states)
204 - k, v = kv.split([self.kv_size, self.kv_size], dim=-1)
205 - else:
206 - qkv, bias = self.query_key_value(hidden_states)
207 - if bias is not None:
208 - qkv += bias
209 - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size],
210 - dim=-1)
179 + qkv, bias = self.query_key_value(hidden_states)
180 + if bias is not None:
181 + qkv += bias
182 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
183 k_cache, v_cache = kv_cache
184 if self.use_rotary:
185 attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
@@ -221,15 +193,19 @@ class FalconAttention(nn.Module):
193
194 class FalconMLP(nn.Module):
195
224 - def __init__(self, config: FalconConfig):
196 + def __init__(
197 + self,
198 + config: FalconConfig,
199 + linear_method: Optional[LinearMethodBase] = None,
200 + ):
201 super().__init__()
202 hidden_size = config.hidden_size
203
204 self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
205 4 * hidden_size,
206 bias=config.bias,
231 - gather_output=False,
232 - skip_bias_add=True)
207 + skip_bias_add=True,
208 + linear_method=linear_method)
209 self.act = nn.GELU()
210 self.reduce_row_parallel_results = not (config.new_decoder_architecture
211 or config.parallel_attn)
@@ -237,9 +213,9 @@ class FalconMLP(nn.Module):
213 4 * hidden_size,
214 hidden_size,
215 bias=config.bias,
240 - input_is_parallel=True,
216 skip_bias_add=True,
242 - reduce_results=self.reduce_row_parallel_results)
217 + reduce_results=self.reduce_row_parallel_results,
218 + linear_method=linear_method)
219
220 def forward(self, x: torch.Tensor) -> torch.Tensor:
221 # NOTE(zhuohan): Following huggingface, we do not fuse bias add here.
@@ -253,12 +229,16 @@ class FalconMLP(nn.Module):
229
230 class FalconDecoderLayer(nn.Module):
231
256 - def __init__(self, config: FalconConfig):
232 + def __init__(
233 + self,
234 + config: FalconConfig,
235 + linear_method: Optional[LinearMethodBase] = None,
236 + ):
237 super().__init__()
238 hidden_size = config.hidden_size
239 self.num_heads = config.num_attention_heads
260 - self.self_attention = FalconAttention(config)
261 - self.mlp = FalconMLP(config)
240 + self.self_attention = FalconAttention(config, linear_method)
241 + self.mlp = FalconMLP(config, linear_method)
242 self.config = config
243
244 if config.new_decoder_architecture:
@@ -334,7 +314,11 @@ class FalconDecoderLayer(nn.Module):
314
315 class FalconModel(nn.Module):
316
337 - def __init__(self, config: FalconConfig):
317 + def __init__(
318 + self,
319 + config: FalconConfig,
320 + linear_method: Optional[LinearMethodBase] = None,
321 + ):
322 super().__init__()
323 self.config = config
324 self.embed_dim = config.hidden_size
@@ -349,7 +333,8 @@ class FalconModel(nn.Module):
333
334 # Transformer blocks
335 self.h = nn.ModuleList([
352 - FalconDecoderLayer(config) for _ in range(config.num_hidden_layers)
336 + FalconDecoderLayer(config, linear_method)
337 + for _ in range(config.num_hidden_layers)
338 ])
339
340 # Final Layer Norm
@@ -383,15 +368,18 @@ class FalconModel(nn.Module):
368
369 class FalconForCausalLM(nn.Module):
370
386 - def __init__(self, config: FalconConfig):
371 + def __init__(
372 + self,
373 + config: FalconConfig,
374 + linear_method: Optional[LinearMethodBase] = None,
375 + ):
376 super().__init__()
377 self.config = config
389 - self.transformer = FalconModel(config)
390 - self.lm_head = ColumnParallelLinear(
391 - config.hidden_size,
378 + self.linear_method = linear_method
379 + self.transformer = FalconModel(config, linear_method)
380 + self.lm_head = ParallelLMHead(
381 config.vocab_size,
393 - bias=False,
394 - gather_output=False,
382 + config.hidden_size,
383 )
384 self.sampler = Sampler(config.vocab_size)
385
@@ -415,89 +403,44 @@ class FalconForCausalLM(nn.Module):
403
404 return next_tokens
405
418 - _column_parallel_weights = [
419 - "word_embeddings.weight", "lm_head.weight", "dense_h_to_4h.weight",
420 - "dense_h_to_4h.bias"
421 - ]
422 - _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
423 -
406 def load_weights(self,
407 model_name_or_path: str,
408 cache_dir: Optional[str] = None,
409 load_format: str = "auto",
410 revision: Optional[str] = None):
429 - tp_size = (get_tensor_model_parallel_world_size())
430 - tp_rank = get_tensor_model_parallel_rank()
431 -
432 - hidden_size = self.config.hidden_size
411 total_num_heads = self.config.num_attention_heads
434 - num_heads = total_num_heads // tp_size
435 - head_size = hidden_size // total_num_heads
436 - head_start = tp_rank * num_heads
437 - head_end = (tp_rank + 1) * num_heads
412 if self.config.new_decoder_architecture:
413 total_num_kv_heads = self.config.num_kv_heads
440 - num_kv_heads = total_num_kv_heads // tp_size
441 - separated_q_kv = False
442 - kv_head_start = tp_rank * num_kv_heads
443 - kv_head_end = (tp_rank + 1) * num_kv_heads
414 elif self.config.multi_query:
415 total_num_kv_heads = 1
446 - num_kv_heads = 1
447 - separated_q_kv = True
448 - kv_head_start = 0
449 - kv_head_end = 1
416 else:
417 total_num_kv_heads = total_num_heads
452 - num_kv_heads = total_num_kv_heads // tp_size
453 - separated_q_kv = False
454 - kv_head_start = tp_rank * num_kv_heads
455 - kv_head_end = (tp_rank + 1) * num_kv_heads
418 num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
457 - state_dict = self.state_dict()
458 -
419 + params_dict = dict(self.named_parameters())
420 for name, loaded_weight in hf_model_weights_iterator(
421 model_name_or_path, cache_dir, load_format, revision):
422 + param = params_dict[name]
423 if "query_key_value" in name:
462 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
463 - loaded_weight_size = loaded_weight.size()
424 + output_dim = getattr(param, "output_dim", None)
425 + loaded_weight_shape = loaded_weight.shape
426 loaded_weight = loaded_weight.view(
465 - total_num_kv_heads, num_query_heads_per_kv_head + 2,
466 - head_size, *loaded_weight_size[1:])
467 -
468 - wq = loaded_weight[:, :-2].reshape(-1, *loaded_weight_size[1:])
469 - wk = loaded_weight[:, [-2]].reshape(-1,
470 - *loaded_weight_size[1:])
471 - wv = loaded_weight[:, [-1]].reshape(-1,
472 - *loaded_weight_size[1:])
473 -
474 - wq = wq[head_size * head_start:head_size * head_end]
475 - wk = wk[head_size * kv_head_start:head_size * kv_head_end]
476 - wv = wv[head_size * kv_head_start:head_size * kv_head_end]
477 -
478 - if separated_q_kv:
479 - loaded_weight_q = wq
480 - loaded_weight_kv = torch.cat([wk, wv], dim=0)
481 - q_weight_name = name.replace("query_key_value", "query")
482 - kv_weight_name = name.replace("query_key_value",
483 - "key_value")
484 - load_tensor_parallel_weights(state_dict[q_weight_name],
485 - loaded_weight_q,
486 - q_weight_name,
487 - self._column_parallel_weights,
488 - self._row_parallel_weights,
489 - tp_rank)
490 - load_tensor_parallel_weights(state_dict[kv_weight_name],
491 - loaded_weight_kv,
492 - kv_weight_name,
493 - self._column_parallel_weights,
494 - self._row_parallel_weights,
495 - tp_rank)
496 - continue
497 - else:
498 - loaded_weight = torch.cat([wq, wk, wv], dim=0)
499 -
500 - param = state_dict[name]
501 - load_tensor_parallel_weights(param, loaded_weight, name,
502 - self._column_parallel_weights,
503 - self._row_parallel_weights, tp_rank)
427 + loaded_weight_shape[:output_dim] +
428 + (total_num_kv_heads, num_query_heads_per_kv_head + 2, -1) +
429 + loaded_weight_shape[output_dim + 1:])
430 + wq = loaded_weight.narrow(
431 + output_dim + 1, 0, num_query_heads_per_kv_head).reshape(
432 + *loaded_weight_shape[:output_dim], -1,
433 + *loaded_weight_shape[output_dim + 1:])
434 + wk = loaded_weight.narrow(
435 + output_dim + 1, num_query_heads_per_kv_head,
436 + 1).reshape(*loaded_weight_shape[:output_dim], -1,
437 + *loaded_weight_shape[output_dim + 1:])
438 + wv = loaded_weight.narrow(
439 + output_dim + 1, num_query_heads_per_kv_head + 1,
440 + 1).reshape(*loaded_weight_shape[:output_dim], -1,
441 + *loaded_weight_shape[output_dim + 1:])
442 + loaded_weight = torch.cat([wq, wk, wv], dim=output_dim)
443 +
444 + weight_loader = getattr(param, "weight_loader",
445 + default_weight_loader)
446 + weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt2.py
+52 -79
@@ -30,15 +30,17 @@ from transformers import GPT2Config
30 from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import get_act_fn
32 from vllm.model_executor.layers.attention import PagedAttention
33 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 + LinearMethodBase,
35 + QKVParallelLinear,
36 + RowParallelLinear)
37 from vllm.model_executor.layers.sampler import Sampler
34 -from vllm.model_executor.weight_utils import (
35 - convert_pyslice_to_tensor, hf_model_weights_iterator,
36 - load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
38 +from vllm.model_executor.layers.vocab_parallel_embedding import (
39 + VocabParallelEmbedding)
40 from vllm.model_executor.parallel_utils.parallel_state import (
38 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40 - ColumnParallelLinear,
41 - RowParallelLinear)
41 + get_tensor_model_parallel_world_size)
42 +from vllm.model_executor.weight_utils import (default_weight_loader,
43 + hf_model_weights_iterator)
44 from vllm.sequence import SamplerOutput
45
46 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -46,7 +48,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
48
49 class GPT2Attention(nn.Module):
50
49 - def __init__(self, config: GPT2Config):
51 + def __init__(
52 + self,
53 + config: GPT2Config,
54 + linear_method: Optional[LinearMethodBase] = None,
55 + ):
56 super().__init__()
57 self.hidden_size = config.hidden_size
58 total_num_heads = config.num_attention_heads
@@ -57,17 +63,18 @@ class GPT2Attention(nn.Module):
63 self.head_dim = self.hidden_size // total_num_heads
64 self.scale = self.head_dim**-0.5
65
60 - self.c_attn = ColumnParallelLinear(
66 + self.c_attn = QKVParallelLinear(
67 self.hidden_size,
62 - 3 * self.hidden_size,
68 + self.head_dim,
69 + total_num_heads,
70 bias=True,
64 - gather_output=False,
71 + linear_method=linear_method,
72 )
73 self.c_proj = RowParallelLinear(
74 self.hidden_size,
75 self.hidden_size,
76 bias=True,
70 - input_is_parallel=True,
77 + linear_method=linear_method,
78 )
79 self.attn = PagedAttention(self.num_heads,
80 self.head_dim,
@@ -95,6 +102,7 @@ class GPT2MLP(nn.Module):
102 self,
103 intermediate_size: int,
104 config: GPT2Config,
105 + linear_method: Optional[LinearMethodBase] = None,
106 ):
107 super().__init__()
108 hidden_size = config.hidden_size
@@ -102,13 +110,13 @@ class GPT2MLP(nn.Module):
110 hidden_size,
111 intermediate_size,
112 bias=True,
105 - gather_output=False,
113 + linear_method=linear_method,
114 )
115 self.c_proj = RowParallelLinear(
116 intermediate_size,
117 hidden_size,
118 bias=True,
111 - input_is_parallel=True,
119 + linear_method=linear_method,
120 )
121 self.act = get_act_fn(config.activation_function)
122
@@ -121,16 +129,20 @@ class GPT2MLP(nn.Module):
129
130 class GPT2Block(nn.Module):
131
124 - def __init__(self, config: GPT2Config):
132 + def __init__(
133 + self,
134 + config: GPT2Config,
135 + linear_method: Optional[LinearMethodBase] = None,
136 + ):
137 super().__init__()
138 hidden_size = config.hidden_size
139 inner_dim = (config.n_inner if config.n_inner is not None else 4 *
140 hidden_size)
141
142 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
131 - self.attn = GPT2Attention(config)
143 + self.attn = GPT2Attention(config, linear_method)
144 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
133 - self.mlp = GPT2MLP(inner_dim, config)
145 + self.mlp = GPT2MLP(inner_dim, config, linear_method)
146
147 def forward(
148 self,
@@ -160,24 +172,23 @@ class GPT2Block(nn.Module):
172
173 class GPT2Model(nn.Module):
174
163 - def __init__(self, config: GPT2Config):
175 + def __init__(
176 + self,
177 + config: GPT2Config,
178 + linear_method: Optional[LinearMethodBase] = None,
179 + ):
180 super().__init__()
181 self.config = config
182 assert not config.add_cross_attention
183 assert not config.scale_attn_by_inverse_layer_idx
184 assert not config.reorder_and_upcast_attn
185 self.embed_dim = config.hidden_size
170 -
171 - # Optimization: While the vocab size of GPT-2 is 50257, we extend it
172 - # to 50304 in order to make it divisible by 64.
173 - # This improves performance since GPUs are faster if the dimension
174 - # is divisible by 64. In addition, it allows us to shard the embedding
175 - # layer across 2, 4, 8, or more GPUs.
176 - vocab_size = ((config.vocab_size + 63) // 64) * 64
177 - self.wte = VocabParallelEmbedding(vocab_size, self.embed_dim)
186 + self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
187 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
179 - self.h = nn.ModuleList(
180 - [GPT2Block(config) for _ in range(config.num_hidden_layers)])
188 + self.h = nn.ModuleList([
189 + GPT2Block(config, linear_method)
190 + for _ in range(config.num_hidden_layers)
191 + ])
192 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
193
194 def forward(
@@ -207,12 +218,15 @@ class GPT2Model(nn.Module):
218
219 class GPT2LMHeadModel(nn.Module):
220
210 - def __init__(self, config: GPT2Config):
221 + def __init__(
222 + self,
223 + config: GPT2Config,
224 + linear_method: Optional[LinearMethodBase] = None,
225 + ):
226 super().__init__()
227 self.config = config
213 - self.transformer = GPT2Model(config)
214 - # TODO(zhuohan): create a new weight after implementing pipeline
215 - # parallelism
228 + self.linear_method = linear_method
229 + self.transformer = GPT2Model(config, linear_method)
230 self.lm_head_weight = self.transformer.wte.weight
231 self.sampler = Sampler(config.vocab_size)
232
@@ -230,19 +244,12 @@ class GPT2LMHeadModel(nn.Module):
244 input_metadata)
245 return next_tokens
246
233 - _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
234 - _row_parallel_weights = ["c_proj.weight"]
235 -
247 def load_weights(self,
248 model_name_or_path: str,
249 cache_dir: Optional[str] = None,
250 load_format: str = "auto",
251 revision: Optional[str] = None):
241 - tensor_model_parallel_world_size = (
242 - get_tensor_model_parallel_world_size())
243 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
244 - state_dict = self.state_dict()
245 -
252 + params_dict = dict(self.named_parameters(remove_duplicate=False))
253 for name, loaded_weight in hf_model_weights_iterator(
254 model_name_or_path, cache_dir, load_format, revision):
255 if "lm_head.weight" in name:
@@ -253,53 +260,19 @@ class GPT2LMHeadModel(nn.Module):
260 # Skip attention mask.
261 # NOTE: "c_attn.bias" should not be skipped.
262 continue
256 -
263 if not name.startswith("transformer."):
264 name = "transformer." + name
259 -
260 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
261 -
265 + param = params_dict[name]
266 # The HF's GPT-2 implementation uses Conv1D instead of Linear.
267 # Because of this, we need to transpose the weights.
268 + # Note(zhuohan): the logic below might break quantized models.
269 for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
270 if conv1d_weight_name not in name:
271 continue
272 if not name.endswith(".weight"):
273 continue
274 loaded_weight = loaded_weight.t()
270 - param = state_dict[name]
271 -
272 - if name == "transformer.wte.weight":
273 - load_padded_tensor_parallel_vocab(param, loaded_weight,
274 - tensor_model_parallel_rank)
275 - continue
276 -
277 - # For the fused QKV linear layer, manually shard the weights.
278 - if "c_attn" in name:
279 - # GPT-2's fused QKV has the shape of
280 - # [3 * num_heads * head_size, hidden_size].
281 - # When tensor parallelism is used, we shard the weights along
282 - # the head dimension.
283 - total_num_heads = self.config.num_attention_heads
284 - hidden_size = self.config.hidden_size
285 - head_size = hidden_size // total_num_heads
286 - num_heads = total_num_heads // tensor_model_parallel_world_size
287 - head_start = tensor_model_parallel_rank * num_heads
288 - head_end = (tensor_model_parallel_rank + 1) * num_heads
275
290 - if name.endswith(".weight"):
291 - loaded_weight = loaded_weight.view(3, total_num_heads,
292 - head_size, hidden_size)
293 - loaded_weight = loaded_weight[:, head_start:head_end, :, :]
294 - loaded_weight = loaded_weight.reshape(-1, hidden_size)
295 - elif name.endswith(".bias"):
296 - loaded_weight = loaded_weight.view(3, total_num_heads,
297 - head_size)
298 - loaded_weight = loaded_weight[:, head_start:head_end, :]
299 - loaded_weight = loaded_weight.reshape(-1)
300 - else:
301 - raise ValueError(f"Unexpected parameter name {name}")
302 - load_tensor_parallel_weights(param, loaded_weight, name,
303 - self._column_parallel_weights,
304 - self._row_parallel_weights,
305 - tensor_model_parallel_rank)
276 + weight_loader = getattr(param, "weight_loader",
277 + default_weight_loader)
278 + weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_bigcode.py
+63 -126
@@ -31,15 +31,17 @@ from transformers import GPTBigCodeConfig
31 from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.activation import get_act_fn
33 from vllm.model_executor.layers.attention import PagedAttention
34 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35 + LinearMethodBase,
36 + QKVParallelLinear,
37 + RowParallelLinear)
38 from vllm.model_executor.layers.sampler import Sampler
35 -from vllm.model_executor.weight_utils import (
36 - convert_pyslice_to_tensor, hf_model_weights_iterator,
37 - load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
39 +from vllm.model_executor.layers.vocab_parallel_embedding import (
40 + VocabParallelEmbedding)
41 from vllm.model_executor.parallel_utils.parallel_state import (
39 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
41 - ColumnParallelLinear,
42 - RowParallelLinear)
42 + get_tensor_model_parallel_world_size)
43 +from vllm.model_executor.weight_utils import (default_weight_loader,
44 + hf_model_weights_iterator)
45 from vllm.sequence import SamplerOutput
46
47 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -47,7 +49,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
49
50 class GPTBigCodeAttention(nn.Module):
51
50 - def __init__(self, config: GPTBigCodeConfig):
52 + def __init__(
53 + self,
54 + config: GPTBigCodeConfig,
55 + linear_method: Optional[LinearMethodBase] = None,
56 + ):
57 super().__init__()
58 self.hidden_size = config.hidden_size
59 total_num_heads = config.num_attention_heads
@@ -61,32 +67,26 @@ class GPTBigCodeAttention(nn.Module):
67
68 self.multi_query = config.multi_query
69 if self.multi_query:
70 + total_num_kv_heads = 1
71 self.num_kv_heads = 1
65 - self.kv_dim = self.head_dim
66 - self.c_attn_q = ColumnParallelLinear(
67 - self.hidden_size,
68 - self.hidden_size,
69 - bias=True,
70 - gather_output=False,
71 - )
72 - self.c_attn_kv = nn.Linear(self.hidden_size,
73 - 2 * self.kv_dim,
74 - bias=True)
72 else:
73 + total_num_kv_heads = total_num_heads
74 self.num_kv_heads = self.num_heads
77 - self.kv_dim = self.num_kv_heads * self.head_dim
78 - self.c_attn = ColumnParallelLinear(
79 - self.hidden_size,
80 - self.hidden_size + 2 * self.kv_dim,
81 - bias=True,
82 - gather_output=False,
83 - )
75 + self.kv_dim = self.head_dim * self.num_kv_heads
76 + self.c_attn = QKVParallelLinear(
77 + self.hidden_size,
78 + self.head_dim,
79 + total_num_heads,
80 + total_num_kv_heads,
81 + bias=True,
82 + linear_method=linear_method,
83 + )
84
85 self.c_proj = RowParallelLinear(
86 self.hidden_size,
87 self.hidden_size,
88 bias=True,
89 - input_is_parallel=True,
89 + linear_method=linear_method,
90 )
91 self.attn = PagedAttention(self.num_heads,
92 self.head_dim,
@@ -100,17 +100,14 @@ class GPTBigCodeAttention(nn.Module):
100 input_metadata: InputMetadata,
101 cache_event: Optional[torch.cuda.Event],
102 ) -> torch.Tensor:
103 - if self.multi_query:
104 - q, _ = self.c_attn_q(hidden_states)
105 - kv = self.c_attn_kv(hidden_states)
106 - k, v = kv.split([self.kv_dim, self.kv_dim], dim=-1)
107 - else:
108 - qkv, _ = self.c_attn(hidden_states)
109 - q, k, v = qkv.split([
103 + qkv, _ = self.c_attn(hidden_states)
104 + q, k, v = qkv.split(
105 + [
106 self.hidden_size // self.tensor_model_parallel_world_size,
107 self.kv_dim, self.kv_dim
108 ],
113 - dim=-1)
109 + dim=-1,
110 + )
111 key_cache, value_cache = kv_cache
112 attn_output = self.attn(q, k, v, key_cache, value_cache,
113 input_metadata, cache_event)
@@ -124,6 +121,7 @@ class GPTBigMLP(nn.Module):
121 self,
122 intermediate_size: int,
123 config: GPTBigCodeConfig,
124 + linear_method: Optional[LinearMethodBase] = None,
125 ):
126 super().__init__()
127 hidden_size = config.hidden_size
@@ -131,13 +129,13 @@ class GPTBigMLP(nn.Module):
129 hidden_size,
130 intermediate_size,
131 bias=True,
134 - gather_output=False,
132 + linear_method=linear_method,
133 )
134 self.c_proj = RowParallelLinear(
135 intermediate_size,
136 hidden_size,
137 bias=True,
140 - input_is_parallel=True,
138 + linear_method=linear_method,
139 )
140 self.act = get_act_fn(config.activation_function)
141
@@ -150,16 +148,20 @@ class GPTBigMLP(nn.Module):
148
149 class GPTBigCodeBlock(nn.Module):
150
153 - def __init__(self, config: GPTBigCodeConfig):
151 + def __init__(
152 + self,
153 + config: GPTBigCodeConfig,
154 + linear_method: Optional[LinearMethodBase] = None,
155 + ):
156 super().__init__()
157 hidden_size = config.hidden_size
158 inner_dim = (config.n_inner if config.n_inner is not None else 4 *
159 hidden_size)
160
161 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
160 - self.attn = GPTBigCodeAttention(config)
162 + self.attn = GPTBigCodeAttention(config, linear_method)
163 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
162 - self.mlp = GPTBigMLP(inner_dim, config)
164 + self.mlp = GPTBigMLP(inner_dim, config, linear_method)
165
166 def forward(
167 self,
@@ -189,23 +191,23 @@ class GPTBigCodeBlock(nn.Module):
191
192 class GPTBigCodeModel(nn.Module):
193
192 - def __init__(self, config: GPTBigCodeConfig):
194 + def __init__(
195 + self,
196 + config: GPTBigCodeConfig,
197 + linear_method: Optional[LinearMethodBase] = None,
198 + ):
199 super().__init__()
200 self.config = config
201 assert not config.add_cross_attention
202
203 self.embed_dim = config.hidden_size
204
199 - # Optimization: While the vocab size of GPT-2 is 50257, we extend it
200 - # to 50304 in order to make it divisible by 64.
201 - # This improves performance since GPUs are faster if the dimension
202 - # is divisible by 64. In addition, it allows us to shard the embedding
203 - # layer across 2, 4, 8, or more GPUs.
204 - vocab_size = ((config.vocab_size + 63) // 64) * 64
205 - self.wte = VocabParallelEmbedding(vocab_size, self.embed_dim)
205 + self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
206 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
207 - self.h = nn.ModuleList(
208 - [GPTBigCodeBlock(config) for _ in range(config.num_hidden_layers)])
207 + self.h = nn.ModuleList([
208 + GPTBigCodeBlock(config, linear_method)
209 + for _ in range(config.num_hidden_layers)
210 + ])
211 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
212
213 def forward(
@@ -235,12 +237,15 @@ class GPTBigCodeModel(nn.Module):
237
238 class GPTBigCodeForCausalLM(nn.Module):
239
238 - def __init__(self, config: GPTBigCodeConfig):
240 + def __init__(
241 + self,
242 + config: GPTBigCodeConfig,
243 + linear_method: Optional[LinearMethodBase] = None,
244 + ):
245 super().__init__()
246 self.config = config
241 - self.transformer = GPTBigCodeModel(config)
242 - # TODO(zhuohan): create a new weight after implementing pipeline
243 - # parallelism
247 + self.linear_method = linear_method
248 + self.transformer = GPTBigCodeModel(config, linear_method)
249 self.lm_head_weight = self.transformer.wte.weight
250 self.sampler = Sampler(config.vocab_size)
251
@@ -258,89 +263,21 @@ class GPTBigCodeForCausalLM(nn.Module):
263 input_metadata)
264 return next_tokens
265
261 - _column_parallel_weights = ["c_fc.weight", "c_fc.bias"]
262 - _row_parallel_weights = ["c_proj.weight"]
263 -
266 def load_weights(self,
267 model_name_or_path: str,
268 cache_dir: Optional[str] = None,
269 load_format: str = "auto",
270 revision: Optional[str] = None):
269 - tensor_model_parallel_world_size = (
270 - get_tensor_model_parallel_world_size())
271 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
272 - state_dict = self.state_dict()
273 -
271 + params_dict = dict(self.named_parameters(remove_duplicate=False))
272 for name, loaded_weight in hf_model_weights_iterator(
273 model_name_or_path, cache_dir, load_format, revision):
274 if "lm_head.weight" in name:
277 - # GPT-2 ties the weights of the embedding layer and the final
278 - # linear layer.
275 continue
276 if ".attn.bias" in name:
277 # Skip attention mask.
278 # NOTE: "c_attn.bias" should not be skipped.
279 continue
284 -
285 - if not name.startswith("transformer."):
286 - name = "transformer." + name
287 -
288 - # For the fused QKV linear layer, manually shard the weights.
289 - if "c_attn" in name:
290 - # GPT-2's fused QKV has the shape of
291 - # [3 * num_heads * head_size, hidden_size].
292 - # When tensor parallelism is used, we shard the weights along
293 - # the head dimension.
294 - total_num_heads = self.config.num_attention_heads
295 - total_num_kv_heads = (1 if self.config.multi_query else
296 - total_num_heads)
297 - hidden_size = self.config.hidden_size
298 - head_size = hidden_size // total_num_heads
299 - total_kv_size = head_size * total_num_kv_heads
300 - num_heads = total_num_heads // tensor_model_parallel_world_size
301 - head_start = tensor_model_parallel_rank * num_heads
302 - head_end = (tensor_model_parallel_rank + 1) * num_heads
303 -
304 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
305 - wq, wk, wv = torch.split(
306 - loaded_weight, [hidden_size, total_kv_size, total_kv_size],
307 - dim=0)
308 -
309 - wq = wq[head_size * head_start:head_size * head_end]
310 - if not self.config.multi_query:
311 - # Split the heads when using normal multi-head attention
312 - wk = wk[head_size * head_start:head_size * head_end]
313 - wv = wv[head_size * head_start:head_size * head_end]
314 - loaded_weight = torch.cat([wq, wk, wv], dim=0)
315 - else:
316 - # For multi-query attention, we split the query
317 - # but replicate the key and value.
318 - loaded_weight_q = wq
319 - loaded_weight_kv = torch.cat([wk, wv], dim=0)
320 - q_weight_name = name.replace("c_attn", "c_attn_q")
321 - kv_weight_name = name.replace("c_attn", "c_attn_kv")
322 - load_tensor_parallel_weights(state_dict[q_weight_name],
323 - loaded_weight_q,
324 - q_weight_name,
325 - self._column_parallel_weights,
326 - self._row_parallel_weights,
327 - tensor_model_parallel_rank)
328 - load_tensor_parallel_weights(state_dict[kv_weight_name],
329 - loaded_weight_kv,
330 - kv_weight_name,
331 - self._column_parallel_weights,
332 - self._row_parallel_weights,
333 - tensor_model_parallel_rank)
334 - continue
335 -
336 - param = state_dict[name]
337 -
338 - if name == "transformer.wte.weight":
339 - load_padded_tensor_parallel_vocab(param, loaded_weight,
340 - tensor_model_parallel_rank)
341 - continue
342 -
343 - load_tensor_parallel_weights(param, loaded_weight, name,
344 - self._column_parallel_weights,
345 - self._row_parallel_weights,
346 - tensor_model_parallel_rank)
280 + param = params_dict[name]
281 + weight_loader = getattr(param, "weight_loader",
282 + default_weight_loader)
283 + weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_j.py
+69 -53
@@ -29,14 +29,17 @@ from transformers import GPTJConfig
29 from vllm.model_executor.input_metadata import InputMetadata
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 + LinearMethodBase,
34 + QKVParallelLinear,
35 + RowParallelLinear)
36 from vllm.model_executor.layers.sampler import Sampler
33 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34 - load_tensor_parallel_weights)
37 +from vllm.model_executor.layers.vocab_parallel_embedding import (
38 + VocabParallelEmbedding, ParallelLMHead)
39 from vllm.model_executor.parallel_utils.parallel_state import (
36 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38 - ColumnParallelLinear,
39 - RowParallelLinear)
40 + get_tensor_model_parallel_world_size)
41 +from vllm.model_executor.weight_utils import (default_weight_loader,
42 + hf_model_weights_iterator)
43 from vllm.sequence import SamplerOutput
44
45 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -44,23 +47,28 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
47
48 class GPTJAttention(nn.Module):
49
47 - def __init__(self, config: GPTJConfig):
50 + def __init__(
51 + self,
52 + config: GPTJConfig,
53 + linear_method: Optional[LinearMethodBase] = None,
54 + ):
55 super().__init__()
56 self.total_num_heads = config.num_attention_heads
57 self.hidden_size = config.hidden_size
58 self.head_size = self.hidden_size // self.total_num_heads
59
53 - self.qkv_proj = ColumnParallelLinear(
60 + self.qkv_proj = QKVParallelLinear(
61 config.hidden_size,
55 - 3 * config.hidden_size,
62 + self.head_size,
63 + self.total_num_heads,
64 bias=False,
57 - gather_output=False,
65 + linear_method=linear_method,
66 )
67 self.out_proj = RowParallelLinear(
68 config.hidden_size,
69 config.hidden_size,
70 bias=False,
63 - input_is_parallel=True,
71 + linear_method=linear_method,
72 )
73
74 tp_world_size = get_tensor_model_parallel_world_size()
@@ -102,18 +110,23 @@ class GPTJAttention(nn.Module):
110
111 class GPTJMLP(nn.Module):
112
105 - def __init__(self, intermediate_size: int, config: GPTJConfig):
113 + def __init__(
114 + self,
115 + intermediate_size: int,
116 + config: GPTJConfig,
117 + linear_method: Optional[LinearMethodBase] = None,
118 + ):
119 super().__init__()
120 hidden_size = config.n_embd
121 self.fc_in = ColumnParallelLinear(
122 hidden_size,
123 intermediate_size,
111 - gather_output=False,
124 + linear_method=linear_method,
125 )
126 self.fc_out = RowParallelLinear(
127 intermediate_size,
128 hidden_size,
116 - input_is_parallel=True,
129 + linear_method=linear_method,
130 )
131 self.act = get_act_fn(config.activation_function)
132
@@ -126,15 +139,19 @@ class GPTJMLP(nn.Module):
139
140 class GPTJBlock(nn.Module):
141
129 - def __init__(self, config: GPTJConfig):
142 + def __init__(
143 + self,
144 + config: GPTJConfig,
145 + linear_method: Optional[LinearMethodBase] = None,
146 + ):
147 super().__init__()
148 if config.n_inner is None:
149 inner_dim = 4 * config.n_embd
150 else:
151 inner_dim = config.n_inner
152 self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
136 - self.attn = GPTJAttention(config)
137 - self.mlp = GPTJMLP(inner_dim, config)
153 + self.attn = GPTJAttention(config, linear_method)
154 + self.mlp = GPTJMLP(inner_dim, config, linear_method)
155
156 def forward(
157 self,
@@ -160,7 +177,11 @@ class GPTJBlock(nn.Module):
177
178 class GPTJModel(nn.Module):
179
163 - def __init__(self, config: GPTJConfig):
180 + def __init__(
181 + self,
182 + config: GPTJConfig,
183 + linear_method: Optional[LinearMethodBase] = None,
184 + ):
185 super().__init__()
186 self.config = config
187 self.embed_dim = config.n_embd
@@ -169,7 +190,7 @@ class GPTJModel(nn.Module):
190 self.embed_dim,
191 )
192 self.h = nn.ModuleList(
172 - [GPTJBlock(config) for _ in range(config.n_layer)])
193 + [GPTJBlock(config, linear_method) for _ in range(config.n_layer)])
194 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
195
196 def forward(
@@ -200,15 +221,20 @@ class GPTJModel(nn.Module):
221
222 class GPTJForCausalLM(nn.Module):
223
203 - def __init__(self, config: GPTJConfig):
224 + def __init__(
225 + self,
226 + config: GPTJConfig,
227 + linear_method: Optional[LinearMethodBase] = None,
228 + ):
229 super().__init__()
230 self.config = config
231 + self.linear_method = linear_method
232 assert not config.tie_word_embeddings
207 - self.transformer = GPTJModel(config)
208 - self.lm_head = ColumnParallelLinear(
209 - config.n_embd,
233 + self.transformer = GPTJModel(config, linear_method)
234 + self.lm_head = ParallelLMHead(
235 config.vocab_size,
211 - gather_output=False,
236 + config.n_embd,
237 + bias=True,
238 )
239 self.sampler = Sampler(config.vocab_size)
240
@@ -226,43 +252,33 @@ class GPTJForCausalLM(nn.Module):
252 input_metadata, self.lm_head.bias)
253 return next_tokens
254
229 - _column_parallel_weights = [
230 - "wte.weight", "fc_in.weight", "fc_in.bias", "lm_head.weight",
231 - "lm_head.bias"
232 - ]
233 - _row_parallel_weights = ["out_proj.weight", "fc_out.weight"]
234 -
255 def load_weights(self,
256 model_name_or_path: str,
257 cache_dir: Optional[str] = None,
258 load_format: str = "auto",
259 revision: Optional[str] = None):
240 - tp_rank = get_tensor_model_parallel_rank()
241 - state_dict = self.state_dict()
260 + stacked_params_mapping = [
261 + # (param_name, shard_name, shard_id)
262 + ("qkv_proj", "q_proj", "q"),
263 + ("qkv_proj", "k_proj", "k"),
264 + ("qkv_proj", "v_proj", "v"),
265 + ("gate_up_proj", "gate_proj", 0),
266 + ("gate_up_proj", "up_proj", 1),
267 + ]
268 + params_dict = dict(self.named_parameters())
269 for name, loaded_weight in hf_model_weights_iterator(
270 model_name_or_path, cache_dir, load_format, revision):
271 if "attn.bias" in name or "attn.masked_bias" in name:
272 continue
246 -
247 - is_attention_weight = False
248 - for stride_id, att_weight_name in enumerate(
249 - ["q_proj", "k_proj", "v_proj"]):
250 - if att_weight_name not in name:
273 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
274 + if weight_name not in name:
275 continue
252 - param = state_dict[name.replace(att_weight_name, "qkv_proj")]
253 - shard_size = param.shape[0] // 3
254 - loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
255 - (tp_rank + 1)]
256 - param_slice = param.data[shard_size * stride_id:shard_size *
257 - (stride_id + 1)]
258 - assert param_slice.shape == loaded_weight.shape
259 - param_slice.copy_(loaded_weight)
260 - is_attention_weight = True
276 + param = params_dict[name.replace(weight_name, param_name)]
277 + weight_loader = param.weight_loader
278 + weight_loader(param, loaded_weight, shard_id)
279 break
262 - if is_attention_weight:
263 - continue
264 -
265 - param = state_dict[name]
266 - load_tensor_parallel_weights(param, loaded_weight, name,
267 - self._column_parallel_weights,
268 - self._row_parallel_weights, tp_rank)
280 + else:
281 + param = params_dict[name]
282 + weight_loader = getattr(param, "weight_loader",
283 + default_weight_loader)
284 + weight_loader(param, loaded_weight)
vllm/model_executor/models/gpt_neox.py
+70 -60
@@ -29,14 +29,17 @@ from transformers import GPTNeoXConfig
29 from vllm.model_executor.input_metadata import InputMetadata
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 + LinearMethodBase,
34 + QKVParallelLinear,
35 + RowParallelLinear)
36 from vllm.model_executor.layers.sampler import Sampler
33 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34 - load_tensor_parallel_weights)
37 +from vllm.model_executor.layers.vocab_parallel_embedding import (
38 + VocabParallelEmbedding, ParallelLMHead)
39 from vllm.model_executor.parallel_utils.parallel_state import (
36 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38 - ColumnParallelLinear,
39 - RowParallelLinear)
40 + get_tensor_model_parallel_world_size)
41 +from vllm.model_executor.weight_utils import (default_weight_loader,
42 + hf_model_weights_iterator)
43 from vllm.sequence import SamplerOutput
44
45 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -44,7 +47,11 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
47
48 class GPTNeoXAttention(nn.Module):
49
47 - def __init__(self, config: GPTNeoXConfig):
50 + def __init__(
51 + self,
52 + config: GPTNeoXConfig,
53 + linear_method: Optional[LinearMethodBase] = None,
54 + ):
55 super().__init__()
56 self.total_num_heads = config.num_attention_heads
57 self.hidden_size = config.hidden_size
@@ -56,15 +63,16 @@ class GPTNeoXAttention(nn.Module):
63 self.num_heads = (self.total_num_heads //
64 tensor_model_parallel_world_size)
65
59 - self.query_key_value = ColumnParallelLinear(
66 + self.query_key_value = QKVParallelLinear(
67 config.hidden_size,
61 - 3 * config.hidden_size,
62 - gather_output=False,
68 + self.head_size,
69 + self.total_num_heads,
70 + linear_method=linear_method,
71 )
72 self.dense = RowParallelLinear(
73 config.hidden_size,
74 config.hidden_size,
67 - input_is_parallel=True,
75 + linear_method=linear_method,
76 )
77
78 scaling = self.head_size**-0.5
@@ -100,17 +108,21 @@ class GPTNeoXAttention(nn.Module):
108
109 class GPTNeoXMLP(nn.Module):
110
103 - def __init__(self, config: GPTNeoXConfig):
111 + def __init__(
112 + self,
113 + config: GPTNeoXConfig,
114 + linear_method: Optional[LinearMethodBase] = None,
115 + ):
116 super().__init__()
117 self.dense_h_to_4h = ColumnParallelLinear(
118 config.hidden_size,
119 config.intermediate_size,
108 - gather_output=False,
120 + linear_method=linear_method,
121 )
122 self.dense_4h_to_h = RowParallelLinear(
123 config.intermediate_size,
124 config.hidden_size,
113 - input_is_parallel=True,
125 + linear_method=linear_method,
126 )
127 self.act = get_act_fn(config.hidden_act)
128
@@ -123,15 +135,19 @@ class GPTNeoXMLP(nn.Module):
135
136 class GPTNeoXLayer(nn.Module):
137
126 - def __init__(self, config: GPTNeoXConfig):
138 + def __init__(
139 + self,
140 + config: GPTNeoXConfig,
141 + linear_method: Optional[LinearMethodBase] = None,
142 + ):
143 super().__init__()
144 self.use_parallel_residual = config.use_parallel_residual
145 self.input_layernorm = nn.LayerNorm(config.hidden_size,
146 eps=config.layer_norm_eps)
147 self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
148 eps=config.layer_norm_eps)
133 - self.attention = GPTNeoXAttention(config)
134 - self.mlp = GPTNeoXMLP(config)
149 + self.attention = GPTNeoXAttention(config, linear_method)
150 + self.mlp = GPTNeoXMLP(config, linear_method)
151
152 def forward(
153 self,
@@ -169,7 +185,11 @@ class GPTNeoXLayer(nn.Module):
185
186 class GPTNeoXModel(nn.Module):
187
172 - def __init__(self, config: GPTNeoXConfig):
188 + def __init__(
189 + self,
190 + config: GPTNeoXConfig,
191 + linear_method: Optional[LinearMethodBase] = None,
192 + ):
193 super().__init__()
194 self.config = config
195
@@ -177,8 +197,10 @@ class GPTNeoXModel(nn.Module):
197 config.vocab_size,
198 config.hidden_size,
199 )
180 - self.layers = nn.ModuleList(
181 - [GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
200 + self.layers = nn.ModuleList([
201 + GPTNeoXLayer(config, linear_method)
202 + for _ in range(config.num_hidden_layers)
203 + ])
204 self.final_layer_norm = nn.LayerNorm(config.hidden_size,
205 eps=config.layer_norm_eps)
206
@@ -210,15 +232,18 @@ class GPTNeoXModel(nn.Module):
232
233 class GPTNeoXForCausalLM(nn.Module):
234
213 - def __init__(self, config):
235 + def __init__(
236 + self,
237 + config,
238 + linear_method: Optional[LinearMethodBase] = None,
239 + ):
240 super().__init__()
241 self.config = config
216 - self.gpt_neox = GPTNeoXModel(config)
217 - self.embed_out = ColumnParallelLinear(
218 - config.hidden_size,
242 + self.linear_method = linear_method
243 + self.gpt_neox = GPTNeoXModel(config, linear_method)
244 + self.embed_out = ParallelLMHead(
245 config.vocab_size,
220 - bias=False,
221 - gather_output=False,
246 + config.hidden_size,
247 )
248 self.sampler = Sampler(config.vocab_size)
249
@@ -236,50 +261,35 @@ class GPTNeoXForCausalLM(nn.Module):
261 input_metadata)
262 return next_tokens
263
239 - _column_parallel_weights = [
240 - "embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight",
241 - "dense_h_to_4h.bias"
242 - ]
243 - _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
244 -
264 def load_weights(self,
265 model_name_or_path: str,
266 cache_dir: Optional[str] = None,
267 load_format: str = "auto",
268 revision: Optional[str] = None):
250 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
251 - state_dict = self.state_dict()
269 + params_dict = dict(self.named_parameters())
270 for name, loaded_weight in hf_model_weights_iterator(
271 model_name_or_path, cache_dir, load_format, revision):
272 if ("attention.bias" in name or "attention.masked_bias" in name
273 or "rotary_emb.inv_freq" in name):
274 continue
257 - param = state_dict[name]
275 + param = params_dict[name]
276 +
277 if "query_key_value" in name:
259 - # NOTE(woosuk): GPT-NeoX's fused QKV has the shape of
260 - # [num_heads * 3 * head_size, hidden_size], while the
261 - # required shape is [3 * num_heads * head_size, hidden_size].
278 + # NOTE: GPT-NeoX's fused QKV's output_dim has the shape of
279 + # (num_heads * 3 * head_size), while the
280 + # required shape is (3 * num_heads * head_size).
281 # Thus, we need weight conversion.
263 - shard_size = param.shape[0]
264 - loaded_weight = loaded_weight[
265 - shard_size * tensor_model_parallel_rank:shard_size *
266 - (tensor_model_parallel_rank + 1)]
267 -
282 + output_dim = getattr(param, "output_dim", None)
283 num_heads = self.config.num_attention_heads
269 - hidden_size = self.config.hidden_size
270 - head_size = hidden_size // num_heads
271 - if "query_key_value.weight" in name:
272 - loaded_weight = loaded_weight.view(-1, 3, head_size,
273 - hidden_size)
274 - loaded_weight = loaded_weight.transpose(0, 1)
275 - loaded_weight = loaded_weight.reshape(-1, hidden_size)
276 - elif "query_key_value.bias" in name:
277 - loaded_weight = loaded_weight.view(-1, 3, head_size)
278 - loaded_weight = loaded_weight.transpose(0, 1)
279 - loaded_weight = loaded_weight.reshape(-1)
280 - else:
281 - raise ValueError(f"Unexpected weight name: {name}")
282 - load_tensor_parallel_weights(param, loaded_weight, name,
283 - self._column_parallel_weights,
284 - self._row_parallel_weights,
285 - tensor_model_parallel_rank)
284 + if output_dim is not None:
285 + loaded_weight_shape = loaded_weight.shape
286 + loaded_weight = loaded_weight.view(
287 + loaded_weight_shape[:output_dim] + (num_heads, 3, -1) +
288 + loaded_weight_shape[output_dim + 1:])
289 + loaded_weight = loaded_weight.transpose(
290 + output_dim, output_dim + 1)
291 + loaded_weight = loaded_weight.reshape(loaded_weight_shape)
292 +
293 + weight_loader = getattr(param, "weight_loader",
294 + default_weight_loader)
295 + weight_loader(param, loaded_weight)
vllm/model_executor/models/internlm.py
+62 -88
@@ -9,15 +9,17 @@ from vllm.model_executor.input_metadata import InputMetadata
9 from vllm.model_executor.layers.activation import SiluAndMul
10 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 +from vllm.model_executor.layers.linear import (LinearMethodBase,
13 + MergedColumnParallelLinear,
14 + QKVParallelLinear,
15 + RowParallelLinear)
16 from vllm.model_executor.layers.sampler import Sampler
17 +from vllm.model_executor.layers.vocab_parallel_embedding import (
18 + VocabParallelEmbedding, ParallelLMHead)
19 from vllm.model_executor.parallel_utils.parallel_state import (
14 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15 -from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
16 - RowParallelLinear,
17 - VocabParallelEmbedding)
18 -from vllm.model_executor.weight_utils import (
19 - hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
20 - load_tensor_parallel_weights)
20 + get_tensor_model_parallel_world_size)
21 +from vllm.model_executor.weight_utils import (default_weight_loader,
22 + hf_model_weights_iterator)
23 from vllm.sequence import SamplerOutput
24
25 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -30,20 +32,17 @@ class InternLMMLP(nn.Module):
32 hidden_size: int,
33 intermediate_size: int,
34 hidden_act: str,
35 + linear_method: Optional[LinearMethodBase] = None,
36 ):
37 super().__init__()
35 - self.gate_up_proj = ColumnParallelLinear(
36 - hidden_size,
37 - 2 * intermediate_size,
38 - bias=False,
39 - gather_output=False,
40 - )
41 - self.down_proj = RowParallelLinear(
42 - intermediate_size,
43 - hidden_size,
38 + self.gate_up_proj = MergedColumnParallelLinear(
39 + hidden_size, [intermediate_size] * 2,
40 bias=False,
45 - input_is_parallel=True,
46 - )
41 + linear_method=linear_method)
42 + self.down_proj = RowParallelLinear(intermediate_size,
43 + hidden_size,
44 + bias=False,
45 + linear_method=linear_method)
46 if hidden_act != "silu":
47 raise ValueError(f"Unsupported activation: {hidden_act}. "
48 "Only silu is supported for now.")
@@ -65,6 +64,7 @@ class InternLMAttention(nn.Module):
64 bias: bool,
65 rope_theta: float = 10000,
66 max_position_embeddings: int = 8192,
67 + linear_method: Optional[LinearMethodBase] = None,
68 ):
69 super().__init__()
70 self.hidden_size = hidden_size
@@ -79,17 +79,18 @@ class InternLMAttention(nn.Module):
79 self.rope_theta = rope_theta
80 self.max_position_embeddings = max_position_embeddings
81
82 - self.qkv_proj = ColumnParallelLinear(
82 + self.qkv_proj = QKVParallelLinear(
83 hidden_size,
84 - 3 * self.total_num_heads * self.head_dim,
84 + self.head_dim,
85 + self.total_num_heads,
86 bias=bias,
86 - gather_output=False,
87 + linear_method=linear_method,
88 )
89 self.o_proj = RowParallelLinear(
90 self.total_num_heads * self.head_dim,
91 hidden_size,
92 bias=bias,
92 - input_is_parallel=True,
93 + linear_method=linear_method,
94 )
95 self.attn = PagedAttentionWithRoPE(
96 self.num_heads,
@@ -118,7 +119,11 @@ class InternLMAttention(nn.Module):
119
120 class InternLMDecoderLayer(nn.Module):
121
121 - def __init__(self, config: LlamaConfig):
122 + def __init__(
123 + self,
124 + config: LlamaConfig,
125 + linear_method: Optional[LinearMethodBase] = None,
126 + ):
127 super().__init__()
128 self.hidden_size = config.hidden_size
129 rope_theta = getattr(config, "rope_theta", 10000)
@@ -130,11 +135,13 @@ class InternLMDecoderLayer(nn.Module):
135 bias=config.bias,
136 rope_theta=rope_theta,
137 max_position_embeddings=max_position_embeddings,
138 + linear_method=linear_method,
139 )
140 self.mlp = InternLMMLP(
141 hidden_size=self.hidden_size,
142 intermediate_size=config.intermediate_size,
143 hidden_act=config.hidden_act,
144 + linear_method=linear_method,
145 )
146 self.input_layernorm = RMSNorm(config.hidden_size,
147 eps=config.rms_norm_eps)
@@ -171,7 +178,11 @@ class InternLMDecoderLayer(nn.Module):
178
179 class InternLMModel(nn.Module):
180
174 - def __init__(self, config: LlamaConfig):
181 + def __init__(
182 + self,
183 + config: LlamaConfig,
184 + linear_method: Optional[LinearMethodBase] = None,
185 + ):
186 super().__init__()
187 self.config = config
188 self.padding_idx = config.pad_token_id
@@ -183,7 +194,7 @@ class InternLMModel(nn.Module):
194 config.hidden_size,
195 )
196 self.layers = nn.ModuleList([
186 - InternLMDecoderLayer(config)
197 + InternLMDecoderLayer(config, linear_method)
198 for _ in range(config.num_hidden_layers)
199 ])
200 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -216,17 +227,16 @@ class InternLMModel(nn.Module):
227
228 class InternLMForCausalLM(nn.Module):
229
219 - def __init__(self, config):
230 + def __init__(
231 + self,
232 + config,
233 + linear_method: Optional[LinearMethodBase] = None,
234 + ):
235 super().__init__()
236 self.config = config
222 - self.model = InternLMModel(config)
223 - vocab_size = ((config.vocab_size + 63) // 64) * 64
224 - self.lm_head = ColumnParallelLinear(
225 - config.hidden_size,
226 - vocab_size,
227 - bias=False,
228 - gather_output=False,
229 - )
237 + self.linear_method = linear_method
238 + self.model = InternLMModel(config, linear_method)
239 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
240 self.sampler = Sampler(config.vocab_size)
241
242 def forward(
@@ -243,69 +253,33 @@ class InternLMForCausalLM(nn.Module):
253 input_metadata)
254 return next_tokens
255
246 - _column_parallel_weights = [
247 - "qkv_proj.weight", "gate_proj.weight", "up_proj.weight"
248 - ]
249 - _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
250 -
256 def load_weights(self,
257 model_name_or_path: str,
258 cache_dir: Optional[str] = None,
259 load_format: str = "auto",
260 revision: Optional[str] = None):
256 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
257 - state_dict = self.state_dict()
258 -
261 + stacked_params_mapping = [
262 + # (param_name, shard_name, shard_id)
263 + ("qkv_proj", "q_proj", "q"),
264 + ("qkv_proj", "k_proj", "k"),
265 + ("qkv_proj", "v_proj", "v"),
266 + ("gate_up_proj", "gate_proj", 0),
267 + ("gate_up_proj", "up_proj", 1),
268 + ]
269 + params_dict = dict(self.named_parameters())
270 for name, loaded_weight in hf_model_weights_iterator(
271 model_name_or_path, cache_dir, load_format, revision):
272 if "rotary_emb.inv_freq" in name:
273 continue
263 -
264 - if "embed_tokens" in name or "lm_head" in name:
265 - param = state_dict[name]
266 - load_padded_tensor_parallel_vocab(param, loaded_weight,
267 - tensor_model_parallel_rank)
268 - continue
269 -
270 - is_attention_weight = False
271 - for stride_id, att_weight_name in enumerate(
272 - ["q_proj", "k_proj", "v_proj"]):
273 - if att_weight_name not in name:
274 - continue
275 - param = state_dict[name.replace(att_weight_name, "qkv_proj")]
276 - shard_size = param.shape[0] // 3
277 - loaded_weight = loaded_weight[
278 - shard_size * tensor_model_parallel_rank:shard_size *
279 - (tensor_model_parallel_rank + 1)]
280 - param_slice = param.data[shard_size * stride_id:shard_size *
281 - (stride_id + 1)]
282 - assert param_slice.shape == loaded_weight.shape
283 - param_slice.copy_(loaded_weight)
284 - is_attention_weight = True
285 - break
286 - if is_attention_weight:
287 - continue
288 -
289 - is_gate_up_weight = False
290 - for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
274 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
275 if weight_name not in name:
276 continue
293 - param = state_dict[name.replace(weight_name, "gate_up_proj")]
294 - shard_size = param.shape[0] // 2
295 - loaded_weight = loaded_weight[
296 - shard_size * tensor_model_parallel_rank:shard_size *
297 - (tensor_model_parallel_rank + 1)]
298 - param_slice = param.data[shard_size * stride_id:shard_size *
299 - (stride_id + 1)]
300 - assert param_slice.shape == loaded_weight.shape
301 - param_slice.copy_(loaded_weight)
302 - is_gate_up_weight = True
277 + param = params_dict[name.replace(weight_name, param_name)]
278 + weight_loader = param.weight_loader
279 + weight_loader(param, loaded_weight, shard_id)
280 break
304 - if is_gate_up_weight:
305 - continue
306 -
307 - param = state_dict[name]
308 - load_tensor_parallel_weights(param, loaded_weight, name,
309 - self._column_parallel_weights,
310 - self._row_parallel_weights,
311 - tensor_model_parallel_rank)
281 + else:
282 + param = params_dict[name]
283 + weight_loader = getattr(param, "weight_loader",
284 + default_weight_loader)
285 + weight_loader(param, loaded_weight)
vllm/model_executor/models/llama.py
+53 -156
@@ -33,17 +33,19 @@ from transformers import LlamaConfig
33
34 from vllm.model_executor.input_metadata import InputMetadata
35 from vllm.model_executor.layers.activation import SiluAndMul
36 -from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
37 +from vllm.model_executor.layers.layernorm import RMSNorm
38 +from vllm.model_executor.layers.linear import (LinearMethodBase,
39 + MergedColumnParallelLinear,
40 + QKVParallelLinear,
41 + RowParallelLinear)
42 from vllm.model_executor.layers.sampler import Sampler
39 -from vllm.model_executor.layers.quantized_linear import ParallelLinear
43 +from vllm.model_executor.layers.vocab_parallel_embedding import (
44 + VocabParallelEmbedding, ParallelLMHead)
45 from vllm.model_executor.parallel_utils.parallel_state import (
41 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43 -from vllm.model_executor.quantization_utils import QuantizationConfig
44 -from vllm.model_executor.weight_utils import (
45 - convert_pyslice_to_tensor, hf_model_weights_iterator,
46 - load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
46 + get_tensor_model_parallel_world_size)
47 +from vllm.model_executor.weight_utils import (default_weight_loader,
48 + hf_model_weights_iterator)
49 from vllm.sequence import SamplerOutput
50
51 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,19 +58,17 @@ class LlamaMLP(nn.Module):
58 hidden_size: int,
59 intermediate_size: int,
60 hidden_act: str,
59 - quant_config: Optional[QuantizationConfig] = None,
61 + linear_method: Optional[LinearMethodBase] = None,
62 ) -> None:
63 super().__init__()
62 - self.gate_up_proj = ParallelLinear.column(hidden_size,
63 - 2 * intermediate_size,
64 - bias=False,
65 - gather_output=False,
66 - quant_config=quant_config)
67 - self.down_proj = ParallelLinear.row(intermediate_size,
68 - hidden_size,
69 - bias=False,
70 - input_is_parallel=True,
71 - quant_config=quant_config)
64 + self.gate_up_proj = MergedColumnParallelLinear(
65 + hidden_size, [intermediate_size] * 2,
66 + bias=False,
67 + linear_method=linear_method)
68 + self.down_proj = RowParallelLinear(intermediate_size,
69 + hidden_size,
70 + bias=False,
71 + linear_method=linear_method)
72 if hidden_act != "silu":
73 raise ValueError(f"Unsupported activation: {hidden_act}. "
74 "Only silu is supported for now.")
@@ -91,7 +91,7 @@ class LlamaAttention(nn.Module):
91 rope_theta: float = 10000,
92 rope_scaling: Optional[Dict[str, Any]] = None,
93 max_position_embeddings: int = 8192,
94 - quant_config: Optional[QuantizationConfig] = None,
94 + linear_method: Optional[LinearMethodBase] = None,
95 ) -> None:
96 super().__init__()
97 self.hidden_size = hidden_size
@@ -109,7 +109,6 @@ class LlamaAttention(nn.Module):
109 # the KV heads across multiple tensor parallel GPUs.
110 assert tp_size % self.total_num_kv_heads == 0
111 self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112 - num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
112 self.head_dim = hidden_size // self.total_num_heads
113 self.q_size = self.num_heads * self.head_dim
114 self.kv_size = self.num_kv_heads * self.head_dim
@@ -117,21 +116,19 @@ class LlamaAttention(nn.Module):
116 self.rope_theta = rope_theta
117 self.max_position_embeddings = max_position_embeddings
118
120 - self.qkv_proj = ParallelLinear.column(
119 + self.qkv_proj = QKVParallelLinear(
120 hidden_size,
122 - (self.total_num_heads +
123 - 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
121 self.head_dim,
122 + self.total_num_heads,
123 + self.total_num_kv_heads,
124 bias=False,
126 - gather_output=False,
127 - quant_config=quant_config,
125 + linear_method=linear_method,
126 )
129 - self.o_proj = ParallelLinear.row(
127 + self.o_proj = RowParallelLinear(
128 self.total_num_heads * self.head_dim,
129 hidden_size,
130 bias=False,
133 - input_is_parallel=True,
134 - quant_config=quant_config,
131 + linear_method=linear_method,
132 )
133 self.attn = PagedAttentionWithRoPE(
134 self.num_heads,
@@ -165,11 +162,10 @@ class LlamaDecoderLayer(nn.Module):
162 def __init__(
163 self,
164 config: LlamaConfig,
168 - quant_config: Optional[QuantizationConfig] = None,
165 + linear_method: Optional[LinearMethodBase] = None,
166 ) -> None:
167 super().__init__()
168 self.hidden_size = config.hidden_size
172 - # Requires transformers > 4.32.0
169 rope_theta = getattr(config, "rope_theta", 10000)
170 rope_scaling = getattr(config, "rope_scaling", None)
171 max_position_embeddings = getattr(config, "max_position_embeddings",
@@ -181,13 +177,13 @@ class LlamaDecoderLayer(nn.Module):
177 rope_theta=rope_theta,
178 rope_scaling=rope_scaling,
179 max_position_embeddings=max_position_embeddings,
184 - quant_config=quant_config,
180 + linear_method=linear_method,
181 )
182 self.mlp = LlamaMLP(
183 hidden_size=self.hidden_size,
184 intermediate_size=config.intermediate_size,
185 hidden_act=config.hidden_act,
190 - quant_config=quant_config,
186 + linear_method=linear_method,
187 )
188 self.input_layernorm = RMSNorm(config.hidden_size,
189 eps=config.rms_norm_eps)
@@ -227,20 +223,18 @@ class LlamaModel(nn.Module):
223 def __init__(
224 self,
225 config: LlamaConfig,
230 - quant_config: Optional[QuantizationConfig] = None,
226 + linear_method: Optional[LinearMethodBase] = None,
227 ) -> None:
228 super().__init__()
229 self.config = config
230 self.padding_idx = config.pad_token_id
231 self.vocab_size = config.vocab_size
236 -
237 - vocab_size = ((config.vocab_size + 63) // 64) * 64
232 self.embed_tokens = VocabParallelEmbedding(
239 - vocab_size,
233 + config.vocab_size,
234 config.hidden_size,
235 )
236 self.layers = nn.ModuleList([
243 - LlamaDecoderLayer(config, quant_config)
237 + LlamaDecoderLayer(config, linear_method)
238 for _ in range(config.num_hidden_layers)
239 ])
240 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -276,19 +270,13 @@ class LlamaForCausalLM(nn.Module):
270 def __init__(
271 self,
272 config: LlamaConfig,
279 - quant_config: Optional[QuantizationConfig] = None,
273 + linear_method: Optional[LinearMethodBase] = None,
274 ) -> None:
275 super().__init__()
276 self.config = config
283 - self.quant_config = quant_config
284 - self.model = LlamaModel(config, quant_config)
285 - vocab_size = ((config.vocab_size + 63) // 64) * 64
286 - # NOTE: The LM head is not quantized.
287 - self.lm_head = ParallelLinear.column(config.hidden_size,
288 - vocab_size,
289 - bias=False,
290 - gather_output=False,
291 - quant_config=None)
277 + self.linear_method = linear_method
278 + self.model = LlamaModel(config, linear_method)
279 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
280 self.sampler = Sampler(config.vocab_size)
281
282 def forward(
@@ -305,124 +293,33 @@ class LlamaForCausalLM(nn.Module):
293 input_metadata)
294 return next_tokens
295
308 - _column_parallel_layers = []
309 - _row_parallel_layers = ["o_proj", "down_proj"]
310 -
296 def load_weights(self,
297 model_name_or_path: str,
298 cache_dir: Optional[str] = None,
299 load_format: str = "auto",
300 revision: Optional[str] = None):
316 - if self.quant_config is None:
317 - col_weight_suffixes = ["weight"]
318 - row_weight_suffixes = ["weight"]
319 - else:
320 - col_weight_suffixes = (
321 - self.quant_config.get_col_parallel_tensor_names())
322 - row_weight_suffixes = (
323 - self.quant_config.get_row_parallel_tensor_names())
324 -
325 - column_parallel_weights: List[str] = []
326 - for layer in self._column_parallel_layers:
327 - for suffix in col_weight_suffixes:
328 - column_parallel_weights.append(f"{layer}.{suffix}")
329 - row_parallel_weights: List[str] = []
330 - for layer in self._row_parallel_layers:
331 - for suffix in row_weight_suffixes:
332 - row_parallel_weights.append(f"{layer}.{suffix}")
333 -
334 - tp_size = get_tensor_model_parallel_world_size()
335 - tp_rank = get_tensor_model_parallel_rank()
336 - q_proj_shard_size = (self.config.hidden_size // tp_size)
337 - num_kv_heads_replicas = max(1,
338 - tp_size // self.config.num_key_value_heads)
339 - num_kv_heads_per_gpu = max(1,
340 - self.config.num_key_value_heads // tp_size)
341 - kv_proj_shard_size = (self.config.hidden_size //
342 - self.config.num_attention_heads *
343 - num_kv_heads_per_gpu)
344 - attention_weight_specs = [
345 - # (weight_name, shard_size, offset)
346 - ("q_proj", q_proj_shard_size, 0),
347 - ("k_proj", kv_proj_shard_size, q_proj_shard_size),
348 - ("v_proj", kv_proj_shard_size,
349 - q_proj_shard_size + kv_proj_shard_size),
301 + stacked_params_mapping = [
302 + # (param_name, shard_name, shard_id)
303 + ("qkv_proj", "q_proj", "q"),
304 + ("qkv_proj", "k_proj", "k"),
305 + ("qkv_proj", "v_proj", "v"),
306 + ("gate_up_proj", "gate_proj", 0),
307 + ("gate_up_proj", "up_proj", 1),
308 ]
351 - state_dict = self.state_dict()
352 -
309 + params_dict = dict(self.named_parameters())
310 for name, loaded_weight in hf_model_weights_iterator(
311 model_name_or_path, cache_dir, load_format, revision):
312 if "rotary_emb.inv_freq" in name:
313 continue
357 -
358 - packed_dim = None
359 - is_transposed = False
360 - if self.quant_config is not None:
361 - packed_dim = self.quant_config.get_packed_dim(name)
362 - is_transposed = self.quant_config.is_transposed(name)
363 - if is_transposed:
364 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
365 - loaded_weight = loaded_weight.T
366 -
367 - is_attention_weight = False
368 - for weight_name, shard_size, offset in attention_weight_specs:
314 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
315 if weight_name not in name:
316 continue
371 - param = state_dict[name.replace(weight_name, "qkv_proj")]
372 - if is_transposed:
373 - param = param.T
374 -
375 - if packed_dim is not None:
376 - shard_dim = 0 if not is_transposed else 1
377 - if packed_dim == shard_dim:
378 - shard_size //= self.quant_config.pack_factor
379 - offset //= self.quant_config.pack_factor
380 -
381 - if weight_name in ["k_proj", "v_proj"]:
382 - shard_id = tp_rank // num_kv_heads_replicas
383 - else:
384 - shard_id = tp_rank
385 - loaded_weight = loaded_weight[shard_size *
386 - shard_id:shard_size *
387 - (shard_id + 1)]
388 - param_slice = param.data[offset:offset + shard_size]
389 - assert param_slice.shape == loaded_weight.shape
390 -
391 - param_slice.copy_(loaded_weight)
392 - is_attention_weight = True
393 - break
394 - if is_attention_weight:
395 - continue
396 -
397 - is_gate_up_weight = False
398 - for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
399 - if weight_name not in name:
400 - continue
401 - param = state_dict[name.replace(weight_name, "gate_up_proj")]
402 - if is_transposed:
403 - param = param.T
404 -
405 - shard_size = param.shape[0] // 2
406 - loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
407 - (tp_rank + 1)]
408 - param_slice = param.data[shard_size * stride_id:shard_size *
409 - (stride_id + 1)]
410 - assert param_slice.shape == loaded_weight.shape
411 - param_slice.copy_(loaded_weight)
412 - is_gate_up_weight = True
317 + param = params_dict[name.replace(weight_name, param_name)]
318 + weight_loader = param.weight_loader
319 + weight_loader(param, loaded_weight, shard_id)
320 break
414 - if is_gate_up_weight:
415 - continue
416 -
417 - param = state_dict[name]
418 - if is_transposed:
419 - param = param.T
420 -
421 - if "embed_tokens" in name or "lm_head" in name:
422 - load_padded_tensor_parallel_vocab(param, loaded_weight,
423 - tp_rank)
424 - continue
425 -
426 - load_tensor_parallel_weights(param, loaded_weight, name,
427 - column_parallel_weights,
428 - row_parallel_weights, tp_rank)
321 + else:
322 + param = params_dict[name]
323 + weight_loader = getattr(param, "weight_loader",
324 + default_weight_loader)
325 + weight_loader(param, loaded_weight)
vllm/model_executor/models/mistral.py
+62 -148
@@ -33,17 +33,19 @@ from transformers import MistralConfig
33
34 from vllm.model_executor.input_metadata import InputMetadata
35 from vllm.model_executor.layers.activation import SiluAndMul
36 -from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
37 +from vllm.model_executor.layers.layernorm import RMSNorm
38 +from vllm.model_executor.layers.linear import (LinearMethodBase,
39 + MergedColumnParallelLinear,
40 + QKVParallelLinear,
41 + RowParallelLinear)
42 from vllm.model_executor.layers.sampler import Sampler
39 -from vllm.model_executor.layers.quantized_linear import ParallelLinear
43 +from vllm.model_executor.layers.vocab_parallel_embedding import (
44 + VocabParallelEmbedding, ParallelLMHead)
45 from vllm.model_executor.parallel_utils.parallel_state import (
41 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43 -from vllm.model_executor.quantization_utils import QuantizationConfig
44 -from vllm.model_executor.weight_utils import (
45 - convert_pyslice_to_tensor, hf_model_weights_iterator,
46 - load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
46 + get_tensor_model_parallel_world_size)
47 +from vllm.model_executor.weight_utils import (default_weight_loader,
48 + hf_model_weights_iterator)
49 from vllm.sequence import SamplerOutput
50
51 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,19 +58,17 @@ class MistralMLP(nn.Module):
58 hidden_size: int,
59 intermediate_size: int,
60 hidden_act: str,
59 - quant_config: Optional[QuantizationConfig] = None,
61 + linear_method: Optional[LinearMethodBase] = None,
62 ) -> None:
63 super().__init__()
62 - self.gate_up_proj = ParallelLinear.column(hidden_size,
63 - 2 * intermediate_size,
64 - bias=False,
65 - gather_output=False,
66 - quant_config=quant_config)
67 - self.down_proj = ParallelLinear.row(intermediate_size,
68 - hidden_size,
69 - bias=False,
70 - input_is_parallel=True,
71 - quant_config=quant_config)
64 + self.gate_up_proj = MergedColumnParallelLinear(
65 + hidden_size, [intermediate_size] * 2,
66 + bias=False,
67 + linear_method=linear_method)
68 + self.down_proj = RowParallelLinear(intermediate_size,
69 + hidden_size,
70 + bias=False,
71 + linear_method=linear_method)
72 if hidden_act != "silu":
73 raise ValueError(f"Unsupported activation: {hidden_act}. "
74 "Only silu is supported for now.")
@@ -89,7 +89,7 @@ class MistralAttention(nn.Module):
89 num_kv_heads: int,
90 max_position: int = 4096 * 32,
91 rope_theta: float = 10000,
92 - quant_config: Optional[QuantizationConfig] = None,
92 + linear_method: Optional[LinearMethodBase] = None,
93 sliding_window: Optional[int] = None) -> None:
94 super().__init__()
95 self.hidden_size = hidden_size
@@ -98,8 +98,15 @@ class MistralAttention(nn.Module):
98 assert self.total_num_heads % tp_size == 0
99 self.num_heads = self.total_num_heads // tp_size
100 self.total_num_kv_heads = num_kv_heads
101 - assert self.total_num_kv_heads % tp_size == 0
102 - self.num_kv_heads = self.total_num_kv_heads // tp_size
101 + if self.total_num_kv_heads >= tp_size:
102 + # Number of KV heads is greater than TP size, so we partition
103 + # the KV heads across multiple tensor parallel GPUs.
104 + assert self.total_num_kv_heads % tp_size == 0
105 + else:
106 + # Number of KV heads is less than TP size, so we replicate
107 + # the KV heads across multiple tensor parallel GPUs.
108 + assert tp_size % self.total_num_kv_heads == 0
109 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
110 self.head_dim = hidden_size // self.total_num_heads
111 self.q_size = self.num_heads * self.head_dim
112 self.kv_size = self.num_kv_heads * self.head_dim
@@ -107,20 +114,19 @@ class MistralAttention(nn.Module):
114 self.rope_theta = rope_theta
115 self.sliding_window = sliding_window
116
110 - self.qkv_proj = ParallelLinear.column(
117 + self.qkv_proj = QKVParallelLinear(
118 hidden_size,
112 - (self.total_num_heads + 2 * self.total_num_kv_heads) *
119 self.head_dim,
120 + self.total_num_heads,
121 + self.total_num_kv_heads,
122 bias=False,
115 - gather_output=False,
116 - quant_config=quant_config,
123 + linear_method=linear_method,
124 )
118 - self.o_proj = ParallelLinear.row(
125 + self.o_proj = RowParallelLinear(
126 self.total_num_heads * self.head_dim,
127 hidden_size,
128 bias=False,
122 - input_is_parallel=True,
123 - quant_config=quant_config,
129 + linear_method=linear_method,
130 )
131 self.attn = PagedAttentionWithRoPE(self.num_heads,
132 self.head_dim,
@@ -153,7 +159,7 @@ class MistralDecoderLayer(nn.Module):
159 def __init__(
160 self,
161 config: MistralConfig,
156 - quant_config: Optional[QuantizationConfig] = None,
162 + linear_method: Optional[LinearMethodBase] = None,
163 ) -> None:
164 super().__init__()
165 self.hidden_size = config.hidden_size
@@ -165,13 +171,13 @@ class MistralDecoderLayer(nn.Module):
171 max_position=config.max_position_embeddings,
172 num_kv_heads=config.num_key_value_heads,
173 rope_theta=rope_theta,
168 - quant_config=quant_config,
174 + linear_method=linear_method,
175 sliding_window=config.sliding_window)
176 self.mlp = MistralMLP(
177 hidden_size=self.hidden_size,
178 intermediate_size=config.intermediate_size,
179 hidden_act=config.hidden_act,
174 - quant_config=quant_config,
180 + linear_method=linear_method,
181 )
182 self.input_layernorm = RMSNorm(config.hidden_size,
183 eps=config.rms_norm_eps)
@@ -211,20 +217,19 @@ class MistralModel(nn.Module):
217 def __init__(
218 self,
219 config: MistralConfig,
214 - quant_config: Optional[QuantizationConfig] = None,
220 + linear_method: Optional[LinearMethodBase] = None,
221 ) -> None:
222 super().__init__()
223 self.config = config
224 self.padding_idx = config.pad_token_id
225 self.vocab_size = config.vocab_size
226
221 - vocab_size = ((config.vocab_size + 63) // 64) * 64
227 self.embed_tokens = VocabParallelEmbedding(
223 - vocab_size,
228 + config.vocab_size,
229 config.hidden_size,
230 )
231 self.layers = nn.ModuleList([
227 - MistralDecoderLayer(config, quant_config)
232 + MistralDecoderLayer(config, linear_method)
233 for _ in range(config.num_hidden_layers)
234 ])
235 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -260,19 +265,13 @@ class MistralForCausalLM(nn.Module):
265 def __init__(
266 self,
267 config: MistralConfig,
263 - quant_config: Optional[QuantizationConfig] = None,
268 + linear_method: Optional[LinearMethodBase] = None,
269 ) -> None:
270 super().__init__()
271 self.config = config
267 - self.quant_config = quant_config
268 - self.model = MistralModel(config, quant_config)
269 - vocab_size = ((config.vocab_size + 63) // 64) * 64
270 - # NOTE: The LM head is not quantized.
271 - self.lm_head = ParallelLinear.column(config.hidden_size,
272 - vocab_size,
273 - bias=False,
274 - gather_output=False,
275 - quant_config=None)
272 + self.linear_method = linear_method
273 + self.model = MistralModel(config, linear_method)
274 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
275 self.sampler = Sampler(config.vocab_size)
276
277 def forward(
@@ -289,118 +288,33 @@ class MistralForCausalLM(nn.Module):
288 input_metadata)
289 return next_tokens
290
292 - _column_parallel_layers = []
293 - _row_parallel_layers = ["o_proj", "down_proj"]
294 -
291 def load_weights(self,
292 model_name_or_path: str,
293 cache_dir: Optional[str] = None,
294 load_format: str = "auto",
295 revision: Optional[str] = None):
300 - if self.quant_config is None:
301 - col_weight_suffixes = ["weight"]
302 - row_weight_suffixes = ["weight"]
303 - else:
304 - col_weight_suffixes = (
305 - self.quant_config.get_col_parallel_tensor_names())
306 - row_weight_suffixes = (
307 - self.quant_config.get_row_parallel_tensor_names())
308 -
309 - column_parallel_weights: List[str] = []
310 - for layer in self._column_parallel_layers:
311 - for suffix in col_weight_suffixes:
312 - column_parallel_weights.append(f"{layer}.{suffix}")
313 - row_parallel_weights: List[str] = []
314 - for layer in self._row_parallel_layers:
315 - for suffix in row_weight_suffixes:
316 - row_parallel_weights.append(f"{layer}.{suffix}")
317 -
318 - tp_size = get_tensor_model_parallel_world_size()
319 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
320 - q_proj_shard_size = (self.config.hidden_size // tp_size)
321 - kv_proj_shard_size = (self.config.hidden_size //
322 - self.config.num_attention_heads *
323 - self.config.num_key_value_heads // tp_size)
324 - attention_weight_specs = [
325 - # (weight_name, shard_size, offset)
326 - ("q_proj", q_proj_shard_size, 0),
327 - ("k_proj", kv_proj_shard_size, q_proj_shard_size),
328 - ("v_proj", kv_proj_shard_size,
329 - q_proj_shard_size + kv_proj_shard_size),
296 + stacked_params_mapping = [
297 + # (param_name, shard_name, shard_id)
298 + ("qkv_proj", "q_proj", "q"),
299 + ("qkv_proj", "k_proj", "k"),
300 + ("qkv_proj", "v_proj", "v"),
301 + ("gate_up_proj", "gate_proj", 0),
302 + ("gate_up_proj", "up_proj", 1),
303 ]
331 - state_dict = self.state_dict()
332 -
304 + params_dict = dict(self.named_parameters())
305 for name, loaded_weight in hf_model_weights_iterator(
306 model_name_or_path, cache_dir, load_format, revision):
307 if "rotary_emb.inv_freq" in name:
308 continue
337 -
338 - packed_dim = None
339 - is_transposed = False
340 - if self.quant_config is not None:
341 - packed_dim = self.quant_config.get_packed_dim(name)
342 - is_transposed = self.quant_config.is_transposed(name)
343 - if is_transposed:
344 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
345 - loaded_weight = loaded_weight.T
346 -
347 - is_attention_weight = False
348 - for weight_name, shard_size, offset in attention_weight_specs:
309 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
310 if weight_name not in name:
311 continue
351 - param = state_dict[name.replace(weight_name, "qkv_proj")]
352 - if is_transposed:
353 - param = param.T
354 -
355 - if packed_dim is not None:
356 - shard_dim = 0 if not is_transposed else 1
357 - if packed_dim == shard_dim:
358 - shard_size //= self.quant_config.pack_factor
359 - offset //= self.quant_config.pack_factor
360 -
361 - loaded_weight = loaded_weight[
362 - shard_size * tensor_model_parallel_rank:shard_size *
363 - (tensor_model_parallel_rank + 1)]
364 - param_slice = param.data[offset:offset + shard_size]
365 - assert param_slice.shape == loaded_weight.shape
366 -
367 - param_slice.copy_(loaded_weight)
368 - is_attention_weight = True
369 - break
370 - if is_attention_weight:
371 - continue
372 -
373 - is_gate_up_weight = False
374 - for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
375 - if weight_name not in name:
376 - continue
377 - param = state_dict[name.replace(weight_name, "gate_up_proj")]
378 - if is_transposed:
379 - param = param.T
380 -
381 - shard_size = param.shape[0] // 2
382 - loaded_weight = loaded_weight[
383 - shard_size * tensor_model_parallel_rank:shard_size *
384 - (tensor_model_parallel_rank + 1)]
385 - param_slice = param.data[shard_size * stride_id:shard_size *
386 - (stride_id + 1)]
387 - assert param_slice.shape == loaded_weight.shape
388 - param_slice.copy_(loaded_weight)
389 - is_gate_up_weight = True
312 + param = params_dict[name.replace(weight_name, param_name)]
313 + weight_loader = param.weight_loader
314 + weight_loader(param, loaded_weight, shard_id)
315 break
391 - if is_gate_up_weight:
392 - continue
393 -
394 - param = state_dict[name]
395 - if is_transposed:
396 - param = param.T
397 -
398 - if "embed_tokens" in name or "lm_head" in name:
399 - load_padded_tensor_parallel_vocab(param, loaded_weight,
400 - tensor_model_parallel_rank)
401 - continue
402 -
403 - load_tensor_parallel_weights(param, loaded_weight, name,
404 - column_parallel_weights,
405 - row_parallel_weights,
406 - tensor_model_parallel_rank)
316 + else:
317 + param = params_dict[name]
318 + weight_loader = getattr(param, "weight_loader",
319 + default_weight_loader)
320 + weight_loader(param, loaded_weight)
vllm/model_executor/models/mpt.py
+52 -59
@@ -10,15 +10,17 @@ from transformers import MptConfig
10 from vllm.model_executor.input_metadata import InputMetadata
11 from vllm.model_executor.layers.activation import get_act_fn
12 from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
13 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 + LinearMethodBase,
15 + QKVParallelLinear,
16 + RowParallelLinear)
17 from vllm.model_executor.layers.sampler import Sampler
14 -from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
15 - hf_model_weights_iterator,
16 - load_tensor_parallel_weights)
18 +from vllm.model_executor.layers.vocab_parallel_embedding import (
19 + VocabParallelEmbedding)
20 from vllm.model_executor.parallel_utils.parallel_state import (
21 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
19 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
20 - ColumnParallelLinear,
21 - RowParallelLinear)
22 +from vllm.model_executor.weight_utils import (default_weight_loader,
23 + hf_model_weights_iterator)
24 from vllm.sequence import SamplerOutput
25
26 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -39,7 +41,11 @@ def _get_alibi_slopes(
41
42 class MptAttention(nn.Module):
43
42 - def __init__(self, config: MptConfig):
44 + def __init__(
45 + self,
46 + config: MptConfig,
47 + linear_method: Optional[LinearMethodBase] = None,
48 + ):
49 super().__init__()
50 self.d_model = config.d_model
51 self.total_num_heads = config.n_heads
@@ -49,11 +55,13 @@ class MptAttention(nn.Module):
55 assert not config.attn_config.prefix_lm
56 assert config.attn_config.alibi
57
52 - self.qkv_proj = ColumnParallelLinear(
58 + # pylint: disable=invalid-name
59 + self.Wqkv = QKVParallelLinear(
60 self.d_model,
54 - 3 * self.d_model,
61 + self.d_model // self.total_num_heads,
62 + self.total_num_heads,
63 bias=not config.no_bias,
56 - gather_output=False,
64 + linear_method=linear_method,
65 )
66 if self.qk_ln:
67 self.q_ln = nn.LayerNorm(self.d_model)
@@ -62,7 +70,7 @@ class MptAttention(nn.Module):
70 self.d_model,
71 self.d_model,
72 bias=not config.no_bias,
65 - input_is_parallel=True,
73 + linear_method=linear_method,
74 )
75
76 tp_world_size = get_tensor_model_parallel_world_size()
@@ -91,7 +99,7 @@ class MptAttention(nn.Module):
99 cache_event: Optional[torch.cuda.Event],
100 ) -> torch.Tensor:
101 del position_ids # unused.
94 - qkv, _ = self.qkv_proj(hidden_states)
102 + qkv, _ = self.Wqkv(hidden_states)
103 if self.clip_qkv is not None:
104 qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
105 q, k, v = qkv.chunk(chunks=3, dim=-1)
@@ -107,7 +115,11 @@ class MptAttention(nn.Module):
115
116 class MptMLP(nn.Module):
117
110 - def __init__(self, config: MptConfig):
118 + def __init__(
119 + self,
120 + config: MptConfig,
121 + linear_method: Optional[LinearMethodBase] = None,
122 + ):
123 super().__init__()
124 hidden_size = config.d_model
125 expansion_ratio = config.expansion_ratio
@@ -116,14 +128,14 @@ class MptMLP(nn.Module):
128 hidden_size,
129 intermediate_size,
130 bias=not config.no_bias,
119 - gather_output=False,
131 + linear_method=linear_method,
132 )
133 self.act = get_act_fn("gelu")
134 self.down_proj = RowParallelLinear(
135 intermediate_size,
136 hidden_size,
137 bias=not config.no_bias,
126 - input_is_parallel=True,
138 + linear_method=linear_method,
139 )
140
141 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -135,13 +147,17 @@ class MptMLP(nn.Module):
147
148 class MptBlock(nn.Module):
149
138 - def __init__(self, config: MptConfig):
150 + def __init__(
151 + self,
152 + config: MptConfig,
153 + linear_method: Optional[LinearMethodBase] = None,
154 + ):
155 super().__init__()
156 hidden_size = config.d_model
157 self.norm_1 = nn.LayerNorm(hidden_size)
142 - self.attn = MptAttention(config)
158 + self.attn = MptAttention(config, linear_method)
159 self.norm_2 = nn.LayerNorm(hidden_size)
144 - self.ffn = MptMLP(config)
160 + self.ffn = MptMLP(config, linear_method)
161
162 def forward(
163 self,
@@ -168,7 +184,11 @@ class MptBlock(nn.Module):
184
185 class MptModel(nn.Module):
186
171 - def __init__(self, config: MptConfig):
187 + def __init__(
188 + self,
189 + config: MptConfig,
190 + linear_method: Optional[LinearMethodBase] = None,
191 + ):
192 super().__init__()
193 assert config.embedding_fraction == 1.0
194 assert config.norm_type == "low_precision_layernorm"
@@ -178,7 +198,7 @@ class MptModel(nn.Module):
198 config.d_model,
199 )
200 self.blocks = nn.ModuleList(
181 - [MptBlock(config) for _ in range(config.n_layers)])
201 + [MptBlock(config, linear_method) for _ in range(config.n_layers)])
202 self.norm_f = nn.LayerNorm(config.d_model)
203 if config.no_bias:
204 for module in self.modules():
@@ -215,14 +235,17 @@ class MptModel(nn.Module):
235
236 class MptForCausalLM(nn.Module):
237
218 - def __init__(self, config: MptConfig):
238 + def __init__(
239 + self,
240 + config: MptConfig,
241 + linear_method: Optional[LinearMethodBase] = None,
242 + ):
243 super().__init__()
244 self.config = config
245 assert config.tie_word_embeddings
246 + self.linear_method = linear_method
247
223 - self.transformer = MptModel(config)
224 - # TODO(zhuohan): create a new weight after implementing pipeline
225 - # parallelism
248 + self.transformer = MptModel(config, linear_method)
249 self.lm_head_weight = self.transformer.wte.weight
250 self.sampler = Sampler(config.vocab_size)
251
@@ -240,45 +263,15 @@ class MptForCausalLM(nn.Module):
263 input_metadata)
264 return next_tokens
265
243 - _column_parallel_weights = ["wte.weight", "up_proj.weight", "up_proj.bias"]
244 - _row_parallel_weights = ["out_proj.weight", "down_proj.weight"]
245 -
266 def load_weights(self,
267 model_name_or_path: str,
268 cache_dir: Optional[str] = None,
269 load_format: str = "auto",
270 revision: Optional[str] = None):
251 - tp_world_size = get_tensor_model_parallel_world_size()
252 - tp_rank = get_tensor_model_parallel_rank()
253 - state_dict = self.state_dict()
271 + params_dict = dict(self.named_parameters(remove_duplicate=False))
272 for name, loaded_weight in hf_model_weights_iterator(
273 model_name_or_path, cache_dir, load_format, revision):
256 - if "Wqkv" in name:
257 - # NOTE(woosuk): MPT's fused QKV has the shape of
258 - # [3 * num_heads * head_size, hidden_size].
259 - # When tensor model parallelism is used, we need to shard
260 - # the weight along the hidden dimension.
261 - total_num_heads = self.config.num_attention_heads
262 - hidden_size = self.config.hidden_size
263 - head_size = hidden_size // total_num_heads
264 - num_heads = total_num_heads // tp_world_size
265 - head_start = tp_rank * num_heads
266 - head_end = (tp_rank + 1) * num_heads
267 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
268 - if name.endswith(".weight"):
269 - loaded_weight = loaded_weight.view(3, total_num_heads,
270 - head_size, hidden_size)
271 - loaded_weight = loaded_weight[:, head_start:head_end, :, :]
272 - loaded_weight = loaded_weight.reshape(-1, hidden_size)
273 - elif name.endswith(".bias"):
274 - loaded_weight = loaded_weight.view(3, total_num_heads,
275 - head_size)
276 - loaded_weight = loaded_weight[:, head_start:head_end, :]
277 - loaded_weight = loaded_weight.reshape(-1)
278 - else:
279 - raise ValueError(f"Unexpected parameter name {name}")
280 - name = name.replace("Wqkv", "qkv_proj")
281 - param = state_dict[name]
282 - load_tensor_parallel_weights(param, loaded_weight, name,
283 - self._column_parallel_weights,
284 - self._row_parallel_weights, tp_rank)
274 + param = params_dict[name]
275 + weight_loader = getattr(param, "weight_loader",
276 + default_weight_loader)
277 + weight_loader(param, loaded_weight)
vllm/model_executor/models/opt.py
+71 -62
@@ -30,14 +30,18 @@ from transformers import OPTConfig
30 from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import get_act_fn
32 from vllm.model_executor.layers.attention import PagedAttention
33 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 + LinearMethodBase,
35 + QKVParallelLinear,
36 + ReplicatedLinear,
37 + RowParallelLinear)
38 from vllm.model_executor.layers.sampler import Sampler
34 -from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35 - load_tensor_parallel_weights)
39 +from vllm.model_executor.layers.vocab_parallel_embedding import (
40 + VocabParallelEmbedding)
41 from vllm.model_executor.parallel_utils.parallel_state import (
37 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 -from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39 - ColumnParallelLinear,
40 - RowParallelLinear)
42 + get_tensor_model_parallel_world_size)
43 +from vllm.model_executor.weight_utils import (default_weight_loader,
44 + hf_model_weights_iterator)
45 from vllm.sequence import SamplerOutput
46
47 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -63,6 +67,7 @@ class OPTAttention(nn.Module):
67 embed_dim: int,
68 num_heads: int,
69 bias: bool = True,
70 + linear_method: Optional[LinearMethodBase] = None,
71 ) -> None:
72 super().__init__()
73 self.embed_dim = embed_dim
@@ -74,17 +79,18 @@ class OPTAttention(nn.Module):
79 self.head_dim = embed_dim // total_num_heads
80 self.scaling = self.head_dim**-0.5
81
77 - self.qkv_proj = ColumnParallelLinear(
82 + self.qkv_proj = QKVParallelLinear(
83 embed_dim,
79 - 3 * embed_dim,
84 + self.head_dim,
85 + total_num_heads,
86 bias=bias,
81 - gather_output=False,
87 + linear_method=linear_method,
88 )
89 self.out_proj = RowParallelLinear(
90 embed_dim,
91 embed_dim,
92 bias=bias,
87 - input_is_parallel=True,
93 + linear_method=linear_method,
94 )
95 self.attn = PagedAttention(self.num_heads,
96 self.head_dim,
@@ -108,7 +114,11 @@ class OPTAttention(nn.Module):
114
115 class OPTDecoderLayer(nn.Module):
116
111 - def __init__(self, config: OPTConfig):
117 + def __init__(
118 + self,
119 + config: OPTConfig,
120 + linear_method: Optional[LinearMethodBase] = None,
121 + ):
122 super().__init__()
123 self.config = config
124 self.embed_dim = config.hidden_size
@@ -116,6 +126,7 @@ class OPTDecoderLayer(nn.Module):
126 embed_dim=self.embed_dim,
127 num_heads=config.num_attention_heads,
128 bias=config.enable_bias,
129 + linear_method=linear_method,
130 )
131 self.do_layer_norm_before = config.do_layer_norm_before
132 self.activation_fn = get_act_fn(config.activation_function)
@@ -127,13 +138,13 @@ class OPTDecoderLayer(nn.Module):
138 self.embed_dim,
139 config.ffn_dim,
140 bias=config.enable_bias,
130 - gather_output=False,
141 + linear_method=linear_method,
142 )
143 self.fc2 = RowParallelLinear(
144 config.ffn_dim,
145 self.embed_dim,
146 bias=config.enable_bias,
136 - input_is_parallel=True,
147 + linear_method=linear_method,
148 )
149 self.final_layer_norm = nn.LayerNorm(
150 self.embed_dim,
@@ -177,7 +188,11 @@ class OPTDecoderLayer(nn.Module):
188
189 class OPTDecoder(nn.Module):
190
180 - def __init__(self, config: OPTConfig):
191 + def __init__(
192 + self,
193 + config: OPTConfig,
194 + linear_method: Optional[LinearMethodBase] = None,
195 + ):
196 super().__init__()
197 self.config = config
198 self.padding_idx = config.pad_token_id
@@ -194,16 +209,18 @@ class OPTDecoder(nn.Module):
209
210 # Project out & in will be replicated if they exist.
211 if config.word_embed_proj_dim != config.hidden_size:
197 - self.project_out = nn.Linear(config.hidden_size,
198 - config.word_embed_proj_dim,
199 - bias=False)
212 + self.project_out = ReplicatedLinear(config.hidden_size,
213 + config.word_embed_proj_dim,
214 + bias=False,
215 + linear_method=linear_method)
216 else:
217 self.project_out = None
218
219 if config.word_embed_proj_dim != config.hidden_size:
204 - self.project_in = nn.Linear(config.word_embed_proj_dim,
205 - config.hidden_size,
206 - bias=False)
220 + self.project_in = ReplicatedLinear(config.word_embed_proj_dim,
221 + config.hidden_size,
222 + bias=False,
223 + linear_method=linear_method)
224 else:
225 self.project_in = None
226
@@ -218,8 +235,10 @@ class OPTDecoder(nn.Module):
235 else:
236 self.final_layer_norm = None
237
221 - self.layers = nn.ModuleList(
222 - [OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
238 + self.layers = nn.ModuleList([
239 + OPTDecoderLayer(config, linear_method)
240 + for _ in range(config.num_hidden_layers)
241 + ])
242
243 def forward(
244 self,
@@ -253,9 +272,13 @@ class OPTDecoder(nn.Module):
272
273 class OPTModel(nn.Module):
274
256 - def __init__(self, config: OPTConfig):
275 + def __init__(
276 + self,
277 + config: OPTConfig,
278 + linear_method: Optional[LinearMethodBase] = None,
279 + ):
280 super().__init__()
258 - self.decoder = OPTDecoder(config)
281 + self.decoder = OPTDecoder(config, linear_method)
282
283 def forward(
284 self,
@@ -271,12 +294,15 @@ class OPTModel(nn.Module):
294
295 class OPTForCausalLM(nn.Module):
296
274 - def __init__(self, config):
297 + def __init__(
298 + self,
299 + config,
300 + linear_method: Optional[LinearMethodBase] = None,
301 + ):
302 super().__init__()
303 self.config = config
277 - self.model = OPTModel(config)
278 - # TODO(zhuohan): create a new weight after implementing pipeline
279 - # parallelism
304 + self.linear_method = linear_method
305 + self.model = OPTModel(config, linear_method)
306 self.lm_head_weight = self.model.decoder.embed_tokens.weight
307 self.sampler = Sampler(config.vocab_size)
308
@@ -294,48 +320,31 @@ class OPTForCausalLM(nn.Module):
320 input_metadata)
321 return next_tokens
322
297 - _column_parallel_weights = [
298 - "embed_tokens.weight", "fc1.weight", "fc1.bias"
299 - ]
300 - _row_parallel_weights = ["out_proj.weight", "fc2.weight"]
301 -
323 def load_weights(self,
324 model_name_or_path: str,
325 cache_dir: Optional[str] = None,
326 load_format: str = "auto",
327 revision: Optional[str] = None):
307 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
308 - state_dict = self.state_dict()
309 -
328 + stacked_params_mapping = [
329 + # (param_name, shard_name, shard_id)
330 + ("qkv_proj", "q_proj", "q"),
331 + ("qkv_proj", "k_proj", "k"),
332 + ("qkv_proj", "v_proj", "v"),
333 + ]
334 + params_dict = dict(self.named_parameters(remove_duplicate=False))
335 for name, loaded_weight in hf_model_weights_iterator(
336 model_name_or_path, cache_dir, load_format, revision):
337 if "lm_head.weight" in name:
338 continue
314 -
315 - if name.startswith("decoder."):
316 - name = "model." + name
317 -
318 - is_attention_weight = False
319 - for stride_id, att_weight_name in enumerate(
320 - ["q_proj", "k_proj", "v_proj"]):
321 - if att_weight_name not in name:
339 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
340 + if weight_name not in name:
341 continue
323 - param = state_dict[name.replace(att_weight_name, "qkv_proj")]
324 - shard_size = param.shape[0] // 3
325 - loaded_weight = loaded_weight[
326 - shard_size * tensor_model_parallel_rank:shard_size *
327 - (tensor_model_parallel_rank + 1)]
328 - param_slice = param.data[shard_size * stride_id:shard_size *
329 - (stride_id + 1)]
330 - assert param_slice.shape == loaded_weight.shape
331 - param_slice.copy_(loaded_weight)
332 - is_attention_weight = True
342 + param = params_dict[name.replace(weight_name, param_name)]
343 + weight_loader = param.weight_loader
344 + weight_loader(param, loaded_weight, shard_id)
345 break
334 - if is_attention_weight:
335 - continue
336 -
337 - param = state_dict[name]
338 - load_tensor_parallel_weights(param, loaded_weight, name,
339 - self._column_parallel_weights,
340 - self._row_parallel_weights,
341 - tensor_model_parallel_rank)
346 + else:
347 + param = params_dict[name]
348 + weight_loader = getattr(param, "weight_loader",
349 + default_weight_loader)
350 + weight_loader(param, loaded_weight)
vllm/model_executor/models/qwen.py
+80 -119
@@ -15,24 +15,19 @@ from torch import nn
15
16 from vllm.model_executor.input_metadata import InputMetadata
17 from vllm.model_executor.layers.activation import SiluAndMul
18 -from vllm.model_executor.layers.layernorm import RMSNorm
18 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
19 +from vllm.model_executor.layers.layernorm import RMSNorm
20 +from vllm.model_executor.layers.linear import (LinearMethodBase,
21 + MergedColumnParallelLinear,
22 + QKVParallelLinear,
23 + RowParallelLinear)
24 from vllm.model_executor.layers.sampler import Sampler
21 -from vllm.model_executor.weight_utils import (
22 - convert_pyslice_to_tensor,
23 - hf_model_weights_iterator,
24 - load_padded_tensor_parallel_vocab,
25 - load_tensor_parallel_weights,
26 -)
25 +from vllm.model_executor.layers.vocab_parallel_embedding import (
26 + VocabParallelEmbedding, ParallelLMHead)
27 from vllm.model_executor.parallel_utils.parallel_state import (
28 - get_tensor_model_parallel_rank,
29 - get_tensor_model_parallel_world_size,
30 -)
31 -from vllm.model_executor.parallel_utils.layers import (
32 - VocabParallelEmbedding,
33 - ColumnParallelLinear,
34 - RowParallelLinear,
35 -)
28 + get_tensor_model_parallel_world_size)
29 +from vllm.model_executor.weight_utils import (default_weight_loader,
30 + hf_model_weights_iterator)
31 from vllm.sequence import SamplerOutput
32 from vllm.transformers_utils.configs.qwen import QWenConfig
33
@@ -46,20 +41,17 @@ class QWenMLP(nn.Module):
41 hidden_size: int,
42 intermediate_size: int,
43 hidden_act: str = "silu",
44 + linear_method: Optional[LinearMethodBase] = None,
45 ):
46 super().__init__()
51 - self.gate_up_proj = ColumnParallelLinear(
52 - hidden_size,
53 - 2 * intermediate_size,
54 - bias=False,
55 - gather_output=False,
56 - )
57 - self.c_proj = RowParallelLinear(
58 - intermediate_size,
59 - hidden_size,
47 + self.gate_up_proj = MergedColumnParallelLinear(
48 + hidden_size, [intermediate_size] * 2,
49 bias=False,
61 - input_is_parallel=True,
62 - )
50 + linear_method=linear_method)
51 + self.c_proj = RowParallelLinear(intermediate_size,
52 + hidden_size,
53 + bias=False,
54 + linear_method=linear_method)
55 if hidden_act != "silu":
56 raise ValueError(f"Unsupported activation: {hidden_act}. "
57 "Only silu is supported for now.")
@@ -74,12 +66,15 @@ class QWenMLP(nn.Module):
66
67 class QWenAttention(nn.Module):
68
77 - def __init__(self,
78 - hidden_size: int,
79 - num_heads: int,
80 - max_position_embeddings: int,
81 - rope_theta: float = 10000,
82 - rope_scaling: Optional[Dict[str, Any]] = None):
69 + def __init__(
70 + self,
71 + hidden_size: int,
72 + num_heads: int,
73 + max_position_embeddings: int,
74 + rope_theta: float = 10000,
75 + rope_scaling: Optional[Dict[str, Any]] = None,
76 + linear_method: Optional[LinearMethodBase] = None,
77 + ):
78 super().__init__()
79 self.hidden_size = hidden_size
80 tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
@@ -90,18 +85,18 @@ class QWenAttention(nn.Module):
85 tensor_model_parallel_world_size)
86 self.head_dim = hidden_size // self.total_num_heads
87
93 - # pylint: disable=invalid-name
94 - self.c_attn = ColumnParallelLinear(
88 + self.c_attn = QKVParallelLinear(
89 hidden_size,
96 - 3 * hidden_size,
90 + self.head_dim,
91 + self.total_num_heads,
92 bias=True,
98 - gather_output=False,
93 + linear_method=linear_method,
94 )
95 self.c_proj = RowParallelLinear(
96 self.total_num_heads * self.head_dim,
97 hidden_size,
98 bias=False,
104 - input_is_parallel=True,
99 + linear_method=linear_method,
100 )
101 self.scaling = self.head_dim**-0.5
102 self.attn = PagedAttentionWithRoPE(
@@ -134,7 +129,11 @@ class QWenAttention(nn.Module):
129
130 class QWenBlock(nn.Module):
131
137 - def __init__(self, config: QWenConfig):
132 + def __init__(
133 + self,
134 + config: QWenConfig,
135 + linear_method: Optional[LinearMethodBase] = None,
136 + ):
137 super().__init__()
138 self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
139
@@ -144,11 +143,14 @@ class QWenBlock(nn.Module):
143 config.num_attention_heads,
144 config.max_position_embeddings,
145 rope_theta=rope_theta,
147 - rope_scaling=rope_scaling)
146 + rope_scaling=rope_scaling,
147 + linear_method=linear_method)
148
149 self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
150
151 - self.mlp = QWenMLP(config.hidden_size, config.intermediate_size // 2)
151 + self.mlp = QWenMLP(config.hidden_size,
152 + config.intermediate_size // 2,
153 + linear_method=linear_method)
154
155 def forward(
156 self,
@@ -180,18 +182,23 @@ class QWenBlock(nn.Module):
182
183 class QWenModel(nn.Module):
184
183 - def __init__(self, config: QWenConfig):
185 + def __init__(
186 + self,
187 + config: QWenConfig,
188 + linear_method: Optional[LinearMethodBase] = None,
189 + ):
190 super().__init__()
191 self.config = config
192 self.vocab_size = config.vocab_size
193
188 - vocab_size = ((config.vocab_size + 63) // 64) * 64
194 self.wte = VocabParallelEmbedding(
190 - vocab_size,
195 + config.vocab_size,
196 config.hidden_size,
197 )
193 - self.h = nn.ModuleList(
194 - [QWenBlock(config) for _ in range(config.num_hidden_layers)])
198 + self.h = nn.ModuleList([
199 + QWenBlock(config, linear_method)
200 + for _ in range(config.num_hidden_layers)
201 + ])
202 self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
203
204 def forward(
@@ -222,17 +229,16 @@ class QWenModel(nn.Module):
229
230 class QWenLMHeadModel(nn.Module):
231
225 - def __init__(self, config: QWenConfig):
232 + def __init__(
233 + self,
234 + config: QWenConfig,
235 + linear_method: Optional[LinearMethodBase] = None,
236 + ):
237 super().__init__()
238 self.config = config
228 - self.transformer = QWenModel(config)
229 - vocab_size = ((config.vocab_size + 63) // 64) * 64
230 - self.lm_head = ColumnParallelLinear(
231 - config.hidden_size,
232 - vocab_size,
233 - bias=False,
234 - gather_output=False,
235 - )
239 + self.linear_method = linear_method
240 + self.transformer = QWenModel(config, linear_method)
241 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
242 self.sampler = Sampler(config.vocab_size)
243
244 def forward(
@@ -249,75 +255,30 @@ class QWenLMHeadModel(nn.Module):
255 input_metadata)
256 return next_tokens
257
252 - _column_parallel_weights = []
253 - _row_parallel_weights = ["c_proj.weight"]
254 -
255 - def load_weights(
256 - self,
257 - model_name_or_path: str,
258 - cache_dir: Optional[str] = None,
259 - load_format: str = "auto",
260 - revision: Optional[str] = None,
261 - ):
262 - tp_world_size = get_tensor_model_parallel_world_size()
263 - tp_rank = get_tensor_model_parallel_rank()
264 - state_dict = self.state_dict()
265 -
258 + def load_weights(self,
259 + model_name_or_path: str,
260 + cache_dir: Optional[str] = None,
261 + load_format: str = "auto",
262 + revision: Optional[str] = None):
263 + stacked_params_mapping = [
264 + # (param_name, shard_name, shard_id)
265 + ("gate_up_proj", "w2", 0),
266 + ("gate_up_proj", "w1", 1),
267 + ]
268 + params_dict = dict(self.named_parameters())
269 for name, loaded_weight in hf_model_weights_iterator(
270 model_name_or_path, cache_dir, load_format, revision):
271 if "rotary_emb.inv_freq" in name:
272 continue
270 -
271 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
272 -
273 - if "c_attn" in name:
274 - total_num_heads = self.config.num_attention_heads
275 - hidden_size = self.config.hidden_size
276 - head_size = hidden_size // total_num_heads
277 - num_heads = total_num_heads // tp_world_size
278 - head_start = tp_rank * num_heads
279 - head_end = (tp_rank + 1) * num_heads
280 -
281 - if "weight" in name:
282 - loaded_weight = loaded_weight.view(3, total_num_heads,
283 - head_size, hidden_size)
284 - loaded_weight = loaded_weight[:, head_start:head_end, :, :]
285 - loaded_weight = loaded_weight.reshape(-1, hidden_size)
286 - elif "bias" in name:
287 - loaded_weight = loaded_weight.view(3, total_num_heads,
288 - head_size)
289 - loaded_weight = loaded_weight[:, head_start:head_end, :]
290 - loaded_weight = loaded_weight.reshape(-1)
291 -
292 - is_gate_up_weight = False
293 - for stride_id, weight_name in enumerate(["w2", "w1"]):
273 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
274 if weight_name not in name:
275 continue
296 - param = state_dict[name.replace(weight_name, "gate_up_proj")]
297 - shard_size = param.shape[0] // 2
298 - loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
299 - (tp_rank + 1)]
300 - param_slice = param.data[shard_size * stride_id:shard_size *
301 - (stride_id + 1)]
302 - assert param_slice.shape == loaded_weight.shape
303 - param_slice.copy_(loaded_weight)
304 - is_gate_up_weight = True
276 + param = params_dict[name.replace(weight_name, param_name)]
277 + weight_loader = param.weight_loader
278 + weight_loader(param, loaded_weight, shard_id)
279 break
306 - if is_gate_up_weight:
307 - continue
308 -
309 - param = state_dict[name]
310 -
311 - if "wte" in name or "lm_head" in name:
312 - load_padded_tensor_parallel_vocab(param, loaded_weight,
313 - tp_rank)
314 - continue
315 -
316 - load_tensor_parallel_weights(
317 - param,
318 - loaded_weight,
319 - name,
320 - self._column_parallel_weights,
321 - self._row_parallel_weights,
322 - tp_rank,
323 - )
280 + else:
281 + param = params_dict[name]
282 + weight_loader = getattr(param, "weight_loader",
283 + default_weight_loader)
284 + weight_loader(param, loaded_weight)
vllm/model_executor/models/yi.py
+53 -156
@@ -33,17 +33,19 @@ from vllm.transformers_utils.configs.yi import YiConfig
33
34 from vllm.model_executor.input_metadata import InputMetadata
35 from vllm.model_executor.layers.activation import SiluAndMul
36 -from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
37 +from vllm.model_executor.layers.layernorm import RMSNorm
38 +from vllm.model_executor.layers.linear import (LinearMethodBase,
39 + MergedColumnParallelLinear,
40 + QKVParallelLinear,
41 + RowParallelLinear)
42 from vllm.model_executor.layers.sampler import Sampler
39 -from vllm.model_executor.layers.quantized_linear import ParallelLinear
43 +from vllm.model_executor.layers.vocab_parallel_embedding import (
44 + VocabParallelEmbedding, ParallelLMHead)
45 from vllm.model_executor.parallel_utils.parallel_state import (
41 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43 -from vllm.model_executor.quantization_utils import QuantizationConfig
44 -from vllm.model_executor.weight_utils import (
45 - convert_pyslice_to_tensor, hf_model_weights_iterator,
46 - load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
46 + get_tensor_model_parallel_world_size)
47 +from vllm.model_executor.weight_utils import (default_weight_loader,
48 + hf_model_weights_iterator)
49 from vllm.sequence import SamplerOutput
50
51 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,19 +58,17 @@ class YiMLP(nn.Module):
58 hidden_size: int,
59 intermediate_size: int,
60 hidden_act: str,
59 - quant_config: Optional[QuantizationConfig] = None,
61 + linear_method: Optional[LinearMethodBase] = None,
62 ) -> None:
63 super().__init__()
62 - self.gate_up_proj = ParallelLinear.column(hidden_size,
63 - 2 * intermediate_size,
64 - bias=False,
65 - gather_output=False,
66 - quant_config=quant_config)
67 - self.down_proj = ParallelLinear.row(intermediate_size,
68 - hidden_size,
69 - bias=False,
70 - input_is_parallel=True,
71 - quant_config=quant_config)
64 + self.gate_up_proj = MergedColumnParallelLinear(
65 + hidden_size, [intermediate_size] * 2,
66 + bias=False,
67 + linear_method=linear_method)
68 + self.down_proj = RowParallelLinear(intermediate_size,
69 + hidden_size,
70 + bias=False,
71 + linear_method=linear_method)
72 if hidden_act != "silu":
73 raise ValueError(f"Unsupported activation: {hidden_act}. "
74 "Only silu is supported for now.")
@@ -91,7 +91,7 @@ class YiAttention(nn.Module):
91 rope_theta: float = 10000,
92 rope_scaling: Optional[Dict[str, Any]] = None,
93 max_position_embeddings: int = 8192,
94 - quant_config: Optional[QuantizationConfig] = None,
94 + linear_method: Optional[LinearMethodBase] = None,
95 ) -> None:
96 super().__init__()
97 self.hidden_size = hidden_size
@@ -109,7 +109,6 @@ class YiAttention(nn.Module):
109 # the KV heads across multiple tensor parallel GPUs.
110 assert tp_size % self.total_num_kv_heads == 0
111 self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
112 - num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
112 self.head_dim = hidden_size // self.total_num_heads
113 self.q_size = self.num_heads * self.head_dim
114 self.kv_size = self.num_kv_heads * self.head_dim
@@ -117,21 +116,19 @@ class YiAttention(nn.Module):
116 self.rope_theta = rope_theta
117 self.max_position_embeddings = max_position_embeddings
118
120 - self.qkv_proj = ParallelLinear.column(
119 + self.qkv_proj = QKVParallelLinear(
120 hidden_size,
122 - (self.total_num_heads +
123 - 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
121 self.head_dim,
122 + self.total_num_heads,
123 + self.total_num_kv_heads,
124 bias=False,
126 - gather_output=False,
127 - quant_config=quant_config,
125 + linear_method=linear_method,
126 )
129 - self.o_proj = ParallelLinear.row(
127 + self.o_proj = RowParallelLinear(
128 self.total_num_heads * self.head_dim,
129 hidden_size,
130 bias=False,
133 - input_is_parallel=True,
134 - quant_config=quant_config,
131 + linear_method=linear_method,
132 )
133 self.attn = PagedAttentionWithRoPE(
134 self.num_heads,
@@ -165,11 +162,10 @@ class YiDecoderLayer(nn.Module):
162 def __init__(
163 self,
164 config: YiConfig,
168 - quant_config: Optional[QuantizationConfig] = None,
165 + linear_method: Optional[LinearMethodBase] = None,
166 ) -> None:
167 super().__init__()
168 self.hidden_size = config.hidden_size
172 - # Requires transformers > 4.32.0
169 rope_theta = getattr(config, "rope_theta", 10000)
170 rope_scaling = getattr(config, "rope_scaling", None)
171 max_position_embeddings = getattr(config, "max_position_embeddings",
@@ -181,13 +177,13 @@ class YiDecoderLayer(nn.Module):
177 rope_theta=rope_theta,
178 rope_scaling=rope_scaling,
179 max_position_embeddings=max_position_embeddings,
184 - quant_config=quant_config,
180 + linear_method=linear_method,
181 )
182 self.mlp = YiMLP(
183 hidden_size=self.hidden_size,
184 intermediate_size=config.intermediate_size,
185 hidden_act=config.hidden_act,
190 - quant_config=quant_config,
186 + linear_method=linear_method,
187 )
188 self.ln1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
189 self.ln2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -225,20 +221,18 @@ class YiModel(nn.Module):
221 def __init__(
222 self,
223 config: YiConfig,
228 - quant_config: Optional[QuantizationConfig] = None,
224 + linear_method: Optional[LinearMethodBase] = None,
225 ) -> None:
226 super().__init__()
227 self.config = config
228 self.padding_idx = config.pad_token_id
229 self.vocab_size = config.vocab_size
234 -
235 - vocab_size = ((config.vocab_size + 63) // 64) * 64
230 self.embed_tokens = VocabParallelEmbedding(
237 - vocab_size,
231 + config.vocab_size,
232 config.hidden_size,
233 )
234 self.layers = nn.ModuleList([
241 - YiDecoderLayer(config, quant_config)
235 + YiDecoderLayer(config, linear_method)
236 for _ in range(config.num_hidden_layers)
237 ])
238 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -274,19 +268,13 @@ class YiForCausalLM(nn.Module):
268 def __init__(
269 self,
270 config: YiConfig,
277 - quant_config: Optional[QuantizationConfig] = None,
271 + linear_method: Optional[LinearMethodBase] = None,
272 ) -> None:
273 super().__init__()
274 self.config = config
281 - self.quant_config = quant_config
282 - self.model = YiModel(config, quant_config)
283 - vocab_size = ((config.vocab_size + 63) // 64) * 64
284 - # NOTE: The LM head is not quantized.
285 - self.lm_head = ParallelLinear.column(config.hidden_size,
286 - vocab_size,
287 - bias=False,
288 - gather_output=False,
289 - quant_config=None)
275 + self.linear_method = linear_method
276 + self.model = YiModel(config, linear_method)
277 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
278 self.sampler = Sampler(config.vocab_size)
279
280 def forward(
@@ -303,124 +291,33 @@ class YiForCausalLM(nn.Module):
291 input_metadata)
292 return next_tokens
293
306 - _column_parallel_layers = []
307 - _row_parallel_layers = ["o_proj", "down_proj"]
308 -
294 def load_weights(self,
295 model_name_or_path: str,
296 cache_dir: Optional[str] = None,
297 load_format: str = "auto",
298 revision: Optional[str] = None):
314 - if self.quant_config is None:
315 - col_weight_suffixes = ["weight"]
316 - row_weight_suffixes = ["weight"]
317 - else:
318 - col_weight_suffixes = (
319 - self.quant_config.get_col_parallel_tensor_names())
320 - row_weight_suffixes = (
321 - self.quant_config.get_row_parallel_tensor_names())
322 -
323 - column_parallel_weights: List[str] = []
324 - for layer in self._column_parallel_layers:
325 - for suffix in col_weight_suffixes:
326 - column_parallel_weights.append(f"{layer}.{suffix}")
327 - row_parallel_weights: List[str] = []
328 - for layer in self._row_parallel_layers:
329 - for suffix in row_weight_suffixes:
330 - row_parallel_weights.append(f"{layer}.{suffix}")
331 -
332 - tp_size = get_tensor_model_parallel_world_size()
333 - tp_rank = get_tensor_model_parallel_rank()
334 - q_proj_shard_size = (self.config.hidden_size // tp_size)
335 - num_kv_heads_replicas = max(1,
336 - tp_size // self.config.num_key_value_heads)
337 - num_kv_heads_per_gpu = max(1,
338 - self.config.num_key_value_heads // tp_size)
339 - kv_proj_shard_size = (self.config.hidden_size //
340 - self.config.num_attention_heads *
341 - num_kv_heads_per_gpu)
342 - attention_weight_specs = [
343 - # (weight_name, shard_size, offset)
344 - ("q_proj", q_proj_shard_size, 0),
345 - ("k_proj", kv_proj_shard_size, q_proj_shard_size),
346 - ("v_proj", kv_proj_shard_size,
347 - q_proj_shard_size + kv_proj_shard_size),
299 + stacked_params_mapping = [
300 + # (param_name, shard_name, shard_id)
301 + ("qkv_proj", "q_proj", "q"),
302 + ("qkv_proj", "k_proj", "k"),
303 + ("qkv_proj", "v_proj", "v"),
304 + ("gate_up_proj", "gate_proj", 0),
305 + ("gate_up_proj", "up_proj", 1),
306 ]
349 - state_dict = self.state_dict()
350 -
307 + params_dict = dict(self.named_parameters())
308 for name, loaded_weight in hf_model_weights_iterator(
309 model_name_or_path, cache_dir, load_format, revision):
310 if "rotary_emb.inv_freq" in name:
311 continue
355 -
356 - packed_dim = None
357 - is_transposed = False
358 - if self.quant_config is not None:
359 - packed_dim = self.quant_config.get_packed_dim(name)
360 - is_transposed = self.quant_config.is_transposed(name)
361 - if is_transposed:
362 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
363 - loaded_weight = loaded_weight.T
364 -
365 - is_attention_weight = False
366 - for weight_name, shard_size, offset in attention_weight_specs:
312 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
313 if weight_name not in name:
314 continue
369 - param = state_dict[name.replace(weight_name, "qkv_proj")]
370 - if is_transposed:
371 - param = param.T
372 -
373 - if packed_dim is not None:
374 - shard_dim = 0 if not is_transposed else 1
375 - if packed_dim == shard_dim:
376 - shard_size //= self.quant_config.pack_factor
377 - offset //= self.quant_config.pack_factor
378 -
379 - if weight_name in ["k_proj", "v_proj"]:
380 - shard_id = tp_rank // num_kv_heads_replicas
381 - else:
382 - shard_id = tp_rank
383 - loaded_weight = loaded_weight[shard_size *
384 - shard_id:shard_size *
385 - (shard_id + 1)]
386 - param_slice = param.data[offset:offset + shard_size]
387 - assert param_slice.shape == loaded_weight.shape
388 -
389 - param_slice.copy_(loaded_weight)
390 - is_attention_weight = True
391 - break
392 - if is_attention_weight:
393 - continue
394 -
395 - is_gate_up_weight = False
396 - for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
397 - if weight_name not in name:
398 - continue
399 - param = state_dict[name.replace(weight_name, "gate_up_proj")]
400 - if is_transposed:
401 - param = param.T
402 -
403 - shard_size = param.shape[0] // 2
404 - loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
405 - (tp_rank + 1)]
406 - param_slice = param.data[shard_size * stride_id:shard_size *
407 - (stride_id + 1)]
408 - assert param_slice.shape == loaded_weight.shape
409 - param_slice.copy_(loaded_weight)
410 - is_gate_up_weight = True
315 + param = params_dict[name.replace(weight_name, param_name)]
316 + weight_loader = param.weight_loader
317 + weight_loader(param, loaded_weight, shard_id)
318 break
412 - if is_gate_up_weight:
413 - continue
414 -
415 - param = state_dict[name]
416 - if is_transposed:
417 - param = param.T
418 -
419 - if "embed_tokens" in name or "lm_head" in name:
420 - load_padded_tensor_parallel_vocab(param, loaded_weight,
421 - tp_rank)
422 - continue
423 -
424 - load_tensor_parallel_weights(param, loaded_weight, name,
425 - column_parallel_weights,
426 - row_parallel_weights, tp_rank)
319 + else:
320 + param = params_dict[name]
321 + weight_loader = getattr(param, "weight_loader",
322 + default_weight_loader)
323 + weight_loader(param, loaded_weight)
vllm/model_executor/parallel_utils/layers.py deleted
-303
@@ -1,303 +0,0 @@
1 -# Copyright 2023 The vLLM team.
2 -# Adapted from
3 -# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
4 -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 -
6 -# Parts of the code here are adapted from PyTorch
7 -# repo: https://github.com/pytorch/pytorch
8 -from typing import Optional
9 -
10 -import torch
11 -import torch.nn.functional as F
12 -from torch.nn.parameter import Parameter
13 -
14 -from vllm.model_executor.parallel_utils.parallel_state import (
15 - get_tensor_model_parallel_rank,
16 - get_tensor_model_parallel_world_size,
17 -)
18 -from vllm.model_executor.quantization_utils import QuantizationConfig
19 -from vllm.model_executor.parallel_utils.communication_op import (
20 - tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
21 -
22 -from vllm.model_executor.parallel_utils.utils import (
23 - divide,
24 - VocabUtility,
25 - split_tensor_along_last_dim,
26 -)
27 -
28 -
29 -class VocabParallelEmbedding(torch.nn.Module):
30 - """Embedding parallelized in the vocabulary dimension.
31 -
32 - This is mainly adapted from torch.nn.Embedding and all the default
33 - values are kept.
34 - Arguments:
35 - num_embeddings: vocabulary size.
36 - embedding_dim: size of hidden state.
37 - params_dtype: type of the parameters.
38 - """
39 -
40 - def __init__(self,
41 - num_embeddings: int,
42 - embedding_dim: int,
43 - params_dtype: Optional[torch.dtype] = None):
44 - super().__init__()
45 -
46 - # Keep the input dimensions.
47 - self.num_embeddings = num_embeddings
48 - self.embedding_dim = embedding_dim
49 - if params_dtype is None:
50 - params_dtype = torch.get_default_dtype()
51 -
52 - self.tp_size = get_tensor_model_parallel_world_size()
53 - # TODO: Handle vocab padding here.
54 - # Divide the weight matrix along the vocaburaly dimension.
55 - self.vocab_start_index, self.vocab_end_index = (
56 - VocabUtility.vocab_range_from_global_vocab_size(
57 - self.num_embeddings, get_tensor_model_parallel_rank(),
58 - self.tp_size))
59 - self.num_embeddings_per_partition = (self.vocab_end_index -
60 - self.vocab_start_index)
61 -
62 - self.weight = Parameter(
63 - torch.empty(self.num_embeddings_per_partition,
64 - self.embedding_dim,
65 - device=torch.cuda.current_device(),
66 - dtype=params_dtype))
67 -
68 - def forward(self, input_):
69 - if self.tp_size > 1:
70 - # Build the mask.
71 - input_mask = ((input_ < self.vocab_start_index) |
72 - (input_ >= self.vocab_end_index))
73 - # Mask the input.
74 - masked_input = input_.clone() - self.vocab_start_index
75 - masked_input[input_mask] = 0
76 - else:
77 - masked_input = input_
78 - # Get the embeddings.
79 - output_parallel = F.embedding(masked_input, self.weight)
80 - # Mask the output embedding.
81 - if self.tp_size > 1:
82 - output_parallel[input_mask, :] = 0.0
83 - # Reduce across all the model parallel GPUs.
84 - output = tensor_model_parallel_all_reduce(output_parallel)
85 - return output
86 -
87 -
88 -class ColumnParallelLinear(torch.nn.Module):
89 - """Linear layer with column parallelism.
90 -
91 - The linear layer is defined as Y = XA + b. A is parallelized along
92 - its second dimension as A = [A_1, ..., A_p].
93 -
94 - Arguments:
95 - input_size: first dimension of matrix A.
96 - output_size: second dimension of matrix A.
97 -
98 - Keyword Arguments
99 - bias: If true, add bias
100 - gather_output: If true, call all-gather on output and make Y available
101 - to all GPUs, otherwise, every GPU will have its output
102 - which is Y_i = XA_i
103 - skip_bias_add: This was added to enable performance optimizations where
104 - bias can be fused with other element-wise operations. we
105 - skip adding bias but instead return it.
106 - params_dtype: Data type for the parameters.
107 - quant_config: Quantization configuration.
108 - """
109 -
110 - def __init__(
111 - self,
112 - input_size: int,
113 - output_size: int,
114 - bias: bool = True,
115 - gather_output: bool = True,
116 - skip_bias_add: bool = False,
117 - params_dtype: Optional[torch.dtype] = None,
118 - quant_config: Optional[QuantizationConfig] = None,
119 - ):
120 - super().__init__()
121 -
122 - # Keep input parameters
123 - self.input_size = input_size
124 - self.output_size = output_size
125 - self.gather_output = gather_output
126 - # Divide the weight matrix along the last dimension.
127 - self.tp_size = get_tensor_model_parallel_world_size()
128 - self.output_size_per_partition = divide(output_size, self.tp_size)
129 - self.skip_bias_add = skip_bias_add
130 - self.quant_config = quant_config
131 -
132 - if params_dtype is None:
133 - params_dtype = torch.get_default_dtype()
134 -
135 - # Parameters.
136 - # NOTE: torch.nn.functional.linear performs XA^T + b and as a result
137 - # we allocate the transpose.
138 - self.create_weights(params_dtype)
139 -
140 - if bias:
141 - self.bias = Parameter(
142 - torch.empty(self.output_size_per_partition,
143 - device=torch.cuda.current_device(),
144 - dtype=params_dtype))
145 - else:
146 - self.register_parameter('bias', None)
147 -
148 - def create_weights(self, dtype: torch.dtype) -> None:
149 - self.weight = Parameter(
150 - torch.empty(self.output_size_per_partition,
151 - self.input_size,
152 - device=torch.cuda.current_device(),
153 - dtype=dtype))
154 -
155 - def apply_weights(
156 - self,
157 - x: torch.Tensor,
158 - bias: Optional[torch.Tensor],
159 - ) -> torch.Tensor:
160 - return F.linear(x, self.weight, bias)
161 -
162 - def forward(self, input_):
163 - """Forward of ColumnParallelLinear
164 -
165 - Args:
166 - input_: Tensor whose last dimension is `input_size`.
167 -
168 - Returns:
169 - - output
170 - - bias
171 - """
172 - bias = self.bias if not self.skip_bias_add else None
173 -
174 - input_parallel = input_
175 - # Matrix multiply.
176 - output_parallel = self.apply_weights(input_parallel, bias)
177 - if self.gather_output:
178 - # All-gather across the partitions.
179 - output = tensor_model_parallel_all_gather(output_parallel)
180 - else:
181 - output = output_parallel
182 - output_bias = self.bias if self.skip_bias_add else None
183 - return output, output_bias
184 -
185 -
186 -class RowParallelLinear(torch.nn.Module):
187 - """Linear layer with row parallelism.
188 -
189 - The linear layer is defined as Y = XA + b. A is parallelized along
190 - its first dimension and X along its second dimension as:
191 - - -
192 - | A_1 |
193 - | . |
194 - A = | . | X = [X_1, ..., X_p]
195 - | . |
196 - | A_p |
197 - - -
198 - Arguments:
199 - input_size: first dimension of matrix A.
200 - output_size: second dimension of matrix A.
201 -
202 - Keyword Arguments:
203 - bias: If true, add bias. Note that bias is not parallelized.
204 - input_is_parallel: If true, we assume that the input is already
205 - split across the GPUs and we do not split
206 - again.
207 - skip_bias_add: This was added to enable performance optimization where
208 - bias can be fused with other element-wise operations.
209 - We skip adding bias but instead return it.
210 - params_dtype: Data type for the parameters.
211 - quant_config: Quantization configuration.
212 - """
213 -
214 - def __init__(
215 - self,
216 - input_size: int,
217 - output_size: int,
218 - bias: bool = True,
219 - input_is_parallel: bool = False,
220 - skip_bias_add: bool = False,
221 - params_dtype: Optional[torch.dtype] = None,
222 - reduce_results: bool = True,
223 - quant_config: Optional[QuantizationConfig] = None,
224 - ):
225 - super().__init__()
226 - # Keep input parameters
227 - self.input_size = input_size
228 - self.output_size = output_size
229 - self.input_is_parallel = input_is_parallel
230 - self.reduce_results = reduce_results
231 - if params_dtype is None:
232 - params_dtype = torch.get_default_dtype()
233 -
234 - # Divide the weight matrix along the last dimension.
235 - self.tp_size = get_tensor_model_parallel_world_size()
236 - self.input_size_per_partition = divide(input_size, self.tp_size)
237 - self.skip_bias_add = skip_bias_add
238 - self.quant_config = quant_config
239 -
240 - self.create_weights(params_dtype)
241 -
242 - if not reduce_results and (bias and not skip_bias_add):
243 - raise ValueError('When not reduce the results, adding bias to the '
244 - 'results can lead to incorrect results')
245 -
246 - if bias:
247 - self.bias = Parameter(
248 - torch.empty(self.output_size,
249 - device=torch.cuda.current_device(),
250 - dtype=params_dtype))
251 -
252 - # Always initialize bias to zero.
253 - with torch.no_grad():
254 - self.bias.zero_()
255 - else:
256 - self.register_parameter('bias', None)
257 -
258 - def create_weights(self, dtype: torch.dtype) -> None:
259 - self.weight = Parameter(
260 - torch.empty(self.output_size,
261 - self.input_size_per_partition,
262 - device=torch.cuda.current_device(),
263 - dtype=dtype))
264 -
265 - def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
266 - return F.linear(x, self.weight)
267 -
268 - def forward(self, input_):
269 - """Forward of RowParallelLinear
270 -
271 - Args:
272 - input_: tensor whose last dimension is `input_size`. If
273 - `input_is_parallel` is set, then the last dimension
274 - is `input_size // tp_size`.
275 -
276 - Returns:
277 - - output
278 - - bias
279 - """
280 - # Set up backprop all-reduce.
281 - if self.input_is_parallel:
282 - input_parallel = input_
283 - else:
284 - # TODO: simplify code below
285 - tp_rank = get_tensor_model_parallel_rank()
286 - splitted_input = split_tensor_along_last_dim(
287 - input_, num_partitions=self.tp_size)
288 - input_parallel = splitted_input[tp_rank].contiguous()
289 -
290 - # Matrix multiply.
291 - output_parallel = self.apply_weights(input_parallel)
292 - if self.reduce_results and self.tp_size > 1:
293 - output_ = tensor_model_parallel_all_reduce(output_parallel)
294 - else:
295 - output_ = output_parallel
296 -
297 - if not self.skip_bias_add:
298 - output = output_ + self.bias if self.bias is not None else output_
299 - output_bias = None
300 - else:
301 - output = output_
302 - output_bias = self.bias
303 - return output, output_bias
vllm/model_executor/parallel_utils/utils.py
+2 -24
@@ -2,7 +2,7 @@
2 # Adapted from
3 # https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 -from typing import List, Sequence
5 +from typing import Sequence
6
7 import torch
8
@@ -24,7 +24,7 @@ def split_tensor_along_last_dim(
24 tensor: torch.Tensor,
25 num_partitions: int,
26 contiguous_split_chunks: bool = False,
27 -) -> List[torch.Tensor]:
27 +) -> Sequence[torch.Tensor]:
28 """ Split a tensor along its last dimension.
29
30 Arguments:
@@ -46,25 +46,3 @@ def split_tensor_along_last_dim(
46 return tuple(chunk.contiguous() for chunk in tensor_list)
47
48 return tensor_list
49 -
50 -
51 -class VocabUtility:
52 - """ Split the vocabulary into `world_size` chunks and return the first
53 - and last index of the vocabulary belonging to the `rank`
54 - partition: Note that indices in [fist, last)
55 -
56 - """
57 -
58 - @staticmethod
59 - def vocab_range_from_per_partition_vocab_size(
60 - per_partition_vocab_size: int, rank: int) -> Sequence[int]:
61 - index_f = rank * per_partition_vocab_size
62 - index_l = index_f + per_partition_vocab_size
63 - return index_f, index_l
64 -
65 - @staticmethod
66 - def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
67 - world_size: int) -> Sequence[int]:
68 - per_partition_vocab_size = divide(global_vocab_size, world_size)
69 - return VocabUtility.vocab_range_from_per_partition_vocab_size(
70 - per_partition_vocab_size, rank)
vllm/model_executor/quantization_utils/__init__.py deleted
-22
@@ -1,22 +0,0 @@
1 -from typing import Type
2 -
3 -from vllm.model_executor.quantization_utils.awq import AWQConfig
4 -from vllm.model_executor.quantization_utils.base import QuantizationConfig
5 -from vllm.model_executor.quantization_utils.squeezellm import SqueezeLLMConfig
6 -
7 -_QUANTIZATION_REGISTRY = {
8 - "awq": AWQConfig,
9 - "squeezellm": SqueezeLLMConfig,
10 -}
11 -
12 -
13 -def get_quant_class(quantization: str) -> Type[QuantizationConfig]:
14 - if quantization not in _QUANTIZATION_REGISTRY:
15 - raise ValueError(f"Invalid quantization method: {quantization}")
16 - return _QUANTIZATION_REGISTRY[quantization]
17 -
18 -
19 -__all__ = [
20 - "QuantizationConfig",
21 - "get_quant_class",
22 -]
vllm/model_executor/quantization_utils/awq.py deleted
-76
@@ -1,76 +0,0 @@
1 -from typing import Any, Dict, List
2 -
3 -import torch
4 -
5 -from vllm.model_executor.quantization_utils.base import QuantizationConfig
6 -
7 -
8 -class AWQConfig(QuantizationConfig):
9 - """Config class for AWQ.
10 -
11 - Reference: https://arxiv.org/abs/2306.00978
12 - """
13 -
14 - def __init__(
15 - self,
16 - weight_bits: int,
17 - group_size: int,
18 - zero_point: bool,
19 - ) -> None:
20 - self.weight_bits = weight_bits
21 - self.group_size = group_size
22 - self.zero_point = zero_point
23 -
24 - if self.weight_bits != 4:
25 - raise ValueError(
26 - "Currently, only 4-bit weight quantization is supported for "
27 - f"AWQ, but got {self.weight_bits} bits.")
28 - self.pack_factor = 32 // self.weight_bits
29 -
30 - def __repr__(self) -> str:
31 - return (f"AWQConfig(weight_bits={self.weight_bits}, "
32 - f"group_size={self.group_size}, "
33 - f"zero_point={self.zero_point})")
34 -
35 - @classmethod
36 - def get_name(cls) -> str:
37 - return "awq"
38 -
39 - @classmethod
40 - def get_supported_act_dtypes(cls) -> List[torch.dtype]:
41 - return [torch.half]
42 -
43 - @classmethod
44 - def get_min_capability(cls) -> int:
45 - # The AWQ kernel only supports Turing or newer GPUs.
46 - return 75
47 -
48 - @classmethod
49 - def get_config_filenames(cls) -> List[str]:
50 - return [
51 - "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
52 - "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq # pylint: disable=line-too-long
53 - ]
54 -
55 - @classmethod
56 - def from_config(cls, config: Dict[str, Any]) -> "AWQConfig":
57 - weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
58 - group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
59 - zero_point = cls.get_from_keys(config, ["zero_point"])
60 - return cls(weight_bits, group_size, zero_point)
61 -
62 - @classmethod
63 - def get_packed_tensors(cls) -> Dict[str, int]:
64 - return {"qweight": 1, "qzeros": 1}
65 -
66 - @classmethod
67 - def get_transposed_tensor_names(cls) -> List[str]:
68 - return ["qweight", "qzeros", "scales"]
69 -
70 - @classmethod
71 - def get_col_parallel_tensor_names(cls) -> List[str]:
72 - return ["qweight", "qzeros", "scales"]
73 -
74 - @classmethod
75 - def get_row_parallel_tensor_names(cls) -> List[str]:
76 - return ["qweight", "qzeros", "scales"]
vllm/model_executor/quantization_utils/base.py deleted
-85
@@ -1,85 +0,0 @@
1 -from typing import Any, Dict, List, Optional
2 -
3 -import torch
4 -
5 -
6 -class QuantizationConfig:
7 -
8 - @classmethod
9 - def get_name(cls) -> str:
10 - """Name of the quantization method."""
11 - raise NotImplementedError
12 -
13 - @classmethod
14 - def get_supported_act_dtypes(cls) -> List[torch.dtype]:
15 - """List of supported activation dtypes."""
16 - raise NotImplementedError
17 -
18 - @classmethod
19 - def get_min_capability(cls) -> int:
20 - """Minimum GPU capability to support the quantization method.
21 -
22 - E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
23 - This requirement is due to the custom CUDA kernels used by the
24 - quantization method.
25 - """
26 - raise NotImplementedError
27 -
28 - @classmethod
29 - def get_config_filenames(cls) -> List[str]:
30 - """List of filenames to search for in the model directory."""
31 - raise NotImplementedError
32 -
33 - @classmethod
34 - def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig":
35 - """Create a config class from the model's quantization config."""
36 - raise NotImplementedError
37 -
38 - @staticmethod
39 - def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any:
40 - """Get a value from the model's quantization config."""
41 - for key in keys:
42 - if key in config:
43 - return config[key]
44 - raise ValueError(f"Cannot find any of {keys} in the model's "
45 - "quantization config.")
46 -
47 - @classmethod
48 - def get_packed_tensors(cls) -> Dict[str, int]:
49 - """Returns a dictionary of packed tensor names and their pack dims."""
50 - raise NotImplementedError
51 -
52 - @classmethod
53 - def get_packed_dim(cls, tensor_name: str) -> Optional[int]:
54 - """Returns the pack dim of a tensor if it is packed.
55 -
56 - A tensor is considered packed if each element in the tensor is a
57 - packed representation of multiple elements in the original tensor.
58 - For example, an INT32 element in the tensor may represent 8 INT4
59 - elements in the original tensor.
60 - If the tensor is not packed, returns None.
61 - """
62 - packed_tensors = cls.get_packed_tensors()
63 - for packed_tensor_name, pack_dim in packed_tensors.items():
64 - if packed_tensor_name in tensor_name:
65 - return pack_dim
66 - return None
67 -
68 - @classmethod
69 - def get_transposed_tensor_names(cls) -> List[str]:
70 - raise NotImplementedError
71 -
72 - @classmethod
73 - def is_transposed(cls, tensor_name: str) -> bool:
74 - """Returns True if a tensor is transposed relative to nn.Linear.weight.
75 - """
76 - return any(tag in tensor_name
77 - for tag in cls.get_transposed_tensor_names())
78 -
79 - @classmethod
80 - def get_col_parallel_tensor_names(cls) -> List[str]:
81 - raise NotImplementedError
82 -
83 - @classmethod
84 - def get_row_parallel_tensor_names(cls) -> List[str]:
85 - raise NotImplementedError
vllm/model_executor/quantization_utils/squeezellm.py deleted
-65
@@ -1,65 +0,0 @@
1 -from typing import Any, Dict, List
2 -
3 -import torch
4 -
5 -from vllm.model_executor.quantization_utils.base import QuantizationConfig
6 -
7 -
8 -class SqueezeLLMConfig(QuantizationConfig):
9 - """Config class for SqueezeLLM.
10 -
11 - Reference: https://arxiv.org/pdf/2306.07629
12 - """
13 -
14 - def __init__(
15 - self,
16 - weight_bits: int,
17 - ) -> None:
18 - self.weight_bits = weight_bits
19 -
20 - if self.weight_bits != 4:
21 - raise ValueError(
22 - "Currently, only 4-bit weight quantization is supported for "
23 - f"SqueezeLLM, but got {self.weight_bits} bits.")
24 -
25 - self.pack_factor = 32 // self.weight_bits
26 -
27 - def __repr__(self) -> str:
28 - return f"SqueezeLLMConfig(weight_bits={self.weight_bits})"
29 -
30 - @classmethod
31 - def get_name(cls) -> str:
32 - return "squeezellm"
33 -
34 - @classmethod
35 - def get_supported_act_dtypes(cls) -> List[torch.dtype]:
36 - return [torch.half]
37 -
38 - @classmethod
39 - def get_min_capability(cls) -> int:
40 - return 70
41 -
42 - @classmethod
43 - def get_config_filenames(cls) -> List[str]:
44 - return ["quant_config.json"]
45 -
46 - @classmethod
47 - def from_config(cls, config: Dict[str, Any]) -> "SqueezeLLMConfig":
48 - weight_bits = cls.get_from_keys(config, ["wbits"])
49 - return cls(weight_bits)
50 -
51 - @classmethod
52 - def get_packed_tensors(cls) -> Dict[str, int]:
53 - return {"qweight": 0}
54 -
55 - @classmethod
56 - def get_transposed_tensor_names(cls) -> List[str]:
57 - return ["qweight"]
58 -
59 - @classmethod
60 - def get_col_parallel_tensor_names(cls) -> List[str]:
61 - return ["qweight", "lookup_table"]
62 -
63 - @classmethod
64 - def get_row_parallel_tensor_names(cls) -> List[str]:
65 - return ["qweight"]
vllm/model_executor/utils.py
+22
@@ -1,5 +1,6 @@
1 """Utils for model executor."""
2 import random
3 +from typing import Any, Dict, Optional
4
5 import numpy as np
6 import torch
@@ -11,3 +12,24 @@ def set_random_seed(seed: int) -> None:
12 torch.manual_seed(seed)
13 if torch.cuda.is_available():
14 torch.cuda.manual_seed_all(seed)
15 +
16 +
17 +def set_weight_attrs(
18 + weight: torch.Tensor,
19 + weight_attrs: Optional[Dict[str, Any]],
20 +):
21 + """Set attributes on a weight tensor.
22 +
23 + This method is used to set attributes on a weight tensor. This method
24 + will not overwrite existing attributes.
25 +
26 + Args:
27 + weight: The weight tensor.
28 + weight_attrs: A dictionary of attributes to set on the weight tensor.
29 + """
30 + if weight_attrs is None:
31 + return
32 + for key, value in weight_attrs.items():
33 + assert not hasattr(
34 + weight, key), (f"Overwriting existing tensor attribute: {key}")
35 + setattr(weight, key, value)
vllm/model_executor/weight_utils.py
+8 -44
@@ -13,8 +13,8 @@ import torch
13 from tqdm.auto import tqdm
14
15 from vllm.logger import init_logger
16 -from vllm.model_executor.quantization_utils import get_quant_class
17 -from vllm.model_executor.quantization_utils.base import QuantizationConfig
16 +from vllm.model_executor.layers.quantization import (get_quantization_config,
17 + QuantizationConfig)
18
19 logger = init_logger(__name__)
20
@@ -98,7 +98,7 @@ def get_quant_config(
98 hf_folder = model_name_or_path
99 config_files = glob.glob(os.path.join(hf_folder, "*.json"))
100
101 - quant_cls = get_quant_class(quantization)
101 + quant_cls = get_quantization_config(quantization)
102 quant_config_files = [
103 f for f in config_files if any(
104 f.endswith(x) for x in quant_cls.get_config_filenames())
@@ -237,7 +237,7 @@ def hf_model_weights_iterator(
237 with safe_open(st_file, framework="pt") as f:
238 for name in f.keys():
239 param = f.get_slice(name)
240 - yield name, param
240 + yield name, convert_pyslice_to_tensor(param)
241 else:
242 for bin_file in hf_weights_files:
243 state = torch.load(bin_file, map_location="cpu")
@@ -262,46 +262,10 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
262 return x
263
264
265 -def load_padded_tensor_parallel_vocab(
266 - param: torch.Tensor,
267 - loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
268 - tensor_model_parallel_rank: int,
269 -) -> None:
270 - shard_size = param.shape[0]
271 - start_idx = tensor_model_parallel_rank * shard_size
272 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
273 - loaded_weight = loaded_weight[start_idx:end_idx]
274 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
275 - param[:loaded_weight.shape[0]].copy_(loaded_weight)
276 -
277 -
278 -def load_tensor_parallel_weights(
279 - param: torch.Tensor,
280 - loaded_weight: Any, # `torch.Tensor` or `PySafeSlice`
281 - param_name: str,
282 - column_parallel_weight_names: List[str],
283 - row_parallel_weight_names: List[str],
284 - tensor_model_parallel_rank: int,
285 -) -> None:
286 - for p in column_parallel_weight_names:
287 - if p in param_name:
288 - shard_size = param.shape[0]
289 - start_idx = tensor_model_parallel_rank * shard_size
290 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
291 - loaded_weight = loaded_weight[start_idx:end_idx]
292 - break
293 - for p in row_parallel_weight_names:
294 - if p in param_name:
295 - shard_size = param.shape[1]
296 - start_idx = tensor_model_parallel_rank * shard_size
297 - end_idx = (tensor_model_parallel_rank + 1) * shard_size
298 - loaded_weight = loaded_weight[:, start_idx:end_idx]
299 - break
300 -
301 - loaded_weight = convert_pyslice_to_tensor(loaded_weight)
302 - assert param.shape == loaded_weight.shape, (
303 - f"{param_name} shape mismatch between model and checkpoint: "
304 - f"{param.shape} != {loaded_weight.shape}")
265 +def default_weight_loader(param: torch.Tensor,
266 + loaded_weight: torch.Tensor) -> None:
267 + """Default weight loader."""
268 + assert param.size() == loaded_weight.size()
269 param.data.copy_(loaded_weight)
270
271