A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Merge QKV into one linear layer (#15)
Zhuohan Li committed
Apr 2, 2023 at 15:23 UTC
1f01a18d39b7fc873b79024b5799597cb6fc88bc
2 files changed
+90
-83
cacheflow/models/llama.py
+52
-47
@@ -33,22 +33,21 @@ class LlamaMLP(nn.Module):
33
hidden_act: str,
34
):
35
super().__init__()
36
- # TODO: Merge the gate and down linear layers.
37
- self.gate_proj = ColumnParallelLinear(hidden_size, intermediate_size,
38
- bias=False, gather_output=False,
39
- perform_initialization=False)
36
+ self.gate_up_proj = ColumnParallelLinear(hidden_size, 2 * intermediate_size,
37
+ bias=False, gather_output=False,
38
+ perform_initialization=False)
39
self.down_proj = RowParallelLinear(intermediate_size, hidden_size,
40
bias=False, input_is_parallel=True,
41
perform_initialization=False)
43
- self.up_proj = ColumnParallelLinear(hidden_size, intermediate_size,
44
- bias=False, gather_output=False,
45
- perform_initialization=False)
42
assert hidden_act == 'silu'
43
self.act_fn = nn.SiLU()
44
45
def forward(self, x):
50
- gate, _ = self.gate_proj(x)
51
- up, _ = self.up_proj(x)
46
+ gate_up, _ = self.gate_up_proj(x)
47
+ gate_up = gate_up.reshape(gate_up.shape[:-1] + (2, -1))
48
+ gate, up = torch.split(gate_up, 1, dim=-2)
49
+ gate = gate.squeeze(dim=-2).contiguous()
50
+ up = up.squeeze(dim=-2).contiguous()
51
x = self.act_fn(gate) * up
52
x, _ = self.down_proj(x)
53
return x
@@ -70,24 +69,9 @@ class LlamaAttention(nn.Module):
69
self.head_dim = hidden_size // self.total_num_heads
70
self.scaling = self.head_dim ** -0.5
71
73
- # TODO: Merge the QKV linear layers.
74
- self.q_proj = ColumnParallelLinear(
75
- hidden_size,
76
- self.total_num_heads * self.head_dim,
77
- bias=False,
78
- gather_output=False,
79
- perform_initialization=False,
80
- )
81
- self.k_proj = ColumnParallelLinear(
82
- hidden_size,
83
- self.total_num_heads * self.head_dim,
84
- bias=False,
85
- gather_output=False,
86
- perform_initialization=False,
87
- )
88
- self.v_proj = ColumnParallelLinear(
72
+ self.qkv_proj = ColumnParallelLinear(
73
hidden_size,
90
- self.total_num_heads * self.head_dim,
74
+ 3 * self.total_num_heads * self.head_dim,
75
bias=False,
76
gather_output=False,
77
perform_initialization=False,
@@ -109,9 +93,12 @@ class LlamaAttention(nn.Module):
93
input_metadata: InputMetadata,
94
cache_event: Optional[torch.cuda.Event],
95
) -> torch.Tensor:
112
- q, _ = self.q_proj(hidden_states)
113
- k, _ = self.k_proj(hidden_states)
114
- v, _ = self.v_proj(hidden_states)
96
+ qkv, _ = self.qkv_proj(hidden_states)
97
+ qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
98
+ q, k, v = torch.split(qkv, 1, dim=-2)
99
+ q = q.squeeze(dim=-2).contiguous()
100
+ k = k.squeeze(dim=-2).contiguous()
101
+ v = v.squeeze(dim=-2).contiguous()
102
k_cache, v_cache = kv_cache
103
attn_output = self.attn(
104
positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
@@ -230,8 +217,7 @@ class LlamaForCausalLM(nn.Module):
217
return next_tokens
218
219
_column_parallel_weights = ["embed_tokens.weight", "lm_head.weight",
233
- "q_proj.weight", "k_proj.weight",
234
- "v_proj.weight", "gate_proj.weight",
220
+ "qkv_proj.weight", "gate_proj.weight",
221
"up_proj.weight"]
222
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
223
@@ -239,23 +225,42 @@ class LlamaForCausalLM(nn.Module):
225
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
226
state_dict = self.state_dict()
227
for name, param in state_dict.items():
242
- loaded_weight = torch.from_numpy(np.load(os.path.join(weights_path,
243
- name)))
244
- for p in self._column_parallel_weights:
245
- if p in name:
246
- shard_size = param.shape[0]
247
- loaded_weight = loaded_weight[
228
+ if "qkv_proj" in name or "gate_up_proj" in name:
229
+ if "qkv_proj" in name:
230
+ original_name = "qkv_proj"
231
+ weight_names = ["q_proj", "k_proj", "v_proj"]
232
+ shard_size = param.shape[0] // 3
233
+ else:
234
+ original_name = "gate_up_proj"
235
+ weight_names = ["gate_proj", "up_proj"]
236
+ shard_size = param.shape[0] // 2
237
+ weights_to_concat = []
238
+ for weight_name in weight_names:
239
+ weight = np.load(os.path.join(
240
+ weights_path, name.replace(original_name, weight_name)))
241
+ weights_to_concat.append(weight[
242
shard_size * tensor_model_parallel_rank
249
- :shard_size * (tensor_model_parallel_rank + 1)]
250
- break
251
- for p in self._row_parallel_weights:
252
- if p in name:
253
- shard_size = param.shape[1]
254
- loaded_weight = loaded_weight[
255
- :,
256
- shard_size * tensor_model_parallel_rank
257
- :shard_size * (tensor_model_parallel_rank + 1)]
258
- break
243
+ :shard_size * (tensor_model_parallel_rank + 1)])
244
+ loaded_weight = torch.from_numpy(
245
+ np.concatenate(weights_to_concat, axis=0))
246
+ else:
247
+ loaded_weight = torch.from_numpy(
248
+ np.load(os.path.join(weights_path, name)))
249
+ for p in self._column_parallel_weights:
250
+ if p in name:
251
+ shard_size = param.shape[0]
252
+ loaded_weight = loaded_weight[
253
+ shard_size * tensor_model_parallel_rank
254
+ :shard_size * (tensor_model_parallel_rank + 1)]
255
+ break
256
+ for p in self._row_parallel_weights:
257
+ if p in name:
258
+ shard_size = param.shape[1]
259
+ loaded_weight = loaded_weight[
260
+ :,
261
+ shard_size * tensor_model_parallel_rank
262
+ :shard_size * (tensor_model_parallel_rank + 1)]
263
+ break
264
265
assert param.shape == loaded_weight.shape
266
param.data.copy_(loaded_weight)
cacheflow/models/opt.py
+38
-36
@@ -53,16 +53,9 @@ class OPTAttention(nn.Module):
53
self.head_dim = embed_dim // total_num_heads
54
self.scaling = self.head_dim ** -0.5
55
56
- # TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
57
- self.k_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
58
- gather_output=False,
59
- perform_initialization=False)
60
- self.v_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
61
- gather_output=False,
62
- perform_initialization=False)
63
- self.q_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
64
- gather_output=False,
65
- perform_initialization=False)
56
+ self.qkv_proj = ColumnParallelLinear(embed_dim, 3 * embed_dim, bias=bias,
57
+ gather_output=False,
58
+ perform_initialization=False)
59
self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
60
input_is_parallel=True,
61
perform_initialization=False)
@@ -75,16 +68,18 @@ class OPTAttention(nn.Module):
68
input_metadata: InputMetadata,
69
cache_event: Optional[torch.cuda.Event],
70
) -> torch.Tensor:
78
- q, _ = self.q_proj(hidden_states)
79
- k, _ = self.k_proj(hidden_states)
80
- v, _ = self.v_proj(hidden_states)
71
+ qkv, _ = self.qkv_proj(hidden_states)
72
+ qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
73
+ q, k, v = torch.split(qkv, 1, dim=-2)
74
+ q = q.squeeze(dim=-2).contiguous()
75
+ k = k.squeeze(dim=-2).contiguous()
76
+ v = v.squeeze(dim=-2).contiguous()
77
key_cache, value_cache = kv_cache
78
attn_output = self.attn(
79
q, k, v, key_cache, value_cache, input_metadata, cache_event)
80
output, _ = self.out_proj(attn_output)
81
return output
82
87
-
83
class OPTDecoderLayer(nn.Module):
84
85
def __init__(self, config: OPTConfig):
@@ -262,11 +257,7 @@ class OPTForCausalLM(nn.Module):
257
self.lm_head_weight, hidden_states, input_metadata)
258
return next_tokens
259
265
- _column_parallel_weights = ["embed_tokens.weight",
266
- "q_proj.weight", "k_proj.weight",
267
- "v_proj.weight", "fc1.weight"]
268
- _column_parallel_biases = ["q_proj.bias", "k_proj.bias",
269
- "v_proj.bias", "fc1.bias"]
260
+ _column_parallel_weights = ["embed_tokens.weight", "fc1.weight", "fc1.bias"]
261
_row_parallel_weights = ["out_proj.weight", "fc2.weight"]
262
263
def load_weights(self, weights_path: str):
@@ -275,24 +266,35 @@ class OPTForCausalLM(nn.Module):
266
for name, param in state_dict.items():
267
if "lm_head_weight" in name:
268
continue
278
- loaded_weight = torch.from_numpy(np.load(os.path.join(weights_path,
279
- name)))
280
- for p in (self._column_parallel_weights
281
- + self._column_parallel_biases):
282
- if p in name:
283
- shard_size = param.shape[0]
284
- loaded_weight = loaded_weight[
285
- shard_size * tensor_model_parallel_rank
286
- :shard_size * (tensor_model_parallel_rank + 1)]
287
- break
288
- for p in self._row_parallel_weights:
289
- if p in name:
290
- shard_size = param.shape[1]
291
- loaded_weight = loaded_weight[
292
- :,
269
+ if "qkv_proj" in name:
270
+ shard_size = param.shape[0] // 3
271
+ weights_to_concat = []
272
+ for weight_name in ["q_proj", "k_proj", "v_proj"]:
273
+ weight = np.load(os.path.join(
274
+ weights_path, name.replace("qkv_proj", weight_name)))
275
+ weights_to_concat.append(weight[
276
shard_size * tensor_model_parallel_rank
294
- :shard_size * (tensor_model_parallel_rank + 1)]
295
- break
277
+ :shard_size * (tensor_model_parallel_rank + 1)])
278
+ loaded_weight = torch.from_numpy(
279
+ np.concatenate(weights_to_concat, axis=0))
280
+ else:
281
+ loaded_weight = torch.from_numpy(
282
+ np.load(os.path.join(weights_path, name)))
283
+ for p in self._column_parallel_weights:
284
+ if p in name:
285
+ shard_size = param.shape[0]
286
+ loaded_weight = loaded_weight[
287
+ shard_size * tensor_model_parallel_rank
288
+ :shard_size * (tensor_model_parallel_rank + 1)]
289
+ break
290
+ for p in self._row_parallel_weights:
291
+ if p in name:
292
+ shard_size = param.shape[1]
293
+ loaded_weight = loaded_weight[
294
+ :,
295
+ shard_size * tensor_model_parallel_rank
296
+ :shard_size * (tensor_model_parallel_rank + 1)]
297
+ break
298
299
assert param.shape == loaded_weight.shape
300
param.data.copy_(loaded_weight)