@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f01a18d39
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Merge QKV into one linear layer (#15)

Zhuohan Li committed Apr 2, 2023 at 15:23 UTC 1f01a18d39b7fc873b79024b5799597cb6fc88bc
2 files changed +90 -83
cacheflow/models/llama.py
+52 -47
@@ -33,22 +33,21 @@ class LlamaMLP(nn.Module):
33 hidden_act: str,
34 ):
35 super().__init__()
36 - # TODO: Merge the gate and down linear layers.
37 - self.gate_proj = ColumnParallelLinear(hidden_size, intermediate_size,
38 - bias=False, gather_output=False,
39 - perform_initialization=False)
36 + self.gate_up_proj = ColumnParallelLinear(hidden_size, 2 * intermediate_size,
37 + bias=False, gather_output=False,
38 + perform_initialization=False)
39 self.down_proj = RowParallelLinear(intermediate_size, hidden_size,
40 bias=False, input_is_parallel=True,
41 perform_initialization=False)
43 - self.up_proj = ColumnParallelLinear(hidden_size, intermediate_size,
44 - bias=False, gather_output=False,
45 - perform_initialization=False)
42 assert hidden_act == 'silu'
43 self.act_fn = nn.SiLU()
44
45 def forward(self, x):
50 - gate, _ = self.gate_proj(x)
51 - up, _ = self.up_proj(x)
46 + gate_up, _ = self.gate_up_proj(x)
47 + gate_up = gate_up.reshape(gate_up.shape[:-1] + (2, -1))
48 + gate, up = torch.split(gate_up, 1, dim=-2)
49 + gate = gate.squeeze(dim=-2).contiguous()
50 + up = up.squeeze(dim=-2).contiguous()
51 x = self.act_fn(gate) * up
52 x, _ = self.down_proj(x)
53 return x
@@ -70,24 +69,9 @@ class LlamaAttention(nn.Module):
69 self.head_dim = hidden_size // self.total_num_heads
70 self.scaling = self.head_dim ** -0.5
71
73 - # TODO: Merge the QKV linear layers.
74 - self.q_proj = ColumnParallelLinear(
75 - hidden_size,
76 - self.total_num_heads * self.head_dim,
77 - bias=False,
78 - gather_output=False,
79 - perform_initialization=False,
80 - )
81 - self.k_proj = ColumnParallelLinear(
82 - hidden_size,
83 - self.total_num_heads * self.head_dim,
84 - bias=False,
85 - gather_output=False,
86 - perform_initialization=False,
87 - )
88 - self.v_proj = ColumnParallelLinear(
72 + self.qkv_proj = ColumnParallelLinear(
73 hidden_size,
90 - self.total_num_heads * self.head_dim,
74 + 3 * self.total_num_heads * self.head_dim,
75 bias=False,
76 gather_output=False,
77 perform_initialization=False,
@@ -109,9 +93,12 @@ class LlamaAttention(nn.Module):
93 input_metadata: InputMetadata,
94 cache_event: Optional[torch.cuda.Event],
95 ) -> torch.Tensor:
112 - q, _ = self.q_proj(hidden_states)
113 - k, _ = self.k_proj(hidden_states)
114 - v, _ = self.v_proj(hidden_states)
96 + qkv, _ = self.qkv_proj(hidden_states)
97 + qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
98 + q, k, v = torch.split(qkv, 1, dim=-2)
99 + q = q.squeeze(dim=-2).contiguous()
100 + k = k.squeeze(dim=-2).contiguous()
101 + v = v.squeeze(dim=-2).contiguous()
102 k_cache, v_cache = kv_cache
103 attn_output = self.attn(
104 positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
@@ -230,8 +217,7 @@ class LlamaForCausalLM(nn.Module):
217 return next_tokens
218
219 _column_parallel_weights = ["embed_tokens.weight", "lm_head.weight",
233 - "q_proj.weight", "k_proj.weight",
234 - "v_proj.weight", "gate_proj.weight",
220 + "qkv_proj.weight", "gate_proj.weight",
221 "up_proj.weight"]
222 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
223
@@ -239,23 +225,42 @@ class LlamaForCausalLM(nn.Module):
225 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
226 state_dict = self.state_dict()
227 for name, param in state_dict.items():
242 - loaded_weight = torch.from_numpy(np.load(os.path.join(weights_path,
243 - name)))
244 - for p in self._column_parallel_weights:
245 - if p in name:
246 - shard_size = param.shape[0]
247 - loaded_weight = loaded_weight[
228 + if "qkv_proj" in name or "gate_up_proj" in name:
229 + if "qkv_proj" in name:
230 + original_name = "qkv_proj"
231 + weight_names = ["q_proj", "k_proj", "v_proj"]
232 + shard_size = param.shape[0] // 3
233 + else:
234 + original_name = "gate_up_proj"
235 + weight_names = ["gate_proj", "up_proj"]
236 + shard_size = param.shape[0] // 2
237 + weights_to_concat = []
238 + for weight_name in weight_names:
239 + weight = np.load(os.path.join(
240 + weights_path, name.replace(original_name, weight_name)))
241 + weights_to_concat.append(weight[
242 shard_size * tensor_model_parallel_rank
249 - :shard_size * (tensor_model_parallel_rank + 1)]
250 - break
251 - for p in self._row_parallel_weights:
252 - if p in name:
253 - shard_size = param.shape[1]
254 - loaded_weight = loaded_weight[
255 - :,
256 - shard_size * tensor_model_parallel_rank
257 - :shard_size * (tensor_model_parallel_rank + 1)]
258 - break
243 + :shard_size * (tensor_model_parallel_rank + 1)])
244 + loaded_weight = torch.from_numpy(
245 + np.concatenate(weights_to_concat, axis=0))
246 + else:
247 + loaded_weight = torch.from_numpy(
248 + np.load(os.path.join(weights_path, name)))
249 + for p in self._column_parallel_weights:
250 + if p in name:
251 + shard_size = param.shape[0]
252 + loaded_weight = loaded_weight[
253 + shard_size * tensor_model_parallel_rank
254 + :shard_size * (tensor_model_parallel_rank + 1)]
255 + break
256 + for p in self._row_parallel_weights:
257 + if p in name:
258 + shard_size = param.shape[1]
259 + loaded_weight = loaded_weight[
260 + :,
261 + shard_size * tensor_model_parallel_rank
262 + :shard_size * (tensor_model_parallel_rank + 1)]
263 + break
264
265 assert param.shape == loaded_weight.shape
266 param.data.copy_(loaded_weight)
cacheflow/models/opt.py
+38 -36
@@ -53,16 +53,9 @@ class OPTAttention(nn.Module):
53 self.head_dim = embed_dim // total_num_heads
54 self.scaling = self.head_dim ** -0.5
55
56 - # TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
57 - self.k_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
58 - gather_output=False,
59 - perform_initialization=False)
60 - self.v_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
61 - gather_output=False,
62 - perform_initialization=False)
63 - self.q_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
64 - gather_output=False,
65 - perform_initialization=False)
56 + self.qkv_proj = ColumnParallelLinear(embed_dim, 3 * embed_dim, bias=bias,
57 + gather_output=False,
58 + perform_initialization=False)
59 self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
60 input_is_parallel=True,
61 perform_initialization=False)
@@ -75,16 +68,18 @@ class OPTAttention(nn.Module):
68 input_metadata: InputMetadata,
69 cache_event: Optional[torch.cuda.Event],
70 ) -> torch.Tensor:
78 - q, _ = self.q_proj(hidden_states)
79 - k, _ = self.k_proj(hidden_states)
80 - v, _ = self.v_proj(hidden_states)
71 + qkv, _ = self.qkv_proj(hidden_states)
72 + qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
73 + q, k, v = torch.split(qkv, 1, dim=-2)
74 + q = q.squeeze(dim=-2).contiguous()
75 + k = k.squeeze(dim=-2).contiguous()
76 + v = v.squeeze(dim=-2).contiguous()
77 key_cache, value_cache = kv_cache
78 attn_output = self.attn(
79 q, k, v, key_cache, value_cache, input_metadata, cache_event)
80 output, _ = self.out_proj(attn_output)
81 return output
82
87 -
83 class OPTDecoderLayer(nn.Module):
84
85 def __init__(self, config: OPTConfig):
@@ -262,11 +257,7 @@ class OPTForCausalLM(nn.Module):
257 self.lm_head_weight, hidden_states, input_metadata)
258 return next_tokens
259
265 - _column_parallel_weights = ["embed_tokens.weight",
266 - "q_proj.weight", "k_proj.weight",
267 - "v_proj.weight", "fc1.weight"]
268 - _column_parallel_biases = ["q_proj.bias", "k_proj.bias",
269 - "v_proj.bias", "fc1.bias"]
260 + _column_parallel_weights = ["embed_tokens.weight", "fc1.weight", "fc1.bias"]
261 _row_parallel_weights = ["out_proj.weight", "fc2.weight"]
262
263 def load_weights(self, weights_path: str):
@@ -275,24 +266,35 @@ class OPTForCausalLM(nn.Module):
266 for name, param in state_dict.items():
267 if "lm_head_weight" in name:
268 continue
278 - loaded_weight = torch.from_numpy(np.load(os.path.join(weights_path,
279 - name)))
280 - for p in (self._column_parallel_weights
281 - + self._column_parallel_biases):
282 - if p in name:
283 - shard_size = param.shape[0]
284 - loaded_weight = loaded_weight[
285 - shard_size * tensor_model_parallel_rank
286 - :shard_size * (tensor_model_parallel_rank + 1)]
287 - break
288 - for p in self._row_parallel_weights:
289 - if p in name:
290 - shard_size = param.shape[1]
291 - loaded_weight = loaded_weight[
292 - :,
269 + if "qkv_proj" in name:
270 + shard_size = param.shape[0] // 3
271 + weights_to_concat = []
272 + for weight_name in ["q_proj", "k_proj", "v_proj"]:
273 + weight = np.load(os.path.join(
274 + weights_path, name.replace("qkv_proj", weight_name)))
275 + weights_to_concat.append(weight[
276 shard_size * tensor_model_parallel_rank
294 - :shard_size * (tensor_model_parallel_rank + 1)]
295 - break
277 + :shard_size * (tensor_model_parallel_rank + 1)])
278 + loaded_weight = torch.from_numpy(
279 + np.concatenate(weights_to_concat, axis=0))
280 + else:
281 + loaded_weight = torch.from_numpy(
282 + np.load(os.path.join(weights_path, name)))
283 + for p in self._column_parallel_weights:
284 + if p in name:
285 + shard_size = param.shape[0]
286 + loaded_weight = loaded_weight[
287 + shard_size * tensor_model_parallel_rank
288 + :shard_size * (tensor_model_parallel_rank + 1)]
289 + break
290 + for p in self._row_parallel_weights:
291 + if p in name:
292 + shard_size = param.shape[1]
293 + loaded_weight = loaded_weight[
294 + :,
295 + shard_size * tensor_model_parallel_rank
296 + :shard_size * (tensor_model_parallel_rank + 1)]
297 + break
298
299 assert param.shape == loaded_weight.shape
300 param.data.copy_(loaded_weight)