A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
support sharding llama2-70b on more than 8 GPUs (#1209)
Co-authored-by: JiCheng <247153481@qq.com>
Zhuohan Li committed
Oct 2, 2023 at 15:26 UTC
a60b35300553448cb35c6356bab16d8a4b437d7d
1 file changed
+29
-14
vllm/model_executor/models/llama.py
+29
-14
@@ -103,8 +103,16 @@ class LlamaAttention(nn.Module):
103
assert self.total_num_heads % tp_size == 0
104
self.num_heads = self.total_num_heads // tp_size
105
self.total_num_kv_heads = num_kv_heads
106
- assert self.total_num_kv_heads % tp_size == 0
107
- self.num_kv_heads = self.total_num_kv_heads // tp_size
106
+ if self.total_num_kv_heads >= tp_size:
107
+ # Number of KV heads is greater than TP size, so we partition
108
+ # the KV heads across multiple tensor parallel GPUs.
109
+ assert self.total_num_kv_heads % tp_size == 0
110
+ else:
111
+ # Number of KV heads is less than TP size, so we replicate
112
+ # the KV heads across multiple tensor parallel GPUs.
113
+ assert tp_size % self.total_num_kv_heads == 0
114
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
115
+ num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
116
self.head_dim = hidden_size // self.total_num_heads
117
self.q_size = self.num_heads * self.head_dim
118
self.kv_size = self.num_kv_heads * self.head_dim
@@ -114,7 +122,8 @@ class LlamaAttention(nn.Module):
122
123
self.qkv_proj = ParallelLinear.column(
124
hidden_size,
117
- (self.total_num_heads + 2 * self.total_num_kv_heads) *
125
+ (self.total_num_heads +
126
+ 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
127
self.head_dim,
128
bias=False,
129
gather_output=False,
@@ -323,11 +332,15 @@ class LlamaForCausalLM(nn.Module):
332
row_parallel_weights.append(f"{layer}.{suffix}")
333
334
tp_size = get_tensor_model_parallel_world_size()
326
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
335
+ tp_rank = get_tensor_model_parallel_rank()
336
q_proj_shard_size = (self.config.hidden_size // tp_size)
337
+ num_kv_heads_replicas = max(1,
338
+ tp_size // self.config.num_key_value_heads)
339
+ num_kv_heads_per_gpu = max(1,
340
+ self.config.num_key_value_heads // tp_size)
341
kv_proj_shard_size = (self.config.hidden_size //
342
self.config.num_attention_heads *
330
- self.config.num_key_value_heads // tp_size)
343
+ num_kv_heads_per_gpu)
344
attention_weight_specs = [
345
# (weight_name, shard_size, offset)
346
("q_proj", q_proj_shard_size, 0),
@@ -363,9 +376,13 @@ class LlamaForCausalLM(nn.Module):
376
shard_size //= self.quant_config.pack_factor
377
offset //= self.quant_config.pack_factor
378
366
- loaded_weight = loaded_weight[
367
- shard_size * tensor_model_parallel_rank:shard_size *
368
- (tensor_model_parallel_rank + 1)]
379
+ if weight_name in ["k_proj", "v_proj"]:
380
+ shard_id = tp_rank // num_kv_heads_replicas
381
+ else:
382
+ shard_id = tp_rank
383
+ loaded_weight = loaded_weight[shard_size *
384
+ shard_id:shard_size *
385
+ (shard_id + 1)]
386
param_slice = param.data[offset:offset + shard_size]
387
assert param_slice.shape == loaded_weight.shape
388
@@ -384,9 +401,8 @@ class LlamaForCausalLM(nn.Module):
401
param = param.T
402
403
shard_size = param.shape[0] // 2
387
- loaded_weight = loaded_weight[
388
- shard_size * tensor_model_parallel_rank:shard_size *
389
- (tensor_model_parallel_rank + 1)]
404
+ loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
405
+ (tp_rank + 1)]
406
param_slice = param.data[shard_size * stride_id:shard_size *
407
(stride_id + 1)]
408
assert param_slice.shape == loaded_weight.shape
@@ -402,10 +418,9 @@ class LlamaForCausalLM(nn.Module):
418
419
if "embed_tokens" in name or "lm_head" in name:
420
load_padded_tensor_parallel_vocab(param, loaded_weight,
405
- tensor_model_parallel_rank)
421
+ tp_rank)
422
continue
423
424
load_tensor_parallel_weights(param, loaded_weight, name,
425
column_parallel_weights,
410
- row_parallel_weights,
411
- tensor_model_parallel_rank)
426
+ row_parallel_weights, tp_rank)