@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a60b353005
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

support sharding llama2-70b on more than 8 GPUs (#1209)

Co-authored-by: JiCheng <247153481@qq.com>

Zhuohan Li committed Oct 2, 2023 at 15:26 UTC a60b35300553448cb35c6356bab16d8a4b437d7d
1 file changed +29 -14
vllm/model_executor/models/llama.py
+29 -14
@@ -103,8 +103,16 @@ class LlamaAttention(nn.Module):
103 assert self.total_num_heads % tp_size == 0
104 self.num_heads = self.total_num_heads // tp_size
105 self.total_num_kv_heads = num_kv_heads
106 - assert self.total_num_kv_heads % tp_size == 0
107 - self.num_kv_heads = self.total_num_kv_heads // tp_size
106 + if self.total_num_kv_heads >= tp_size:
107 + # Number of KV heads is greater than TP size, so we partition
108 + # the KV heads across multiple tensor parallel GPUs.
109 + assert self.total_num_kv_heads % tp_size == 0
110 + else:
111 + # Number of KV heads is less than TP size, so we replicate
112 + # the KV heads across multiple tensor parallel GPUs.
113 + assert tp_size % self.total_num_kv_heads == 0
114 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
115 + num_kv_heads_replicas = max(1, tp_size // self.total_num_kv_heads)
116 self.head_dim = hidden_size // self.total_num_heads
117 self.q_size = self.num_heads * self.head_dim
118 self.kv_size = self.num_kv_heads * self.head_dim
@@ -114,7 +122,8 @@ class LlamaAttention(nn.Module):
122
123 self.qkv_proj = ParallelLinear.column(
124 hidden_size,
117 - (self.total_num_heads + 2 * self.total_num_kv_heads) *
125 + (self.total_num_heads +
126 + 2 * self.total_num_kv_heads * num_kv_heads_replicas) *
127 self.head_dim,
128 bias=False,
129 gather_output=False,
@@ -323,11 +332,15 @@ class LlamaForCausalLM(nn.Module):
332 row_parallel_weights.append(f"{layer}.{suffix}")
333
334 tp_size = get_tensor_model_parallel_world_size()
326 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
335 + tp_rank = get_tensor_model_parallel_rank()
336 q_proj_shard_size = (self.config.hidden_size // tp_size)
337 + num_kv_heads_replicas = max(1,
338 + tp_size // self.config.num_key_value_heads)
339 + num_kv_heads_per_gpu = max(1,
340 + self.config.num_key_value_heads // tp_size)
341 kv_proj_shard_size = (self.config.hidden_size //
342 self.config.num_attention_heads *
330 - self.config.num_key_value_heads // tp_size)
343 + num_kv_heads_per_gpu)
344 attention_weight_specs = [
345 # (weight_name, shard_size, offset)
346 ("q_proj", q_proj_shard_size, 0),
@@ -363,9 +376,13 @@ class LlamaForCausalLM(nn.Module):
376 shard_size //= self.quant_config.pack_factor
377 offset //= self.quant_config.pack_factor
378
366 - loaded_weight = loaded_weight[
367 - shard_size * tensor_model_parallel_rank:shard_size *
368 - (tensor_model_parallel_rank + 1)]
379 + if weight_name in ["k_proj", "v_proj"]:
380 + shard_id = tp_rank // num_kv_heads_replicas
381 + else:
382 + shard_id = tp_rank
383 + loaded_weight = loaded_weight[shard_size *
384 + shard_id:shard_size *
385 + (shard_id + 1)]
386 param_slice = param.data[offset:offset + shard_size]
387 assert param_slice.shape == loaded_weight.shape
388
@@ -384,9 +401,8 @@ class LlamaForCausalLM(nn.Module):
401 param = param.T
402
403 shard_size = param.shape[0] // 2
387 - loaded_weight = loaded_weight[
388 - shard_size * tensor_model_parallel_rank:shard_size *
389 - (tensor_model_parallel_rank + 1)]
404 + loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
405 + (tp_rank + 1)]
406 param_slice = param.data[shard_size * stride_id:shard_size *
407 (stride_id + 1)]
408 assert param_slice.shape == loaded_weight.shape
@@ -402,10 +418,9 @@ class LlamaForCausalLM(nn.Module):
418
419 if "embed_tokens" in name or "lm_head" in name:
420 load_padded_tensor_parallel_vocab(param, loaded_weight,
405 - tensor_model_parallel_rank)
421 + tp_rank)
422 continue
423
424 load_tensor_parallel_weights(param, loaded_weight, name,
425 column_parallel_weights,
410 - row_parallel_weights,
411 - tensor_model_parallel_rank)
426 + row_parallel_weights, tp_rank)