A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add support for LLaMA-2 (#505)
Zhuohan Li committed
Jul 20, 2023 at 11:38 UTC
6fc2a38b110f9ba6037b31ee016f20df32426877
7 files changed
+67
-38
README.md
+2
-1
@@ -17,6 +17,7 @@ Easy, fast, and cheap LLM serving for everyone
17
---
18
19
*Latest News* 🔥
20
+- [2023/07] Added support for LLaMA-2! You can run and serve 7B/13B/70B LLaMA-2s on vLLM with a single command!
21
- [2023/06] Serving vLLM On any Cloud with SkyPilot. Check out a 1-click [example](https://github.com/skypilot-org/skypilot/blob/master/llm/vllm) to start the vLLM demo, and the [blog post](https://blog.skypilot.co/serving-llm-24x-faster-on-the-cloud-with-vllm-and-skypilot/) for the story behind vLLM development on the clouds.
22
- [2023/06] We officially released vLLM! FastChat-vLLM integration has powered [LMSYS Vicuna and Chatbot Arena](https://chat.lmsys.org) since mid-April. Check out our [blog post](https://vllm.ai).
23
@@ -46,7 +47,7 @@ vLLM seamlessly supports many Huggingface models, including the following archit
47
- GPT BigCode (`bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, etc.)
48
- GPT-J (`EleutherAI/gpt-j-6b`, `nomic-ai/gpt4all-j`, etc.)
49
- GPT-NeoX (`EleutherAI/gpt-neox-20b`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc.)
49
-- LLaMA (`lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
50
+- LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
51
- MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
52
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
53
csrc/pos_encoding_kernels.cu
+13
-8
@@ -7,11 +7,12 @@ template<typename scalar_t>
7
__global__ void rotary_embedding_neox_kernel(
8
const int64_t* __restrict__ positions, // [num_tokens]
9
scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
10
- scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
10
+ scalar_t* __restrict__ key, // [num_tokens, num_kv_heads, head_size]
11
const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
12
const int rot_dim,
13
const int stride,
14
const int num_heads,
15
+ const int num_kv_heads,
16
const int head_size) {
17
// Each thread block is responsible for one token.
18
const int token_idx = blockIdx.x;
@@ -19,8 +20,8 @@ __global__ void rotary_embedding_neox_kernel(
20
const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
21
22
const int embed_dim = rot_dim / 2;
22
- const int n = num_heads * embed_dim;
23
- for (int i = threadIdx.x; i < n; i += blockDim.x) {
23
+ const int nq = num_heads * embed_dim;
24
+ for (int i = threadIdx.x; i < nq; i += blockDim.x) {
25
const int head_idx = i / embed_dim;
26
const int token_head = token_idx * stride + head_idx * head_size;
27
@@ -39,10 +40,12 @@ __global__ void rotary_embedding_neox_kernel(
40
query[out_x] = q_x * cos - q_y * sin;
41
query[out_y] = q_y * cos + q_x * sin;
42
42
- const scalar_t k_x = key[token_head + x_index];
43
- const scalar_t k_y = key[token_head + y_index];
44
- key[out_x] = k_x * cos - k_y * sin;
45
- key[out_y] = k_y * cos + k_x * sin;
43
+ if (head_idx < num_kv_heads) {
44
+ const scalar_t k_x = key[token_head + x_index];
45
+ const scalar_t k_y = key[token_head + y_index];
46
+ key[out_x] = k_x * cos - k_y * sin;
47
+ key[out_y] = k_y * cos + k_x * sin;
48
+ }
49
}
50
}
51
@@ -51,13 +54,14 @@ __global__ void rotary_embedding_neox_kernel(
54
void rotary_embedding_neox(
55
torch::Tensor& positions, // [num_tokens]
56
torch::Tensor& query, // [num_tokens, num_heads * head_size]
54
- torch::Tensor& key, // [num_tokens, num_heads * head_size]
57
+ torch::Tensor& key, // [num_tokens, num_kv_heads * head_size]
58
int head_size,
59
torch::Tensor& cos_sin_cache) // [max_position, rot_dim]
60
{
61
int num_tokens = query.size(0);
62
int rot_dim = cos_sin_cache.size(1);
63
int num_heads = query.size(1) / head_size;
64
+ int num_kv_heads = key.size(1) / head_size;
65
int stride = query.stride(0);
66
TORCH_CHECK(stride == key.stride(0));
67
@@ -78,6 +82,7 @@ void rotary_embedding_neox(
82
rot_dim,
83
stride,
84
num_heads,
85
+ num_kv_heads,
86
head_size);
87
});
88
}
docs/source/models/supported_models.rst
+2
-2
@@ -30,8 +30,8 @@ Alongside each architecture, we include some popular models that use it.
30
- GPT-NeoX, Pythia, OpenAssistant, Dolly V2, StableLM
31
- :code:`EleutherAI/gpt-neox-20b`, :code:`EleutherAI/pythia-12b`, :code:`OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5`, :code:`databricks/dolly-v2-12b`, :code:`stabilityai/stablelm-tuned-alpha-7b`, etc.
32
* - :code:`LlamaForCausalLM`
33
- - LLaMA, Vicuna, Alpaca, Koala, Guanaco
34
- - :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`young-geng/koala`, :code:`JosephusCheung/Guanaco`, etc.
33
+ - LLaMA, LLaMA-2, Vicuna, Alpaca, Koala, Guanaco
34
+ - :code:`meta-llama/Llama-2-13b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`young-geng/koala`, :code:`JosephusCheung/Guanaco`, etc.
35
* - :code:`MPTForCausalLM`
36
- MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter
37
- :code:`mosaicml/mpt-7b`, :code:`mosaicml/mpt-7b-storywriter`, :code:`mosaicml/mpt-30b`, etc.
requirements.txt
+1
-1
@@ -4,7 +4,7 @@ ray >= 2.5.1
4
sentencepiece # Required for LLaMA tokenizer.
5
numpy
6
torch >= 2.0.0
7
-transformers >= 4.28.0 # Required for LLaMA.
7
+transformers >= 4.31.0 # Required for LLaMA-2.
8
xformers >= 0.0.19
9
fastapi
10
uvicorn
vllm/config.py
+6
-1
@@ -100,7 +100,12 @@ class ModelConfig:
100
return 1
101
# For Falcon:
102
if getattr(self.hf_config, "n_head_kv", None) is not None:
103
- return self.hf_config.n_head_kv
103
+ return (self.hf_config.n_head_kv //
104
+ parallel_config.tensor_parallel_size)
105
+ # For LLaMA-2:
106
+ if getattr(self.hf_config, "num_key_value_heads", None) is not None:
107
+ return (self.hf_config.num_key_value_heads //
108
+ parallel_config.tensor_parallel_size)
109
total_num_attention_heads = self.hf_config.num_attention_heads
110
return total_num_attention_heads // parallel_config.tensor_parallel_size
111
vllm/model_executor/layers/attention.py
+9
-6
@@ -128,7 +128,8 @@ class PagedAttention(nn.Module):
128
query: shape = [num_generation_tokens, num_heads, head_size]
129
key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
130
block_size, x]
131
- value_cache: shape = [num_blocks, num_kv_heads, head_size, block_size]
131
+ value_cache: shape = [num_blocks, num_kv_heads, head_size,
132
+ block_size]
133
input_metadata: metadata for paged attention.
134
"""
135
block_size = value_cache.shape[3]
@@ -241,8 +242,9 @@ class PagedAttentionWithRoPE(PagedAttention):
242
rotary_dim: int,
243
max_position: int = 8192,
244
base: int = 10000,
245
+ num_kv_heads: Optional[int] = None,
246
) -> None:
245
- super().__init__(num_heads, head_size, scale)
247
+ super().__init__(num_heads, head_size, scale, num_kv_heads)
248
249
# Create the cos and sin cache.
250
inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
@@ -276,11 +278,12 @@ class PagedAttentionWithRoPE(PagedAttention):
278
Args:
279
positions: shape = [num_tokens]
280
query: shape = [num_tokens, num_heads * head_size]
279
- key: shape = [num_tokens, num_heads * head_size]
280
- value: shape = [num_tokens, num_heads * head_size]
281
- key_cache: shape = [num_blocks, num_heads, head_size/x,
281
+ key: shape = [num_tokens, num_kv_heads * head_size]
282
+ value: shape = [num_tokens, num_kv_heads * head_size]
283
+ key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
284
block_size, x]
283
- value_cache: shape = [num_blocks, num_heads, head_size, block_size]
285
+ value_cache: shape = [num_blocks, num_kv_heads, head_size,
286
+ block_size]
287
input_metadata: metadata for paged attention.
288
cache_event: event to wait for the cache operations to finish.
289
vllm/model_executor/models/llama.py
+34
-19
@@ -84,21 +84,26 @@ class LlamaAttention(nn.Module):
84
self,
85
hidden_size: int,
86
num_heads: int,
87
+ num_kv_heads: int,
88
):
89
super().__init__()
90
self.hidden_size = hidden_size
90
- tensor_model_parallel_world_size = (
91
- get_tensor_model_parallel_world_size())
91
+ tp_size = get_tensor_model_parallel_world_size()
92
self.total_num_heads = num_heads
93
- assert self.total_num_heads % tensor_model_parallel_world_size == 0
94
- self.num_heads = (self.total_num_heads //
95
- tensor_model_parallel_world_size)
93
+ assert self.total_num_heads % tp_size == 0
94
+ self.num_heads = self.total_num_heads // tp_size
95
+ self.total_num_kv_heads = num_kv_heads
96
+ assert self.total_num_kv_heads % tp_size == 0
97
+ self.num_kv_heads = self.total_num_kv_heads // tp_size
98
self.head_dim = hidden_size // self.total_num_heads
99
+ self.q_size = self.num_heads * self.head_dim
100
+ self.kv_size = self.num_kv_heads * self.head_dim
101
self.scaling = self.head_dim**-0.5
102
103
self.qkv_proj = ColumnParallelLinear(
104
hidden_size,
101
- 3 * self.total_num_heads * self.head_dim,
105
+ (self.total_num_heads + 2 * self.total_num_kv_heads) *
106
+ self.head_dim,
107
bias=False,
108
gather_output=False,
109
perform_initialization=False,
@@ -113,7 +118,8 @@ class LlamaAttention(nn.Module):
118
self.attn = PagedAttentionWithRoPE(self.num_heads,
119
self.head_dim,
120
self.scaling,
116
- rotary_dim=self.head_dim)
121
+ rotary_dim=self.head_dim,
122
+ num_kv_heads=self.num_kv_heads)
123
124
def forward(
125
self,
@@ -124,7 +130,7 @@ class LlamaAttention(nn.Module):
130
cache_event: Optional[torch.cuda.Event],
131
) -> torch.Tensor:
132
qkv, _ = self.qkv_proj(hidden_states)
127
- q, k, v = qkv.chunk(chunks=3, dim=-1)
133
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
134
k_cache, v_cache = kv_cache
135
attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
136
input_metadata, cache_event)
@@ -140,6 +146,7 @@ class LlamaDecoderLayer(nn.Module):
146
self.self_attn = LlamaAttention(
147
hidden_size=self.hidden_size,
148
num_heads=config.num_attention_heads,
149
+ num_kv_heads=config.num_key_value_heads,
150
)
151
self.mlp = LlamaMLP(
152
hidden_size=self.hidden_size,
@@ -259,9 +266,19 @@ class LlamaForCausalLM(nn.Module):
266
model_name_or_path: str,
267
cache_dir: Optional[str] = None,
268
use_np_cache: bool = False):
262
- tensor_model_parallel_world_size = (
263
- get_tensor_model_parallel_world_size())
269
+ tp_size = get_tensor_model_parallel_world_size()
270
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
271
+ q_proj_shard_size = (self.config.hidden_size // tp_size)
272
+ kv_proj_shard_size = (self.config.hidden_size //
273
+ self.config.num_attention_heads *
274
+ self.config.num_key_value_heads // tp_size)
275
+ attention_weight_specs = [
276
+ # (weight_name, shard_size, offset)
277
+ ("q_proj", q_proj_shard_size, 0),
278
+ ("k_proj", kv_proj_shard_size, q_proj_shard_size),
279
+ ("v_proj", kv_proj_shard_size,
280
+ q_proj_shard_size + kv_proj_shard_size),
281
+ ]
282
state_dict = self.state_dict()
283
284
for name, loaded_weight in hf_model_weights_iterator(
@@ -272,8 +289,7 @@ class LlamaForCausalLM(nn.Module):
289
if "embed_tokens" in name or "lm_head" in name:
290
param = state_dict[name]
291
# Consider padding in the vocab size.
275
- padded_vocab_size = (param.shape[0] *
276
- tensor_model_parallel_world_size)
292
+ padded_vocab_size = (param.shape[0] * tp_size)
293
num_extra_rows = padded_vocab_size - self.config.vocab_size
294
extra_rows = torch.empty(num_extra_rows,
295
loaded_weight.shape[1])
@@ -281,18 +297,17 @@ class LlamaForCausalLM(nn.Module):
297
loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
298
299
is_attention_weight = False
284
- for stride_id, att_weight_name in enumerate(
285
- ["q_proj", "k_proj", "v_proj"]):
286
- if att_weight_name not in name:
300
+ for weight_name, shard_size, offset in attention_weight_specs:
301
+ if weight_name not in name:
302
continue
288
- param = state_dict[name.replace(att_weight_name, "qkv_proj")]
289
- shard_size = param.shape[0] // 3
303
+ param = state_dict[name.replace(weight_name, "qkv_proj")]
304
+
305
loaded_weight = loaded_weight[
306
shard_size * tensor_model_parallel_rank:shard_size *
307
(tensor_model_parallel_rank + 1)]
293
- param_slice = param.data[shard_size * stride_id:shard_size *
294
- (stride_id + 1)]
308
+ param_slice = param.data[offset:offset + shard_size]
309
assert param_slice.shape == loaded_weight.shape
310
+
311
param_slice.copy_(loaded_weight)
312
is_attention_weight = True
313
break