A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add Falcon support (new) (#592)
Zhuohan Li committed
Aug 2, 2023 at 14:04 UTC
1b0bd0fe8a4a9de749b9d4618758ff20c8658d86
16 files changed
+680
-122
README.md
+1
@@ -44,6 +44,7 @@ vLLM seamlessly supports many Huggingface models, including the following archit
44
45
- Baichuan-7B (`baichuan-inc/Baichuan-7B`)
46
- BLOOM (`bigscience/bloom`, `bigscience/bloomz`, etc.)
47
+- Falcon (`tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc.)
48
- GPT-2 (`gpt2`, `gpt2-xl`, etc.)
49
- GPT BigCode (`bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, etc.)
50
- GPT-J (`EleutherAI/gpt-j-6b`, `nomic-ai/gpt4all-j`, etc.)
csrc/pos_encoding_kernels.cu
+29
-13
@@ -10,7 +10,8 @@ __global__ void rotary_embedding_neox_kernel(
10
scalar_t* __restrict__ key, // [num_tokens, num_kv_heads, head_size]
11
const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
12
const int rot_dim,
13
- const int stride,
13
+ const int query_stride,
14
+ const int key_stride,
15
const int num_heads,
16
const int num_kv_heads,
17
const int head_size) {
@@ -23,14 +24,14 @@ __global__ void rotary_embedding_neox_kernel(
24
const int nq = num_heads * embed_dim;
25
for (int i = threadIdx.x; i < nq; i += blockDim.x) {
26
const int head_idx = i / embed_dim;
26
- const int token_head = token_idx * stride + head_idx * head_size;
27
+ const int token_head = token_idx * query_stride + head_idx * head_size;
28
29
const int rot_offset = i % embed_dim;
30
const int x_index = rot_offset;
31
const int y_index = embed_dim + rot_offset;
32
32
- const int out_x = token_idx * stride + head_idx * head_size + x_index;
33
- const int out_y = token_idx * stride + head_idx * head_size + y_index;
33
+ const int out_x = token_idx * query_stride + head_idx * head_size + x_index;
34
+ const int out_y = token_idx * query_stride + head_idx * head_size + y_index;
35
36
const scalar_t cos = __ldg(cache_ptr + x_index);
37
const scalar_t sin = __ldg(cache_ptr + y_index);
@@ -39,13 +40,27 @@ __global__ void rotary_embedding_neox_kernel(
40
const scalar_t q_y = query[token_head + y_index];
41
query[out_x] = q_x * cos - q_y * sin;
42
query[out_y] = q_y * cos + q_x * sin;
43
+ }
44
+
45
+ const int nk = num_kv_heads * embed_dim;
46
+ for (int i = threadIdx.x; i < nk; i += blockDim.x) {
47
+ const int head_idx = i / embed_dim;
48
+ const int token_head = token_idx * key_stride + head_idx * head_size;
49
+
50
+ const int rot_offset = i % embed_dim;
51
+ const int x_index = rot_offset;
52
+ const int y_index = embed_dim + rot_offset;
53
+
54
+ const int out_x = token_idx * key_stride + head_idx * head_size + x_index;
55
+ const int out_y = token_idx * key_stride + head_idx * head_size + y_index;
56
+
57
+ const scalar_t cos = __ldg(cache_ptr + x_index);
58
+ const scalar_t sin = __ldg(cache_ptr + y_index);
59
43
- if (head_idx < num_kv_heads) {
44
- const scalar_t k_x = key[token_head + x_index];
45
- const scalar_t k_y = key[token_head + y_index];
46
- key[out_x] = k_x * cos - k_y * sin;
47
- key[out_y] = k_y * cos + k_x * sin;
48
- }
60
+ const scalar_t k_x = key[token_head + x_index];
61
+ const scalar_t k_y = key[token_head + y_index];
62
+ key[out_x] = k_x * cos - k_y * sin;
63
+ key[out_y] = k_y * cos + k_x * sin;
64
}
65
}
66
@@ -62,8 +77,8 @@ void rotary_embedding_neox(
77
int rot_dim = cos_sin_cache.size(1);
78
int num_heads = query.size(1) / head_size;
79
int num_kv_heads = key.size(1) / head_size;
65
- int stride = query.stride(0);
66
- TORCH_CHECK(stride == key.stride(0));
80
+ int query_stride = query.stride(0);
81
+ int key_stride = key.stride(0);
82
83
dim3 grid(num_tokens);
84
dim3 block(std::min(num_heads * rot_dim / 2, 512));
@@ -80,7 +95,8 @@ void rotary_embedding_neox(
95
key.data_ptr<scalar_t>(),
96
cos_sin_cache.data_ptr<scalar_t>(),
97
rot_dim,
83
- stride,
98
+ query_stride,
99
+ key_stride,
100
num_heads,
101
num_kv_heads,
102
head_size);
docs/source/models/supported_models.rst
+3
@@ -20,6 +20,9 @@ Alongside each architecture, we include some popular models that use it.
20
* - :code:`BloomForCausalLM`
21
- BLOOM, BLOOMZ, BLOOMChat
22
- :code:`bigscience/bloom`, :code:`bigscience/bloomz`, etc.
23
+ * - :code:`FalconForCausalLM`
24
+ - Falcon
25
+ - :code:`tiiuae/falcon-7b``, :code:`tiiuae/falcon-40b`, :code:`tiiuae/falcon-rw-7b`, etc.
26
* - :code:`GPT2LMHeadModel`
27
- GPT-2
28
- :code:`gpt2`, :code:`gpt2-xl`, etc.
examples/llm_engine_example.py
+2
-1
@@ -10,7 +10,8 @@ def main(args: argparse.Namespace):
10
11
# Test the following prompts.
12
test_prompts = [
13
- ("A robot may not injure a human being", SamplingParams()),
13
+ ("A robot may not injure a human being",
14
+ SamplingParams(temperature=0.0)),
15
("To be or not to be,",
16
SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17
("What is the meaning of life?",
vllm/config.py
+7
-2
@@ -94,8 +94,13 @@ class ModelConfig:
94
return self.hf_config.hidden_size // self.hf_config.num_attention_heads
95
96
def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
97
- # For GPTBigCode:
98
- if getattr(self.hf_config, "multi_query", False):
97
+ # For GPTBigCode & Falcon:
98
+ # Note: for falcon, when new_decoder_architecture is True, the
99
+ # multi_query flag is ignored and we use n_head_kv for the number of
100
+ # KV heads.
101
+ if (getattr(self.hf_config, "multi_query", False) and
102
+ (self.hf_config.model_type == "falcon" and
103
+ not getattr(self.hf_config, "new_decoder_architecture", False))):
104
# Multi-query attention, only one KV head.
105
return 1
106
# For Falcon:
vllm/model_executor/layers/attention.py
+24
-12
@@ -314,14 +314,13 @@ class PagedAttentionWithRoPE(PagedAttention):
314
class PagedAttentionWithALiBi(PagedAttention):
315
"""PagedAttention with ALiBi attention bias."""
316
317
- def __init__(
318
- self,
319
- num_heads: int,
320
- head_size: int,
321
- scale: float,
322
- slopes: List[float],
323
- ) -> None:
324
- super().__init__(num_heads, head_size, scale)
317
+ def __init__(self,
318
+ num_heads: int,
319
+ head_size: int,
320
+ scale: float,
321
+ slopes: List[float],
322
+ num_kv_heads: Optional[int] = None) -> None:
323
+ super().__init__(num_heads, head_size, scale, num_kv_heads)
324
assert len(slopes) == num_heads
325
326
slopes = torch.tensor(slopes, dtype=torch.float32)
@@ -334,6 +333,11 @@ class PagedAttentionWithALiBi(PagedAttention):
333
# Generates ALiBi mask for each prompt.
334
for prompt_len in input_metadata.prompt_lens:
335
bias = torch.arange(prompt_len)
336
+ # Note(zhuohan): HF uses
337
+ # `bias = bias[None, :].repeat(prompt_len, 1)`
338
+ # here. We find that both biases give the same results, but
339
+ # the bias below more accurately follows the original ALiBi
340
+ # paper.
341
bias = bias[None, :] - bias[:, None]
342
bias = bias.to(self.alibi_slopes.device)
343
@@ -363,10 +367,17 @@ class PagedAttentionWithALiBi(PagedAttention):
367
Args:
368
output: shape = [num_prompt_tokens, num_heads, head_size]
369
query: shape = [num_prompt_tokens, num_heads, head_size]
366
- key: shape = [num_prompt_tokens, num_heads, head_size]
367
- value: shape = [num_prompt_tokens, num_heads, head_size]
370
+ key: shape = [num_prompt_tokens, num_kv_heads, head_size]
371
+ value: shape = [num_prompt_tokens, num_kv_heads, head_size]
372
input_metadata: metadata for paged attention.
373
"""
374
+ if self.num_kv_heads != self.num_heads:
375
+ # Project the key and value tensors to the desired number of heads.
376
+ key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
377
+ value = torch.repeat_interleave(value,
378
+ self.num_queries_per_kv,
379
+ dim=1)
380
+
381
# FIXME(woosuk): Because xformers does not support dynamic sequence
382
# lengths with custom attention bias, we process each prompt one by
383
# one. This is inefficient, especially when we have many short prompts.
@@ -400,9 +411,10 @@ class PagedAttentionWithALiBi(PagedAttention):
411
Args:
412
output: shape = [num_generation_tokens, num_heads, head_size]
413
query: shape = [num_generation_tokens, num_heads, head_size]
403
- key_cache: shape = [num_blocks, num_heads, head_size/x,
414
+ key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
415
block_size, x]
405
- value_cache: shape = [num_blocks, num_heads, head_size, block_size]
416
+ value_cache: shape = [num_blocks, num_kv_heads, head_size,
417
+ block_size]
418
input_metadata: metadata for paged attention.
419
"""
420
block_size = value_cache.shape[3]
vllm/model_executor/model_loader.py
+2
@@ -14,6 +14,7 @@ _MODEL_REGISTRY = {
14
"BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
15
"BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
16
"BloomForCausalLM": BloomForCausalLM,
17
+ "FalconForCausalLM": FalconForCausalLM,
18
"GPT2LMHeadModel": GPT2LMHeadModel,
19
"GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
20
"GPTJForCausalLM": GPTJForCausalLM,
@@ -22,6 +23,7 @@ _MODEL_REGISTRY = {
23
"LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
24
"MPTForCausalLM": MPTForCausalLM,
25
"OPTForCausalLM": OPTForCausalLM,
26
+ "RWForCausalLM": FalconForCausalLM,
27
}
28
29
vllm/model_executor/models/__init__.py
+2
@@ -1,5 +1,6 @@
1
from vllm.model_executor.models.baichuan import BaiChuanForCausalLM, BaichuanForCausalLM
2
from vllm.model_executor.models.bloom import BloomForCausalLM
3
+from vllm.model_executor.models.falcon import FalconForCausalLM
4
from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
5
from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
6
from vllm.model_executor.models.gpt_j import GPTJForCausalLM
@@ -12,6 +13,7 @@ __all__ = [
13
"BaiChuanForCausalLM",
14
"BaichuanForCausalLM",
15
"BloomForCausalLM",
16
+ "FalconForCausalLM",
17
"GPT2LMHeadModel",
18
"GPTBigCodeForCausalLM",
19
"GPTJForCausalLM",
vllm/model_executor/models/falcon.py
new
+496
@@ -0,0 +1,496 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/a5cc30d72ae2dc19af534e4b35c986cc28db1275/src/transformers/models/falcon/modeling_falcon.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2023 the Falcon authors and HuggingFace Inc. team. All rights
6
+# reserved.
7
+#
8
+# Licensed under the Apache License, Version 2.0 (the "License");
9
+# you may not use this file except in compliance with the License.
10
+# You may obtain a copy of the License at
11
+#
12
+# http://www.apache.org/licenses/LICENSE-2.0
13
+#
14
+# Unless required by applicable law or agreed to in writing, software
15
+# distributed under the License is distributed on an "AS IS" BASIS,
16
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+# See the License for the specific language governing permissions and
18
+# limitations under the License.
19
+"""PyTorch Falcon model."""
20
+
21
+import math
22
+from typing import Dict, List, Optional, Tuple, Union
23
+
24
+import torch
25
+from torch import nn
26
+from torch.nn import LayerNorm
27
+from transformers import FalconConfig as HF_FalconConfig
28
+
29
+from vllm.model_executor.input_metadata import InputMetadata
30
+from vllm.model_executor.layers.attention import (PagedAttention,
31
+ PagedAttentionWithALiBi,
32
+ PagedAttentionWithRoPE)
33
+from vllm.model_executor.layers.sampler import Sampler
34
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35
+ load_tensor_parallel_weights)
36
+from vllm.model_executor.parallel_utils.parallel_state import (
37
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
+from vllm.model_executor.parallel_utils.tensor_parallel import (
39
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear,
40
+ reduce_from_tensor_model_parallel_region)
41
+from vllm.sequence import SequenceOutputs
42
+from vllm.transformers_utils.configs import RWConfig
43
+
44
+KVCache = Tuple[torch.Tensor, torch.Tensor]
45
+FalconConfig = Union[HF_FalconConfig, RWConfig]
46
+
47
+
48
+# NOTE(Hesslow): Unfortunately we did not fuse matmul and bias during
49
+# training, this means that there's one additional quantization to bfloat16
50
+# between the operations. In order not to degrade the quality of our HF-port,
51
+# we keep these characteristics in the final model.
52
+class FalconLinear(nn.Linear):
53
+
54
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
55
+ hidden_states = x @ self.weight.T
56
+ if self.bias is None:
57
+ return hidden_states
58
+ return hidden_states + self.bias
59
+
60
+
61
+def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
62
+ closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
63
+ base = torch.tensor(2**(-(2**-(math.log2(closest_power_of_2) - 3))),
64
+ dtype=torch.float32)
65
+ powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
66
+ slopes = torch.pow(base, powers)
67
+
68
+ if closest_power_of_2 != total_num_heads:
69
+ extra_base = torch.tensor(
70
+ 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
71
+ dtype=torch.float32)
72
+ num_remaining_heads = min(closest_power_of_2,
73
+ total_num_heads - closest_power_of_2)
74
+ extra_powers = torch.arange(1,
75
+ 1 + 2 * num_remaining_heads,
76
+ 2,
77
+ dtype=torch.int32)
78
+ slopes = torch.cat(
79
+ [slopes, torch.pow(extra_base, extra_powers)], dim=0)
80
+
81
+ return slopes
82
+
83
+
84
+class FalconAttention(nn.Module):
85
+
86
+ def __init__(self, config: FalconConfig):
87
+ super().__init__()
88
+
89
+ self.hidden_size = config.hidden_size
90
+ tp_size = get_tensor_model_parallel_world_size()
91
+
92
+ self.total_num_heads = config.num_attention_heads
93
+ assert self.total_num_heads % tp_size == 0
94
+ self.num_heads = self.total_num_heads // tp_size
95
+ self.head_dim = self.hidden_size // self.total_num_heads
96
+ assert self.head_dim * self.total_num_heads == self.hidden_size
97
+
98
+ self.new_decoder_architecture = config.new_decoder_architecture
99
+ self.multi_query = config.multi_query
100
+
101
+ if self.new_decoder_architecture:
102
+ self.total_num_kv_heads = config.num_kv_heads
103
+ assert self.total_num_heads % tp_size == 0
104
+ self.num_kv_heads = self.total_num_kv_heads // tp_size
105
+ self.query_key_value = ColumnParallelLinear(
106
+ self.hidden_size,
107
+ (self.total_num_heads + 2 * self.total_num_kv_heads) *
108
+ self.head_dim,
109
+ bias=config.bias,
110
+ gather_output=False,
111
+ perform_initialization=False,
112
+ skip_bias_add=True,
113
+ )
114
+ elif self.multi_query:
115
+ self.total_num_kv_heads = 1
116
+ self.num_kv_heads = 1
117
+ self.query = ColumnParallelLinear(
118
+ self.hidden_size,
119
+ self.total_num_heads * self.head_dim,
120
+ bias=config.bias,
121
+ gather_output=False,
122
+ perform_initialization=False,
123
+ skip_bias_add=True,
124
+ )
125
+ self.key_value = FalconLinear(self.hidden_size,
126
+ 2 * self.head_dim,
127
+ bias=config.bias)
128
+ else:
129
+ self.total_num_kv_heads = self.total_num_heads
130
+ self.num_kv_heads = self.num_heads
131
+ self.query_key_value = ColumnParallelLinear(
132
+ self.hidden_size,
133
+ (self.total_num_heads + 2 * self.total_num_kv_heads) *
134
+ self.head_dim,
135
+ bias=config.bias,
136
+ gather_output=False,
137
+ perform_initialization=False,
138
+ skip_bias_add=True,
139
+ )
140
+
141
+ self.q_size = self.num_heads * self.head_dim
142
+ self.kv_size = self.num_kv_heads * self.head_dim
143
+
144
+ # Layer-wise attention scaling
145
+ self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim)
146
+ self.reduce_row_parallel_results = not (config.new_decoder_architecture
147
+ or config.parallel_attn)
148
+ self.dense = RowParallelLinear(
149
+ self.hidden_size,
150
+ self.hidden_size,
151
+ bias=config.bias,
152
+ input_is_parallel=True,
153
+ perform_initialization=False,
154
+ skip_bias_add=True,
155
+ reduce_results=self.reduce_row_parallel_results)
156
+
157
+ self.use_rotary = config.rotary
158
+ self.use_alibi = config.alibi
159
+ assert not (self.use_rotary and self.use_alibi), (
160
+ "Rotary and alibi are mutually exclusive.")
161
+
162
+ if self.use_rotary:
163
+ # TODO(zhuohan): Pass in correct `max_position``
164
+ self.attn = PagedAttentionWithRoPE(self.num_heads,
165
+ self.head_dim,
166
+ self.inv_norm_factor,
167
+ rotary_dim=self.head_dim,
168
+ num_kv_heads=self.num_kv_heads)
169
+ elif self.use_alibi:
170
+ tp_rank = get_tensor_model_parallel_rank()
171
+ head_start = tp_rank * self.num_heads
172
+ head_end = (tp_rank + 1) * self.num_heads
173
+ alibi_slopes = (_get_alibi_slopes(self.total_num_heads) *
174
+ self.inv_norm_factor)
175
+ alibi_slopes = alibi_slopes[head_start:head_end].tolist()
176
+ self.attn = PagedAttentionWithALiBi(self.num_heads,
177
+ self.head_dim,
178
+ self.inv_norm_factor,
179
+ alibi_slopes,
180
+ num_kv_heads=self.num_kv_heads)
181
+ else:
182
+ self.attn = PagedAttention(self.num_heads,
183
+ self.head_dim,
184
+ scale=self.inv_norm_factor,
185
+ num_kv_heads=self.num_kv_heads)
186
+
187
+ def forward(
188
+ self,
189
+ positions: torch.Tensor,
190
+ hidden_states: torch.Tensor,
191
+ kv_cache: KVCache,
192
+ input_metadata: InputMetadata,
193
+ cache_event: Optional[torch.cuda.Event],
194
+ ) -> torch.Tensor:
195
+ if not self.new_decoder_architecture and self.multi_query:
196
+ q, bias = self.query(hidden_states)
197
+ if bias is not None:
198
+ q += bias
199
+ kv = self.key_value(hidden_states)
200
+ k, v = kv.split([self.kv_size, self.kv_size], dim=-1)
201
+ else:
202
+ qkv, bias = self.query_key_value(hidden_states)
203
+ if bias is not None:
204
+ qkv += bias
205
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size],
206
+ dim=-1)
207
+ k_cache, v_cache = kv_cache
208
+ if self.use_rotary:
209
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
210
+ input_metadata, cache_event)
211
+ else:
212
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
213
+ cache_event)
214
+ attn_output, bias = self.dense(attn_output)
215
+ return attn_output, bias
216
+
217
+
218
+class FalconMLP(nn.Module):
219
+
220
+ def __init__(self, config: FalconConfig):
221
+ super().__init__()
222
+ hidden_size = config.hidden_size
223
+
224
+ self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
225
+ 4 * hidden_size,
226
+ bias=config.bias,
227
+ gather_output=False,
228
+ perform_initialization=False,
229
+ skip_bias_add=True)
230
+ self.act = nn.GELU()
231
+ self.reduce_row_parallel_results = not (config.new_decoder_architecture
232
+ or config.parallel_attn)
233
+ self.dense_4h_to_h = RowParallelLinear(
234
+ 4 * hidden_size,
235
+ hidden_size,
236
+ bias=config.bias,
237
+ input_is_parallel=True,
238
+ perform_initialization=False,
239
+ skip_bias_add=True,
240
+ reduce_results=self.reduce_row_parallel_results)
241
+
242
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
243
+ # NOTE(zhuohan): Following huggingface, we do not fuse bias add here.
244
+ x, bias = self.dense_h_to_4h(x)
245
+ if bias is not None:
246
+ x += bias
247
+ x = self.act(x)
248
+ x, bias = self.dense_4h_to_h(x)
249
+ return x, bias
250
+
251
+
252
+class FalconDecoderLayer(nn.Module):
253
+
254
+ def __init__(self, config: FalconConfig):
255
+ super().__init__()
256
+ hidden_size = config.hidden_size
257
+ self.num_heads = config.num_attention_heads
258
+ self.self_attention = FalconAttention(config)
259
+ self.mlp = FalconMLP(config)
260
+ self.config = config
261
+
262
+ if config.new_decoder_architecture:
263
+ # The layer norm before self-attention
264
+ self.ln_attn = LayerNorm(hidden_size,
265
+ eps=config.layer_norm_epsilon)
266
+ # The layer norm before the MLP
267
+ self.ln_mlp = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
268
+ else:
269
+ self.input_layernorm = LayerNorm(hidden_size,
270
+ eps=config.layer_norm_epsilon)
271
+ if not config.parallel_attn:
272
+ self.post_attention_layernorm = LayerNorm(
273
+ hidden_size, eps=config.layer_norm_epsilon)
274
+
275
+ self.reduce_row_parallel_results = not (config.new_decoder_architecture
276
+ or config.parallel_attn)
277
+
278
+ def forward(
279
+ self,
280
+ positions: torch.Tensor,
281
+ hidden_states: torch.Tensor,
282
+ kv_cache: KVCache,
283
+ input_metadata: InputMetadata,
284
+ cache_event: Optional[torch.cuda.Event],
285
+ ):
286
+ residual = hidden_states
287
+
288
+ if self.config.new_decoder_architecture:
289
+ attention_layernorm_out = self.ln_attn(hidden_states)
290
+ mlp_layernorm_out = self.ln_mlp(hidden_states)
291
+ else:
292
+ attention_layernorm_out = self.input_layernorm(hidden_states)
293
+
294
+ # Self attention.
295
+ attention_output, attention_bias = self.self_attention(
296
+ positions=positions,
297
+ hidden_states=attention_layernorm_out,
298
+ kv_cache=kv_cache,
299
+ input_metadata=input_metadata,
300
+ cache_event=cache_event,
301
+ )
302
+ if self.reduce_row_parallel_results and attention_bias is not None:
303
+ attention_output += attention_bias
304
+
305
+ if not self.config.new_decoder_architecture:
306
+ if self.config.parallel_attn:
307
+ mlp_layernorm_out = attention_layernorm_out
308
+ else:
309
+ residual += attention_output
310
+ mlp_layernorm_out = self.post_attention_layernorm(residual)
311
+
312
+ # MLP.
313
+ mlp_output, mlp_bias = self.mlp(mlp_layernorm_out)
314
+ if self.reduce_row_parallel_results and mlp_bias is not None:
315
+ mlp_output += mlp_bias
316
+
317
+ if not self.reduce_row_parallel_results:
318
+ # When MLP and Attention layers are parallel, we can use
319
+ # only one all-reduce operator to reduce the results from
320
+ # both MLP and Attention layers.
321
+ mlp_output += attention_output
322
+ mlp_output = reduce_from_tensor_model_parallel_region(mlp_output)
323
+ if attention_bias is not None:
324
+ mlp_output += attention_bias
325
+ if mlp_bias is not None:
326
+ mlp_output += mlp_bias
327
+
328
+ output = mlp_output + residual
329
+
330
+ return output
331
+
332
+
333
+class FalconModel(nn.Module):
334
+
335
+ def __init__(self, config: FalconConfig):
336
+ super().__init__()
337
+ self.config = config
338
+ self.embed_dim = config.hidden_size
339
+ self.num_heads = config.num_attention_heads
340
+ self.use_alibi = config.alibi
341
+
342
+ # Embedding + LN Embedding
343
+ self.word_embeddings = VocabParallelEmbedding(
344
+ config.vocab_size, self.embed_dim, perform_initialization=False)
345
+
346
+ # Transformer blocks
347
+ self.h = nn.ModuleList([
348
+ FalconDecoderLayer(config) for _ in range(config.num_hidden_layers)
349
+ ])
350
+
351
+ # Final Layer Norm
352
+ self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
353
+
354
+ def forward(
355
+ self,
356
+ input_ids: torch.LongTensor,
357
+ positions: torch.Tensor,
358
+ kv_caches: List[KVCache],
359
+ input_metadata: InputMetadata,
360
+ cache_events: Optional[List[torch.cuda.Event]],
361
+ ) -> torch.Tensor:
362
+ hidden_states = self.word_embeddings(input_ids)
363
+ for i in range(len(self.h)):
364
+ if cache_events is None:
365
+ cache_event = None
366
+ else:
367
+ cache_event = cache_events[i]
368
+ layer = self.h[i]
369
+ hidden_states = layer(
370
+ positions,
371
+ hidden_states,
372
+ kv_caches[i],
373
+ input_metadata,
374
+ cache_event,
375
+ )
376
+ hidden_states = self.ln_f(hidden_states)
377
+ return hidden_states
378
+
379
+
380
+class FalconForCausalLM(nn.Module):
381
+
382
+ def __init__(self, config: FalconConfig):
383
+ super().__init__()
384
+ self.config = config
385
+ self.transformer = FalconModel(config)
386
+ self.lm_head = ColumnParallelLinear(config.hidden_size,
387
+ config.vocab_size,
388
+ bias=False,
389
+ gather_output=False,
390
+ perform_initialization=False)
391
+ self.sampler = Sampler(config.vocab_size)
392
+
393
+ def forward(
394
+ self,
395
+ input_ids: torch.LongTensor,
396
+ positions: torch.Tensor,
397
+ kv_caches: List[KVCache],
398
+ input_metadata: InputMetadata,
399
+ cache_events: Optional[List[torch.cuda.Event]],
400
+ ) -> Dict[int, SequenceOutputs]:
401
+ hidden_states = self.transformer(
402
+ input_ids,
403
+ positions,
404
+ kv_caches,
405
+ input_metadata,
406
+ cache_events,
407
+ )
408
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
409
+ input_metadata)
410
+
411
+ return next_tokens
412
+
413
+ _column_parallel_weights = [
414
+ "word_embeddings.weight", "lm_head.weight", "dense_h_to_4h.weight",
415
+ "dense_h_to_4h.bias"
416
+ ]
417
+ _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
418
+
419
+ def load_weights(self,
420
+ model_name_or_path: str,
421
+ cache_dir: Optional[str] = None,
422
+ use_np_cache: bool = False):
423
+ tp_size = (get_tensor_model_parallel_world_size())
424
+ tp_rank = get_tensor_model_parallel_rank()
425
+
426
+ hidden_size = self.config.hidden_size
427
+ total_num_heads = self.config.num_attention_heads
428
+ num_heads = total_num_heads // tp_size
429
+ head_size = hidden_size // total_num_heads
430
+ head_start = tp_rank * num_heads
431
+ head_end = (tp_rank + 1) * num_heads
432
+ if self.config.new_decoder_architecture:
433
+ total_num_kv_heads = self.config.num_kv_heads
434
+ num_kv_heads = total_num_kv_heads // tp_size
435
+ separated_q_kv = False
436
+ kv_head_start = tp_rank * num_kv_heads
437
+ kv_head_end = (tp_rank + 1) * num_kv_heads
438
+ elif self.config.multi_query:
439
+ total_num_kv_heads = 1
440
+ num_kv_heads = 1
441
+ separated_q_kv = True
442
+ kv_head_start = 0
443
+ kv_head_end = 1
444
+ else:
445
+ total_num_kv_heads = total_num_heads
446
+ num_kv_heads = total_num_kv_heads // tp_size
447
+ separated_q_kv = False
448
+ kv_head_start = tp_rank * num_kv_heads
449
+ kv_head_end = (tp_rank + 1) * num_kv_heads
450
+ num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
451
+ state_dict = self.state_dict()
452
+
453
+ for name, loaded_weight in hf_model_weights_iterator(
454
+ model_name_or_path, cache_dir, use_np_cache):
455
+ if "query_key_value" in name:
456
+ loaded_weight_size = loaded_weight.size()
457
+ loaded_weight = loaded_weight.view(
458
+ total_num_kv_heads, num_query_heads_per_kv_head + 2,
459
+ head_size, *loaded_weight_size[1:])
460
+
461
+ wq = loaded_weight[:, :-2].reshape(-1, *loaded_weight_size[1:])
462
+ wk = loaded_weight[:, [-2]].reshape(-1,
463
+ *loaded_weight_size[1:])
464
+ wv = loaded_weight[:, [-1]].reshape(-1,
465
+ *loaded_weight_size[1:])
466
+
467
+ wq = wq[head_size * head_start:head_size * head_end]
468
+ wk = wk[head_size * kv_head_start:head_size * kv_head_end]
469
+ wv = wv[head_size * kv_head_start:head_size * kv_head_end]
470
+
471
+ if separated_q_kv:
472
+ loaded_weight_q = wq
473
+ loaded_weight_kv = torch.cat([wk, wv], dim=0)
474
+ q_weight_name = name.replace("query_key_value", "query")
475
+ kv_weight_name = name.replace("query_key_value",
476
+ "key_value")
477
+ load_tensor_parallel_weights(state_dict[q_weight_name],
478
+ loaded_weight_q,
479
+ q_weight_name,
480
+ self._column_parallel_weights,
481
+ self._row_parallel_weights,
482
+ tp_rank)
483
+ load_tensor_parallel_weights(state_dict[kv_weight_name],
484
+ loaded_weight_kv,
485
+ kv_weight_name,
486
+ self._column_parallel_weights,
487
+ self._row_parallel_weights,
488
+ tp_rank)
489
+ continue
490
+ else:
491
+ loaded_weight = torch.cat([wq, wk, wv], dim=0)
492
+
493
+ param = state_dict[name]
494
+ load_tensor_parallel_weights(param, loaded_weight, name,
495
+ self._column_parallel_weights,
496
+ self._row_parallel_weights, tp_rank)
vllm/model_executor/parallel_utils/parallel_state.py
+1
-72
@@ -44,7 +44,6 @@ _PIPELINE_GLOBAL_RANKS = None
44
# rank when broadcasting weights from src to all other data parallel ranks
45
_DATA_PARALLEL_GLOBAL_RANKS = None
46
47
-_ALL_REDUCE_LAUNCHER: Optional['GraphAllReduce'] = None
47
48
def initialize_model_parallel(
49
tensor_model_parallel_size: int = 1,
@@ -196,20 +195,6 @@ def initialize_model_parallel(
195
if rank in ranks:
196
_POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
197
199
-def initialize_all_reduce_launcher(
200
- max_num_tokens: int,
201
- hidden_size: int,
202
- dtype: torch.dtype,
203
- disable_graph: bool = False,
204
-) -> None:
205
- global _ALL_REDUCE_LAUNCHER
206
- _ALL_REDUCE_LAUNCHER = GraphAllReduce(
207
- max_num_tokens=max_num_tokens,
208
- hidden_size=hidden_size,
209
- dtype=dtype,
210
- disable_graph=disable_graph,
211
- )
212
-
198
def model_parallel_is_initialized():
199
"""Check if model and data parallel groups are initialized."""
200
if _TENSOR_MODEL_PARALLEL_GROUP is None or \
@@ -458,6 +443,7 @@ def get_pipeline_model_parallel_last_rank():
443
last_rank_local = get_pipeline_model_parallel_world_size() - 1
444
return _PIPELINE_GLOBAL_RANKS[last_rank_local]
445
446
+
447
def get_pipeline_model_parallel_next_rank():
448
"""Return the global rank that follows the caller in the pipeline"""
449
assert _PIPELINE_GLOBAL_RANKS is not None, \
@@ -485,10 +471,6 @@ def get_data_parallel_rank():
471
"""Return my rank for the data parallel group."""
472
return torch.distributed.get_rank(group=get_data_parallel_group())
473
488
-def get_all_reduce_launcher() -> 'GraphAllReduce':
489
- assert _ALL_REDUCE_LAUNCHER is not None, 'all reduce launcher is not initialized'
490
- return _ALL_REDUCE_LAUNCHER
491
-
474
def destroy_model_parallel():
475
"""Set the groups to none."""
476
global _MODEL_PARALLEL_GROUP
@@ -515,56 +497,3 @@ def destroy_model_parallel():
497
_MPU_TENSOR_MODEL_PARALLEL_RANK = None
498
global _MPU_PIPELINE_MODEL_PARALLEL_RANK
499
_MPU_PIPELINE_MODEL_PARALLEL_RANK = None
518
-
519
-
520
-class GraphAllReduce:
521
-
522
- def __init__(
523
- self,
524
- max_num_tokens: int,
525
- hidden_size: int,
526
- dtype: torch.dtype,
527
- disable_graph: bool = False,
528
- ) -> None:
529
- self.max_num_tokens = max_num_tokens
530
- self.hidden_size = hidden_size
531
- self.disable_graph = disable_graph
532
-
533
- tp_world_size = get_tensor_model_parallel_world_size()
534
- if tp_world_size == 1:
535
- return
536
-
537
- self.group = get_tensor_model_parallel_group()
538
- self.buffer = torch.empty(
539
- size=(max_num_tokens, hidden_size),
540
- dtype=dtype,
541
- device='cuda',
542
- )
543
-
544
- # Build graphs for different number of tokens.
545
- if not self.disable_graph:
546
- self.graphs = {}
547
- for num_tokens in range(8, max_num_tokens + 1, 8):
548
- self.graphs[num_tokens] = self._build_graph(num_tokens)
549
-
550
- def _build_graph(self, num_tokens: int) -> torch.cuda.CUDAGraph:
551
- # Warm up.
552
- torch.distributed.all_reduce(self.buffer[:num_tokens], group=self.group)
553
- torch.cuda.synchronize()
554
-
555
- # Build graph.
556
- graph = torch.cuda.CUDAGraph()
557
- with torch.cuda.graph(graph):
558
- torch.distributed.all_reduce(
559
- self.buffer[:num_tokens], group=self.group)
560
- torch.cuda.synchronize()
561
- return graph
562
-
563
- def launch(self, x: torch.Tensor) -> torch.Tensor:
564
- # NOTE: x must be a slice of self.buffer.
565
- num_tokens = x.shape[0]
566
- if self.disable_graph:
567
- torch.distributed.all_reduce(x, group=self.group)
568
- else:
569
- self.graphs[num_tokens].replay()
570
- return x
vllm/model_executor/parallel_utils/tensor_parallel/__init__.py
+2
-1
@@ -12,6 +12,7 @@ from .mappings import (
12
copy_to_tensor_model_parallel_region,
13
gather_from_tensor_model_parallel_region,
14
gather_from_sequence_parallel_region,
15
+ reduce_from_tensor_model_parallel_region,
16
scatter_to_tensor_model_parallel_region,
17
scatter_to_sequence_parallel_region,
18
)
@@ -38,7 +39,7 @@ __all__ = [
39
"copy_to_tensor_model_parallel_region",
40
"gather_from_tensor_model_parallel_region",
41
"gather_from_sequence_parallel_region",
41
-# "reduce_from_tensor_model_parallel_region",
42
+ "reduce_from_tensor_model_parallel_region",
43
"scatter_to_tensor_model_parallel_region",
44
"scatter_to_sequence_parallel_region",
45
# random.py
vllm/model_executor/parallel_utils/tensor_parallel/layers.py
+16
-15
@@ -14,7 +14,6 @@ from torch.nn.parameter import Parameter
14
from vllm.model_executor.parallel_utils.parallel_state import (
15
get_tensor_model_parallel_rank,
16
get_tensor_model_parallel_world_size,
17
- get_all_reduce_launcher,
17
)
18
from .mappings import (
19
copy_to_tensor_model_parallel_region,
@@ -248,8 +247,8 @@ class ColumnParallelLinear(torch.nn.Module):
247
self.output_size = output_size
248
self.gather_output = gather_output
249
# Divide the weight matrix along the last dimension.
251
- world_size = get_tensor_model_parallel_world_size()
252
- self.output_size_per_partition = divide(output_size, world_size)
250
+ self.world_size = get_tensor_model_parallel_world_size()
251
+ self.output_size_per_partition = divide(output_size, self.world_size)
252
self.skip_bias_add = skip_bias_add
253
254
if params_dtype is None:
@@ -350,6 +349,7 @@ class RowParallelLinear(torch.nn.Module):
349
params_dtype:
350
use_cpu_initialization:
351
perform_initialization:
352
+ reduce_results:
353
"""
354
355
def __init__(self, input_size, output_size, *,
@@ -360,6 +360,7 @@ class RowParallelLinear(torch.nn.Module):
360
params_dtype=None,
361
use_cpu_initialization=False,
362
perform_initialization=True,
363
+ reduce_results=True,
364
):
365
super(RowParallelLinear, self).__init__()
366
@@ -367,14 +368,19 @@ class RowParallelLinear(torch.nn.Module):
368
self.input_size = input_size
369
self.output_size = output_size
370
self.input_is_parallel = input_is_parallel
371
+ self.reduce_results = reduce_results
372
if params_dtype is None:
373
params_dtype = torch.get_default_dtype()
374
375
# Divide the weight matrix along the last dimension.
374
- world_size = get_tensor_model_parallel_world_size()
375
- self.input_size_per_partition = divide(input_size, world_size)
376
+ self.world_size = get_tensor_model_parallel_world_size()
377
+ self.input_size_per_partition = divide(input_size, self.world_size)
378
self.skip_bias_add = skip_bias_add
379
380
+ if not reduce_results and (bias and not skip_bias_add):
381
+ raise ValueError("When not reduce the results, adding bias to the "
382
+ "results can lead to incorrect results")
383
+
384
# Parameters.
385
# Note: torch.nn.functional.linear performs XA^T + b and as a result
386
# we allocate the transpose.
@@ -427,17 +433,12 @@ class RowParallelLinear(torch.nn.Module):
433
input_parallel = input_
434
else:
435
input_parallel = scatter_to_tensor_model_parallel_region(input_)
430
- if get_tensor_model_parallel_world_size() == 1:
431
- # Matrix multiply.
432
- output_ = F.linear(input_parallel, self.weight)
436
+ # Matrix multiply.
437
+ output_parallel = F.linear(input_parallel, self.weight)
438
+ if self.reduce_results and self.world_size > 1:
439
+ output_ = reduce_from_tensor_model_parallel_region(output_parallel)
440
else:
434
- # Matrix multiply.
435
- all_reduce_launcher = get_all_reduce_launcher()
436
- num_tokens = input_parallel.shape[0]
437
- output_buffer = all_reduce_launcher.buffer[:num_tokens]
438
- torch.matmul(input_parallel, self.weight_t, out=output_buffer)
439
- # All-reduce across all the partitions.
440
- output_ = all_reduce_launcher.launch(output_buffer)
441
+ output_ = output_parallel
442
443
if not self.skip_bias_add:
444
output = output_ + self.bias if self.bias is not None else output_
vllm/transformers_utils/config.py
+2
@@ -5,6 +5,8 @@ from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
5
_CONFIG_REGISTRY = {
6
"mpt": MPTConfig,
7
"baichuan": BaiChuanConfig,
8
+ "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
9
+ "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
10
}
11
12
vllm/transformers_utils/configs/__init__.py
+5
@@ -1,7 +1,12 @@
1
from vllm.transformers_utils.configs.mpt import MPTConfig
2
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3
+# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
4
+# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5
+# `FalconConfig` class from the official HuggingFace transformers library.
6
+from vllm.transformers_utils.configs.falcon import RWConfig
7
8
__all__ = [
9
"MPTConfig",
10
"BaiChuanConfig",
11
+ "RWConfig",
12
]
vllm/transformers_utils/configs/falcon.py
new
+87
@@ -0,0 +1,87 @@
1
+# Adapted from
2
+# https://huggingface.co/tiiuae/falcon-7b/blob/main/configuration_RW.py
3
+# Copyright 2023 The vLLM team.
4
+# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team.
5
+# All rights reserved.
6
+#
7
+# Licensed under the Apache License, Version 2.0 (the "License");
8
+# you may not use this file except in compliance with the License.
9
+# You may obtain a copy of the License at
10
+#
11
+# http://www.apache.org/licenses/LICENSE-2.0
12
+#
13
+# Unless required by applicable law or agreed to in writing, software
14
+# distributed under the License is distributed on an "AS IS" BASIS,
15
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+# See the License for the specific language governing permissions and
17
+# limitations under the License.
18
+"""Falcon configuration"""
19
+from transformers.configuration_utils import PretrainedConfig
20
+
21
+
22
+class RWConfig(PretrainedConfig):
23
+ model_type = "falcon"
24
+ keys_to_ignore_at_inference = ["past_key_values"]
25
+ attribute_map = {
26
+ "num_hidden_layers": "n_layer",
27
+ "num_attention_heads": "n_head",
28
+ "num_kv_heads": "n_head_kv",
29
+ }
30
+
31
+ def __init__(
32
+ self,
33
+ vocab_size=250880,
34
+ hidden_size=64,
35
+ n_layer=2,
36
+ n_head=8,
37
+ layer_norm_epsilon=1e-5,
38
+ initializer_range=0.02,
39
+ use_cache=True,
40
+ bos_token_id=1,
41
+ eos_token_id=2,
42
+ hidden_dropout=0.0,
43
+ attention_dropout=0.0,
44
+ multi_query=True,
45
+ n_head_kv=None,
46
+ alibi=False,
47
+ bias=False,
48
+ parallel_attn=False,
49
+ new_decoder_architecture=False,
50
+ **kwargs,
51
+ ) -> None:
52
+ self.vocab_size = vocab_size
53
+ # Backward compatibility with n_embed kwarg
54
+ n_embed = kwargs.pop("n_embed", None)
55
+ self.hidden_size = hidden_size if n_embed is None else n_embed
56
+ self.n_layer = n_layer
57
+ self.n_head = n_head
58
+ self.layer_norm_epsilon = layer_norm_epsilon
59
+ self.initializer_range = initializer_range
60
+ self.use_cache = use_cache
61
+ self.hidden_dropout = hidden_dropout
62
+ self.attention_dropout = attention_dropout
63
+
64
+ self.bos_token_id = bos_token_id
65
+ self.eos_token_id = eos_token_id
66
+ self.multi_query = multi_query
67
+ self.n_head_kv = 1 if n_head_kv is None else n_head_kv
68
+ self.alibi = alibi
69
+ self.bias = bias
70
+ self.parallel_attn = parallel_attn
71
+ self.new_decoder_architecture = new_decoder_architecture
72
+
73
+ if self.hidden_size == 8192:
74
+ # Hack for falcon-40b
75
+ self.new_decoder_architecture = True
76
+
77
+ super().__init__(bos_token_id=bos_token_id,
78
+ eos_token_id=eos_token_id,
79
+ **kwargs)
80
+
81
+ @property
82
+ def head_dim(self):
83
+ return self.hidden_size // self.n_head
84
+
85
+ @property
86
+ def rotary(self):
87
+ return not self.alibi
vllm/worker/worker.py
+1
-6
@@ -9,7 +9,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig)
10
from vllm.model_executor import get_model, InputMetadata, set_random_seed
11
from vllm.model_executor.parallel_utils.parallel_state import (
12
- initialize_model_parallel, initialize_all_reduce_launcher)
12
+ initialize_model_parallel)
13
from vllm.sampling_params import SamplingParams
14
from vllm.sequence import SequenceData, SequenceGroupMetadata, SequenceOutputs
15
from vllm.worker.cache_engine import CacheEngine
@@ -65,11 +65,6 @@ class Worker:
65
# Initialize the model.
66
set_random_seed(self.model_config.seed)
67
self.model = get_model(self.model_config)
68
- initialize_all_reduce_launcher(
69
- self.scheduler_config.max_num_batched_tokens,
70
- self.model_config.get_hidden_size(),
71
- self.model_config.dtype,
72
- )
68
69
@torch.inference_mode()
70
def profile_num_available_blocks(