@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1b0bd0fe8a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add Falcon support (new) (#592)

Zhuohan Li committed Aug 2, 2023 at 14:04 UTC 1b0bd0fe8a4a9de749b9d4618758ff20c8658d86
16 files changed +680 -122
README.md
+1
@@ -44,6 +44,7 @@ vLLM seamlessly supports many Huggingface models, including the following archit
44
45 - Baichuan-7B (`baichuan-inc/Baichuan-7B`)
46 - BLOOM (`bigscience/bloom`, `bigscience/bloomz`, etc.)
47 +- Falcon (`tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc.)
48 - GPT-2 (`gpt2`, `gpt2-xl`, etc.)
49 - GPT BigCode (`bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, etc.)
50 - GPT-J (`EleutherAI/gpt-j-6b`, `nomic-ai/gpt4all-j`, etc.)
csrc/pos_encoding_kernels.cu
+29 -13
@@ -10,7 +10,8 @@ __global__ void rotary_embedding_neox_kernel(
10 scalar_t* __restrict__ key, // [num_tokens, num_kv_heads, head_size]
11 const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
12 const int rot_dim,
13 - const int stride,
13 + const int query_stride,
14 + const int key_stride,
15 const int num_heads,
16 const int num_kv_heads,
17 const int head_size) {
@@ -23,14 +24,14 @@ __global__ void rotary_embedding_neox_kernel(
24 const int nq = num_heads * embed_dim;
25 for (int i = threadIdx.x; i < nq; i += blockDim.x) {
26 const int head_idx = i / embed_dim;
26 - const int token_head = token_idx * stride + head_idx * head_size;
27 + const int token_head = token_idx * query_stride + head_idx * head_size;
28
29 const int rot_offset = i % embed_dim;
30 const int x_index = rot_offset;
31 const int y_index = embed_dim + rot_offset;
32
32 - const int out_x = token_idx * stride + head_idx * head_size + x_index;
33 - const int out_y = token_idx * stride + head_idx * head_size + y_index;
33 + const int out_x = token_idx * query_stride + head_idx * head_size + x_index;
34 + const int out_y = token_idx * query_stride + head_idx * head_size + y_index;
35
36 const scalar_t cos = __ldg(cache_ptr + x_index);
37 const scalar_t sin = __ldg(cache_ptr + y_index);
@@ -39,13 +40,27 @@ __global__ void rotary_embedding_neox_kernel(
40 const scalar_t q_y = query[token_head + y_index];
41 query[out_x] = q_x * cos - q_y * sin;
42 query[out_y] = q_y * cos + q_x * sin;
43 + }
44 +
45 + const int nk = num_kv_heads * embed_dim;
46 + for (int i = threadIdx.x; i < nk; i += blockDim.x) {
47 + const int head_idx = i / embed_dim;
48 + const int token_head = token_idx * key_stride + head_idx * head_size;
49 +
50 + const int rot_offset = i % embed_dim;
51 + const int x_index = rot_offset;
52 + const int y_index = embed_dim + rot_offset;
53 +
54 + const int out_x = token_idx * key_stride + head_idx * head_size + x_index;
55 + const int out_y = token_idx * key_stride + head_idx * head_size + y_index;
56 +
57 + const scalar_t cos = __ldg(cache_ptr + x_index);
58 + const scalar_t sin = __ldg(cache_ptr + y_index);
59
43 - if (head_idx < num_kv_heads) {
44 - const scalar_t k_x = key[token_head + x_index];
45 - const scalar_t k_y = key[token_head + y_index];
46 - key[out_x] = k_x * cos - k_y * sin;
47 - key[out_y] = k_y * cos + k_x * sin;
48 - }
60 + const scalar_t k_x = key[token_head + x_index];
61 + const scalar_t k_y = key[token_head + y_index];
62 + key[out_x] = k_x * cos - k_y * sin;
63 + key[out_y] = k_y * cos + k_x * sin;
64 }
65 }
66
@@ -62,8 +77,8 @@ void rotary_embedding_neox(
77 int rot_dim = cos_sin_cache.size(1);
78 int num_heads = query.size(1) / head_size;
79 int num_kv_heads = key.size(1) / head_size;
65 - int stride = query.stride(0);
66 - TORCH_CHECK(stride == key.stride(0));
80 + int query_stride = query.stride(0);
81 + int key_stride = key.stride(0);
82
83 dim3 grid(num_tokens);
84 dim3 block(std::min(num_heads * rot_dim / 2, 512));
@@ -80,7 +95,8 @@ void rotary_embedding_neox(
95 key.data_ptr<scalar_t>(),
96 cos_sin_cache.data_ptr<scalar_t>(),
97 rot_dim,
83 - stride,
98 + query_stride,
99 + key_stride,
100 num_heads,
101 num_kv_heads,
102 head_size);
docs/source/models/supported_models.rst
+3
@@ -20,6 +20,9 @@ Alongside each architecture, we include some popular models that use it.
20 * - :code:`BloomForCausalLM`
21 - BLOOM, BLOOMZ, BLOOMChat
22 - :code:`bigscience/bloom`, :code:`bigscience/bloomz`, etc.
23 + * - :code:`FalconForCausalLM`
24 + - Falcon
25 + - :code:`tiiuae/falcon-7b``, :code:`tiiuae/falcon-40b`, :code:`tiiuae/falcon-rw-7b`, etc.
26 * - :code:`GPT2LMHeadModel`
27 - GPT-2
28 - :code:`gpt2`, :code:`gpt2-xl`, etc.
examples/llm_engine_example.py
+2 -1
@@ -10,7 +10,8 @@ def main(args: argparse.Namespace):
10
11 # Test the following prompts.
12 test_prompts = [
13 - ("A robot may not injure a human being", SamplingParams()),
13 + ("A robot may not injure a human being",
14 + SamplingParams(temperature=0.0)),
15 ("To be or not to be,",
16 SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17 ("What is the meaning of life?",
vllm/config.py
+7 -2
@@ -94,8 +94,13 @@ class ModelConfig:
94 return self.hf_config.hidden_size // self.hf_config.num_attention_heads
95
96 def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
97 - # For GPTBigCode:
98 - if getattr(self.hf_config, "multi_query", False):
97 + # For GPTBigCode & Falcon:
98 + # Note: for falcon, when new_decoder_architecture is True, the
99 + # multi_query flag is ignored and we use n_head_kv for the number of
100 + # KV heads.
101 + if (getattr(self.hf_config, "multi_query", False) and
102 + (self.hf_config.model_type == "falcon" and
103 + not getattr(self.hf_config, "new_decoder_architecture", False))):
104 # Multi-query attention, only one KV head.
105 return 1
106 # For Falcon:
vllm/model_executor/layers/attention.py
+24 -12
@@ -314,14 +314,13 @@ class PagedAttentionWithRoPE(PagedAttention):
314 class PagedAttentionWithALiBi(PagedAttention):
315 """PagedAttention with ALiBi attention bias."""
316
317 - def __init__(
318 - self,
319 - num_heads: int,
320 - head_size: int,
321 - scale: float,
322 - slopes: List[float],
323 - ) -> None:
324 - super().__init__(num_heads, head_size, scale)
317 + def __init__(self,
318 + num_heads: int,
319 + head_size: int,
320 + scale: float,
321 + slopes: List[float],
322 + num_kv_heads: Optional[int] = None) -> None:
323 + super().__init__(num_heads, head_size, scale, num_kv_heads)
324 assert len(slopes) == num_heads
325
326 slopes = torch.tensor(slopes, dtype=torch.float32)
@@ -334,6 +333,11 @@ class PagedAttentionWithALiBi(PagedAttention):
333 # Generates ALiBi mask for each prompt.
334 for prompt_len in input_metadata.prompt_lens:
335 bias = torch.arange(prompt_len)
336 + # Note(zhuohan): HF uses
337 + # `bias = bias[None, :].repeat(prompt_len, 1)`
338 + # here. We find that both biases give the same results, but
339 + # the bias below more accurately follows the original ALiBi
340 + # paper.
341 bias = bias[None, :] - bias[:, None]
342 bias = bias.to(self.alibi_slopes.device)
343
@@ -363,10 +367,17 @@ class PagedAttentionWithALiBi(PagedAttention):
367 Args:
368 output: shape = [num_prompt_tokens, num_heads, head_size]
369 query: shape = [num_prompt_tokens, num_heads, head_size]
366 - key: shape = [num_prompt_tokens, num_heads, head_size]
367 - value: shape = [num_prompt_tokens, num_heads, head_size]
370 + key: shape = [num_prompt_tokens, num_kv_heads, head_size]
371 + value: shape = [num_prompt_tokens, num_kv_heads, head_size]
372 input_metadata: metadata for paged attention.
373 """
374 + if self.num_kv_heads != self.num_heads:
375 + # Project the key and value tensors to the desired number of heads.
376 + key = torch.repeat_interleave(key, self.num_queries_per_kv, dim=1)
377 + value = torch.repeat_interleave(value,
378 + self.num_queries_per_kv,
379 + dim=1)
380 +
381 # FIXME(woosuk): Because xformers does not support dynamic sequence
382 # lengths with custom attention bias, we process each prompt one by
383 # one. This is inefficient, especially when we have many short prompts.
@@ -400,9 +411,10 @@ class PagedAttentionWithALiBi(PagedAttention):
411 Args:
412 output: shape = [num_generation_tokens, num_heads, head_size]
413 query: shape = [num_generation_tokens, num_heads, head_size]
403 - key_cache: shape = [num_blocks, num_heads, head_size/x,
414 + key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
415 block_size, x]
405 - value_cache: shape = [num_blocks, num_heads, head_size, block_size]
416 + value_cache: shape = [num_blocks, num_kv_heads, head_size,
417 + block_size]
418 input_metadata: metadata for paged attention.
419 """
420 block_size = value_cache.shape[3]
vllm/model_executor/model_loader.py
+2
@@ -14,6 +14,7 @@ _MODEL_REGISTRY = {
14 "BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
15 "BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
16 "BloomForCausalLM": BloomForCausalLM,
17 + "FalconForCausalLM": FalconForCausalLM,
18 "GPT2LMHeadModel": GPT2LMHeadModel,
19 "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
20 "GPTJForCausalLM": GPTJForCausalLM,
@@ -22,6 +23,7 @@ _MODEL_REGISTRY = {
23 "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
24 "MPTForCausalLM": MPTForCausalLM,
25 "OPTForCausalLM": OPTForCausalLM,
26 + "RWForCausalLM": FalconForCausalLM,
27 }
28
29
vllm/model_executor/models/__init__.py
+2
@@ -1,5 +1,6 @@
1 from vllm.model_executor.models.baichuan import BaiChuanForCausalLM, BaichuanForCausalLM
2 from vllm.model_executor.models.bloom import BloomForCausalLM
3 +from vllm.model_executor.models.falcon import FalconForCausalLM
4 from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
5 from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
6 from vllm.model_executor.models.gpt_j import GPTJForCausalLM
@@ -12,6 +13,7 @@ __all__ = [
13 "BaiChuanForCausalLM",
14 "BaichuanForCausalLM",
15 "BloomForCausalLM",
16 + "FalconForCausalLM",
17 "GPT2LMHeadModel",
18 "GPTBigCodeForCausalLM",
19 "GPTJForCausalLM",
vllm/model_executor/models/falcon.py new
+496
@@ -0,0 +1,496 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/a5cc30d72ae2dc19af534e4b35c986cc28db1275/src/transformers/models/falcon/modeling_falcon.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2023 the Falcon authors and HuggingFace Inc. team. All rights
6 +# reserved.
7 +#
8 +# Licensed under the Apache License, Version 2.0 (the "License");
9 +# you may not use this file except in compliance with the License.
10 +# You may obtain a copy of the License at
11 +#
12 +# http://www.apache.org/licenses/LICENSE-2.0
13 +#
14 +# Unless required by applicable law or agreed to in writing, software
15 +# distributed under the License is distributed on an "AS IS" BASIS,
16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 +# See the License for the specific language governing permissions and
18 +# limitations under the License.
19 +"""PyTorch Falcon model."""
20 +
21 +import math
22 +from typing import Dict, List, Optional, Tuple, Union
23 +
24 +import torch
25 +from torch import nn
26 +from torch.nn import LayerNorm
27 +from transformers import FalconConfig as HF_FalconConfig
28 +
29 +from vllm.model_executor.input_metadata import InputMetadata
30 +from vllm.model_executor.layers.attention import (PagedAttention,
31 + PagedAttentionWithALiBi,
32 + PagedAttentionWithRoPE)
33 +from vllm.model_executor.layers.sampler import Sampler
34 +from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35 + load_tensor_parallel_weights)
36 +from vllm.model_executor.parallel_utils.parallel_state import (
37 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 +from vllm.model_executor.parallel_utils.tensor_parallel import (
39 + VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear,
40 + reduce_from_tensor_model_parallel_region)
41 +from vllm.sequence import SequenceOutputs
42 +from vllm.transformers_utils.configs import RWConfig
43 +
44 +KVCache = Tuple[torch.Tensor, torch.Tensor]
45 +FalconConfig = Union[HF_FalconConfig, RWConfig]
46 +
47 +
48 +# NOTE(Hesslow): Unfortunately we did not fuse matmul and bias during
49 +# training, this means that there's one additional quantization to bfloat16
50 +# between the operations. In order not to degrade the quality of our HF-port,
51 +# we keep these characteristics in the final model.
52 +class FalconLinear(nn.Linear):
53 +
54 + def forward(self, x: torch.Tensor) -> torch.Tensor:
55 + hidden_states = x @ self.weight.T
56 + if self.bias is None:
57 + return hidden_states
58 + return hidden_states + self.bias
59 +
60 +
61 +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
62 + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
63 + base = torch.tensor(2**(-(2**-(math.log2(closest_power_of_2) - 3))),
64 + dtype=torch.float32)
65 + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
66 + slopes = torch.pow(base, powers)
67 +
68 + if closest_power_of_2 != total_num_heads:
69 + extra_base = torch.tensor(
70 + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
71 + dtype=torch.float32)
72 + num_remaining_heads = min(closest_power_of_2,
73 + total_num_heads - closest_power_of_2)
74 + extra_powers = torch.arange(1,
75 + 1 + 2 * num_remaining_heads,
76 + 2,
77 + dtype=torch.int32)
78 + slopes = torch.cat(
79 + [slopes, torch.pow(extra_base, extra_powers)], dim=0)
80 +
81 + return slopes
82 +
83 +
84 +class FalconAttention(nn.Module):
85 +
86 + def __init__(self, config: FalconConfig):
87 + super().__init__()
88 +
89 + self.hidden_size = config.hidden_size
90 + tp_size = get_tensor_model_parallel_world_size()
91 +
92 + self.total_num_heads = config.num_attention_heads
93 + assert self.total_num_heads % tp_size == 0
94 + self.num_heads = self.total_num_heads // tp_size
95 + self.head_dim = self.hidden_size // self.total_num_heads
96 + assert self.head_dim * self.total_num_heads == self.hidden_size
97 +
98 + self.new_decoder_architecture = config.new_decoder_architecture
99 + self.multi_query = config.multi_query
100 +
101 + if self.new_decoder_architecture:
102 + self.total_num_kv_heads = config.num_kv_heads
103 + assert self.total_num_heads % tp_size == 0
104 + self.num_kv_heads = self.total_num_kv_heads // tp_size
105 + self.query_key_value = ColumnParallelLinear(
106 + self.hidden_size,
107 + (self.total_num_heads + 2 * self.total_num_kv_heads) *
108 + self.head_dim,
109 + bias=config.bias,
110 + gather_output=False,
111 + perform_initialization=False,
112 + skip_bias_add=True,
113 + )
114 + elif self.multi_query:
115 + self.total_num_kv_heads = 1
116 + self.num_kv_heads = 1
117 + self.query = ColumnParallelLinear(
118 + self.hidden_size,
119 + self.total_num_heads * self.head_dim,
120 + bias=config.bias,
121 + gather_output=False,
122 + perform_initialization=False,
123 + skip_bias_add=True,
124 + )
125 + self.key_value = FalconLinear(self.hidden_size,
126 + 2 * self.head_dim,
127 + bias=config.bias)
128 + else:
129 + self.total_num_kv_heads = self.total_num_heads
130 + self.num_kv_heads = self.num_heads
131 + self.query_key_value = ColumnParallelLinear(
132 + self.hidden_size,
133 + (self.total_num_heads + 2 * self.total_num_kv_heads) *
134 + self.head_dim,
135 + bias=config.bias,
136 + gather_output=False,
137 + perform_initialization=False,
138 + skip_bias_add=True,
139 + )
140 +
141 + self.q_size = self.num_heads * self.head_dim
142 + self.kv_size = self.num_kv_heads * self.head_dim
143 +
144 + # Layer-wise attention scaling
145 + self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim)
146 + self.reduce_row_parallel_results = not (config.new_decoder_architecture
147 + or config.parallel_attn)
148 + self.dense = RowParallelLinear(
149 + self.hidden_size,
150 + self.hidden_size,
151 + bias=config.bias,
152 + input_is_parallel=True,
153 + perform_initialization=False,
154 + skip_bias_add=True,
155 + reduce_results=self.reduce_row_parallel_results)
156 +
157 + self.use_rotary = config.rotary
158 + self.use_alibi = config.alibi
159 + assert not (self.use_rotary and self.use_alibi), (
160 + "Rotary and alibi are mutually exclusive.")
161 +
162 + if self.use_rotary:
163 + # TODO(zhuohan): Pass in correct `max_position``
164 + self.attn = PagedAttentionWithRoPE(self.num_heads,
165 + self.head_dim,
166 + self.inv_norm_factor,
167 + rotary_dim=self.head_dim,
168 + num_kv_heads=self.num_kv_heads)
169 + elif self.use_alibi:
170 + tp_rank = get_tensor_model_parallel_rank()
171 + head_start = tp_rank * self.num_heads
172 + head_end = (tp_rank + 1) * self.num_heads
173 + alibi_slopes = (_get_alibi_slopes(self.total_num_heads) *
174 + self.inv_norm_factor)
175 + alibi_slopes = alibi_slopes[head_start:head_end].tolist()
176 + self.attn = PagedAttentionWithALiBi(self.num_heads,
177 + self.head_dim,
178 + self.inv_norm_factor,
179 + alibi_slopes,
180 + num_kv_heads=self.num_kv_heads)
181 + else:
182 + self.attn = PagedAttention(self.num_heads,
183 + self.head_dim,
184 + scale=self.inv_norm_factor,
185 + num_kv_heads=self.num_kv_heads)
186 +
187 + def forward(
188 + self,
189 + positions: torch.Tensor,
190 + hidden_states: torch.Tensor,
191 + kv_cache: KVCache,
192 + input_metadata: InputMetadata,
193 + cache_event: Optional[torch.cuda.Event],
194 + ) -> torch.Tensor:
195 + if not self.new_decoder_architecture and self.multi_query:
196 + q, bias = self.query(hidden_states)
197 + if bias is not None:
198 + q += bias
199 + kv = self.key_value(hidden_states)
200 + k, v = kv.split([self.kv_size, self.kv_size], dim=-1)
201 + else:
202 + qkv, bias = self.query_key_value(hidden_states)
203 + if bias is not None:
204 + qkv += bias
205 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size],
206 + dim=-1)
207 + k_cache, v_cache = kv_cache
208 + if self.use_rotary:
209 + attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
210 + input_metadata, cache_event)
211 + else:
212 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
213 + cache_event)
214 + attn_output, bias = self.dense(attn_output)
215 + return attn_output, bias
216 +
217 +
218 +class FalconMLP(nn.Module):
219 +
220 + def __init__(self, config: FalconConfig):
221 + super().__init__()
222 + hidden_size = config.hidden_size
223 +
224 + self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
225 + 4 * hidden_size,
226 + bias=config.bias,
227 + gather_output=False,
228 + perform_initialization=False,
229 + skip_bias_add=True)
230 + self.act = nn.GELU()
231 + self.reduce_row_parallel_results = not (config.new_decoder_architecture
232 + or config.parallel_attn)
233 + self.dense_4h_to_h = RowParallelLinear(
234 + 4 * hidden_size,
235 + hidden_size,
236 + bias=config.bias,
237 + input_is_parallel=True,
238 + perform_initialization=False,
239 + skip_bias_add=True,
240 + reduce_results=self.reduce_row_parallel_results)
241 +
242 + def forward(self, x: torch.Tensor) -> torch.Tensor:
243 + # NOTE(zhuohan): Following huggingface, we do not fuse bias add here.
244 + x, bias = self.dense_h_to_4h(x)
245 + if bias is not None:
246 + x += bias
247 + x = self.act(x)
248 + x, bias = self.dense_4h_to_h(x)
249 + return x, bias
250 +
251 +
252 +class FalconDecoderLayer(nn.Module):
253 +
254 + def __init__(self, config: FalconConfig):
255 + super().__init__()
256 + hidden_size = config.hidden_size
257 + self.num_heads = config.num_attention_heads
258 + self.self_attention = FalconAttention(config)
259 + self.mlp = FalconMLP(config)
260 + self.config = config
261 +
262 + if config.new_decoder_architecture:
263 + # The layer norm before self-attention
264 + self.ln_attn = LayerNorm(hidden_size,
265 + eps=config.layer_norm_epsilon)
266 + # The layer norm before the MLP
267 + self.ln_mlp = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
268 + else:
269 + self.input_layernorm = LayerNorm(hidden_size,
270 + eps=config.layer_norm_epsilon)
271 + if not config.parallel_attn:
272 + self.post_attention_layernorm = LayerNorm(
273 + hidden_size, eps=config.layer_norm_epsilon)
274 +
275 + self.reduce_row_parallel_results = not (config.new_decoder_architecture
276 + or config.parallel_attn)
277 +
278 + def forward(
279 + self,
280 + positions: torch.Tensor,
281 + hidden_states: torch.Tensor,
282 + kv_cache: KVCache,
283 + input_metadata: InputMetadata,
284 + cache_event: Optional[torch.cuda.Event],
285 + ):
286 + residual = hidden_states
287 +
288 + if self.config.new_decoder_architecture:
289 + attention_layernorm_out = self.ln_attn(hidden_states)
290 + mlp_layernorm_out = self.ln_mlp(hidden_states)
291 + else:
292 + attention_layernorm_out = self.input_layernorm(hidden_states)
293 +
294 + # Self attention.
295 + attention_output, attention_bias = self.self_attention(
296 + positions=positions,
297 + hidden_states=attention_layernorm_out,
298 + kv_cache=kv_cache,
299 + input_metadata=input_metadata,
300 + cache_event=cache_event,
301 + )
302 + if self.reduce_row_parallel_results and attention_bias is not None:
303 + attention_output += attention_bias
304 +
305 + if not self.config.new_decoder_architecture:
306 + if self.config.parallel_attn:
307 + mlp_layernorm_out = attention_layernorm_out
308 + else:
309 + residual += attention_output
310 + mlp_layernorm_out = self.post_attention_layernorm(residual)
311 +
312 + # MLP.
313 + mlp_output, mlp_bias = self.mlp(mlp_layernorm_out)
314 + if self.reduce_row_parallel_results and mlp_bias is not None:
315 + mlp_output += mlp_bias
316 +
317 + if not self.reduce_row_parallel_results:
318 + # When MLP and Attention layers are parallel, we can use
319 + # only one all-reduce operator to reduce the results from
320 + # both MLP and Attention layers.
321 + mlp_output += attention_output
322 + mlp_output = reduce_from_tensor_model_parallel_region(mlp_output)
323 + if attention_bias is not None:
324 + mlp_output += attention_bias
325 + if mlp_bias is not None:
326 + mlp_output += mlp_bias
327 +
328 + output = mlp_output + residual
329 +
330 + return output
331 +
332 +
333 +class FalconModel(nn.Module):
334 +
335 + def __init__(self, config: FalconConfig):
336 + super().__init__()
337 + self.config = config
338 + self.embed_dim = config.hidden_size
339 + self.num_heads = config.num_attention_heads
340 + self.use_alibi = config.alibi
341 +
342 + # Embedding + LN Embedding
343 + self.word_embeddings = VocabParallelEmbedding(
344 + config.vocab_size, self.embed_dim, perform_initialization=False)
345 +
346 + # Transformer blocks
347 + self.h = nn.ModuleList([
348 + FalconDecoderLayer(config) for _ in range(config.num_hidden_layers)
349 + ])
350 +
351 + # Final Layer Norm
352 + self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
353 +
354 + def forward(
355 + self,
356 + input_ids: torch.LongTensor,
357 + positions: torch.Tensor,
358 + kv_caches: List[KVCache],
359 + input_metadata: InputMetadata,
360 + cache_events: Optional[List[torch.cuda.Event]],
361 + ) -> torch.Tensor:
362 + hidden_states = self.word_embeddings(input_ids)
363 + for i in range(len(self.h)):
364 + if cache_events is None:
365 + cache_event = None
366 + else:
367 + cache_event = cache_events[i]
368 + layer = self.h[i]
369 + hidden_states = layer(
370 + positions,
371 + hidden_states,
372 + kv_caches[i],
373 + input_metadata,
374 + cache_event,
375 + )
376 + hidden_states = self.ln_f(hidden_states)
377 + return hidden_states
378 +
379 +
380 +class FalconForCausalLM(nn.Module):
381 +
382 + def __init__(self, config: FalconConfig):
383 + super().__init__()
384 + self.config = config
385 + self.transformer = FalconModel(config)
386 + self.lm_head = ColumnParallelLinear(config.hidden_size,
387 + config.vocab_size,
388 + bias=False,
389 + gather_output=False,
390 + perform_initialization=False)
391 + self.sampler = Sampler(config.vocab_size)
392 +
393 + def forward(
394 + self,
395 + input_ids: torch.LongTensor,
396 + positions: torch.Tensor,
397 + kv_caches: List[KVCache],
398 + input_metadata: InputMetadata,
399 + cache_events: Optional[List[torch.cuda.Event]],
400 + ) -> Dict[int, SequenceOutputs]:
401 + hidden_states = self.transformer(
402 + input_ids,
403 + positions,
404 + kv_caches,
405 + input_metadata,
406 + cache_events,
407 + )
408 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
409 + input_metadata)
410 +
411 + return next_tokens
412 +
413 + _column_parallel_weights = [
414 + "word_embeddings.weight", "lm_head.weight", "dense_h_to_4h.weight",
415 + "dense_h_to_4h.bias"
416 + ]
417 + _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
418 +
419 + def load_weights(self,
420 + model_name_or_path: str,
421 + cache_dir: Optional[str] = None,
422 + use_np_cache: bool = False):
423 + tp_size = (get_tensor_model_parallel_world_size())
424 + tp_rank = get_tensor_model_parallel_rank()
425 +
426 + hidden_size = self.config.hidden_size
427 + total_num_heads = self.config.num_attention_heads
428 + num_heads = total_num_heads // tp_size
429 + head_size = hidden_size // total_num_heads
430 + head_start = tp_rank * num_heads
431 + head_end = (tp_rank + 1) * num_heads
432 + if self.config.new_decoder_architecture:
433 + total_num_kv_heads = self.config.num_kv_heads
434 + num_kv_heads = total_num_kv_heads // tp_size
435 + separated_q_kv = False
436 + kv_head_start = tp_rank * num_kv_heads
437 + kv_head_end = (tp_rank + 1) * num_kv_heads
438 + elif self.config.multi_query:
439 + total_num_kv_heads = 1
440 + num_kv_heads = 1
441 + separated_q_kv = True
442 + kv_head_start = 0
443 + kv_head_end = 1
444 + else:
445 + total_num_kv_heads = total_num_heads
446 + num_kv_heads = total_num_kv_heads // tp_size
447 + separated_q_kv = False
448 + kv_head_start = tp_rank * num_kv_heads
449 + kv_head_end = (tp_rank + 1) * num_kv_heads
450 + num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
451 + state_dict = self.state_dict()
452 +
453 + for name, loaded_weight in hf_model_weights_iterator(
454 + model_name_or_path, cache_dir, use_np_cache):
455 + if "query_key_value" in name:
456 + loaded_weight_size = loaded_weight.size()
457 + loaded_weight = loaded_weight.view(
458 + total_num_kv_heads, num_query_heads_per_kv_head + 2,
459 + head_size, *loaded_weight_size[1:])
460 +
461 + wq = loaded_weight[:, :-2].reshape(-1, *loaded_weight_size[1:])
462 + wk = loaded_weight[:, [-2]].reshape(-1,
463 + *loaded_weight_size[1:])
464 + wv = loaded_weight[:, [-1]].reshape(-1,
465 + *loaded_weight_size[1:])
466 +
467 + wq = wq[head_size * head_start:head_size * head_end]
468 + wk = wk[head_size * kv_head_start:head_size * kv_head_end]
469 + wv = wv[head_size * kv_head_start:head_size * kv_head_end]
470 +
471 + if separated_q_kv:
472 + loaded_weight_q = wq
473 + loaded_weight_kv = torch.cat([wk, wv], dim=0)
474 + q_weight_name = name.replace("query_key_value", "query")
475 + kv_weight_name = name.replace("query_key_value",
476 + "key_value")
477 + load_tensor_parallel_weights(state_dict[q_weight_name],
478 + loaded_weight_q,
479 + q_weight_name,
480 + self._column_parallel_weights,
481 + self._row_parallel_weights,
482 + tp_rank)
483 + load_tensor_parallel_weights(state_dict[kv_weight_name],
484 + loaded_weight_kv,
485 + kv_weight_name,
486 + self._column_parallel_weights,
487 + self._row_parallel_weights,
488 + tp_rank)
489 + continue
490 + else:
491 + loaded_weight = torch.cat([wq, wk, wv], dim=0)
492 +
493 + param = state_dict[name]
494 + load_tensor_parallel_weights(param, loaded_weight, name,
495 + self._column_parallel_weights,
496 + self._row_parallel_weights, tp_rank)
vllm/model_executor/parallel_utils/parallel_state.py
+1 -72
@@ -44,7 +44,6 @@ _PIPELINE_GLOBAL_RANKS = None
44 # rank when broadcasting weights from src to all other data parallel ranks
45 _DATA_PARALLEL_GLOBAL_RANKS = None
46
47 -_ALL_REDUCE_LAUNCHER: Optional['GraphAllReduce'] = None
47
48 def initialize_model_parallel(
49 tensor_model_parallel_size: int = 1,
@@ -196,20 +195,6 @@ def initialize_model_parallel(
195 if rank in ranks:
196 _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
197
199 -def initialize_all_reduce_launcher(
200 - max_num_tokens: int,
201 - hidden_size: int,
202 - dtype: torch.dtype,
203 - disable_graph: bool = False,
204 -) -> None:
205 - global _ALL_REDUCE_LAUNCHER
206 - _ALL_REDUCE_LAUNCHER = GraphAllReduce(
207 - max_num_tokens=max_num_tokens,
208 - hidden_size=hidden_size,
209 - dtype=dtype,
210 - disable_graph=disable_graph,
211 - )
212 -
198 def model_parallel_is_initialized():
199 """Check if model and data parallel groups are initialized."""
200 if _TENSOR_MODEL_PARALLEL_GROUP is None or \
@@ -458,6 +443,7 @@ def get_pipeline_model_parallel_last_rank():
443 last_rank_local = get_pipeline_model_parallel_world_size() - 1
444 return _PIPELINE_GLOBAL_RANKS[last_rank_local]
445
446 +
447 def get_pipeline_model_parallel_next_rank():
448 """Return the global rank that follows the caller in the pipeline"""
449 assert _PIPELINE_GLOBAL_RANKS is not None, \
@@ -485,10 +471,6 @@ def get_data_parallel_rank():
471 """Return my rank for the data parallel group."""
472 return torch.distributed.get_rank(group=get_data_parallel_group())
473
488 -def get_all_reduce_launcher() -> 'GraphAllReduce':
489 - assert _ALL_REDUCE_LAUNCHER is not None, 'all reduce launcher is not initialized'
490 - return _ALL_REDUCE_LAUNCHER
491 -
474 def destroy_model_parallel():
475 """Set the groups to none."""
476 global _MODEL_PARALLEL_GROUP
@@ -515,56 +497,3 @@ def destroy_model_parallel():
497 _MPU_TENSOR_MODEL_PARALLEL_RANK = None
498 global _MPU_PIPELINE_MODEL_PARALLEL_RANK
499 _MPU_PIPELINE_MODEL_PARALLEL_RANK = None
518 -
519 -
520 -class GraphAllReduce:
521 -
522 - def __init__(
523 - self,
524 - max_num_tokens: int,
525 - hidden_size: int,
526 - dtype: torch.dtype,
527 - disable_graph: bool = False,
528 - ) -> None:
529 - self.max_num_tokens = max_num_tokens
530 - self.hidden_size = hidden_size
531 - self.disable_graph = disable_graph
532 -
533 - tp_world_size = get_tensor_model_parallel_world_size()
534 - if tp_world_size == 1:
535 - return
536 -
537 - self.group = get_tensor_model_parallel_group()
538 - self.buffer = torch.empty(
539 - size=(max_num_tokens, hidden_size),
540 - dtype=dtype,
541 - device='cuda',
542 - )
543 -
544 - # Build graphs for different number of tokens.
545 - if not self.disable_graph:
546 - self.graphs = {}
547 - for num_tokens in range(8, max_num_tokens + 1, 8):
548 - self.graphs[num_tokens] = self._build_graph(num_tokens)
549 -
550 - def _build_graph(self, num_tokens: int) -> torch.cuda.CUDAGraph:
551 - # Warm up.
552 - torch.distributed.all_reduce(self.buffer[:num_tokens], group=self.group)
553 - torch.cuda.synchronize()
554 -
555 - # Build graph.
556 - graph = torch.cuda.CUDAGraph()
557 - with torch.cuda.graph(graph):
558 - torch.distributed.all_reduce(
559 - self.buffer[:num_tokens], group=self.group)
560 - torch.cuda.synchronize()
561 - return graph
562 -
563 - def launch(self, x: torch.Tensor) -> torch.Tensor:
564 - # NOTE: x must be a slice of self.buffer.
565 - num_tokens = x.shape[0]
566 - if self.disable_graph:
567 - torch.distributed.all_reduce(x, group=self.group)
568 - else:
569 - self.graphs[num_tokens].replay()
570 - return x
vllm/model_executor/parallel_utils/tensor_parallel/__init__.py
+2 -1
@@ -12,6 +12,7 @@ from .mappings import (
12 copy_to_tensor_model_parallel_region,
13 gather_from_tensor_model_parallel_region,
14 gather_from_sequence_parallel_region,
15 + reduce_from_tensor_model_parallel_region,
16 scatter_to_tensor_model_parallel_region,
17 scatter_to_sequence_parallel_region,
18 )
@@ -38,7 +39,7 @@ __all__ = [
39 "copy_to_tensor_model_parallel_region",
40 "gather_from_tensor_model_parallel_region",
41 "gather_from_sequence_parallel_region",
41 -# "reduce_from_tensor_model_parallel_region",
42 + "reduce_from_tensor_model_parallel_region",
43 "scatter_to_tensor_model_parallel_region",
44 "scatter_to_sequence_parallel_region",
45 # random.py
vllm/model_executor/parallel_utils/tensor_parallel/layers.py
+16 -15
@@ -14,7 +14,6 @@ from torch.nn.parameter import Parameter
14 from vllm.model_executor.parallel_utils.parallel_state import (
15 get_tensor_model_parallel_rank,
16 get_tensor_model_parallel_world_size,
17 - get_all_reduce_launcher,
17 )
18 from .mappings import (
19 copy_to_tensor_model_parallel_region,
@@ -248,8 +247,8 @@ class ColumnParallelLinear(torch.nn.Module):
247 self.output_size = output_size
248 self.gather_output = gather_output
249 # Divide the weight matrix along the last dimension.
251 - world_size = get_tensor_model_parallel_world_size()
252 - self.output_size_per_partition = divide(output_size, world_size)
250 + self.world_size = get_tensor_model_parallel_world_size()
251 + self.output_size_per_partition = divide(output_size, self.world_size)
252 self.skip_bias_add = skip_bias_add
253
254 if params_dtype is None:
@@ -350,6 +349,7 @@ class RowParallelLinear(torch.nn.Module):
349 params_dtype:
350 use_cpu_initialization:
351 perform_initialization:
352 + reduce_results:
353 """
354
355 def __init__(self, input_size, output_size, *,
@@ -360,6 +360,7 @@ class RowParallelLinear(torch.nn.Module):
360 params_dtype=None,
361 use_cpu_initialization=False,
362 perform_initialization=True,
363 + reduce_results=True,
364 ):
365 super(RowParallelLinear, self).__init__()
366
@@ -367,14 +368,19 @@ class RowParallelLinear(torch.nn.Module):
368 self.input_size = input_size
369 self.output_size = output_size
370 self.input_is_parallel = input_is_parallel
371 + self.reduce_results = reduce_results
372 if params_dtype is None:
373 params_dtype = torch.get_default_dtype()
374
375 # Divide the weight matrix along the last dimension.
374 - world_size = get_tensor_model_parallel_world_size()
375 - self.input_size_per_partition = divide(input_size, world_size)
376 + self.world_size = get_tensor_model_parallel_world_size()
377 + self.input_size_per_partition = divide(input_size, self.world_size)
378 self.skip_bias_add = skip_bias_add
379
380 + if not reduce_results and (bias and not skip_bias_add):
381 + raise ValueError("When not reduce the results, adding bias to the "
382 + "results can lead to incorrect results")
383 +
384 # Parameters.
385 # Note: torch.nn.functional.linear performs XA^T + b and as a result
386 # we allocate the transpose.
@@ -427,17 +433,12 @@ class RowParallelLinear(torch.nn.Module):
433 input_parallel = input_
434 else:
435 input_parallel = scatter_to_tensor_model_parallel_region(input_)
430 - if get_tensor_model_parallel_world_size() == 1:
431 - # Matrix multiply.
432 - output_ = F.linear(input_parallel, self.weight)
436 + # Matrix multiply.
437 + output_parallel = F.linear(input_parallel, self.weight)
438 + if self.reduce_results and self.world_size > 1:
439 + output_ = reduce_from_tensor_model_parallel_region(output_parallel)
440 else:
434 - # Matrix multiply.
435 - all_reduce_launcher = get_all_reduce_launcher()
436 - num_tokens = input_parallel.shape[0]
437 - output_buffer = all_reduce_launcher.buffer[:num_tokens]
438 - torch.matmul(input_parallel, self.weight_t, out=output_buffer)
439 - # All-reduce across all the partitions.
440 - output_ = all_reduce_launcher.launch(output_buffer)
441 + output_ = output_parallel
442
443 if not self.skip_bias_add:
444 output = output_ + self.bias if self.bias is not None else output_
vllm/transformers_utils/config.py
+2
@@ -5,6 +5,8 @@ from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
5 _CONFIG_REGISTRY = {
6 "mpt": MPTConfig,
7 "baichuan": BaiChuanConfig,
8 + "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
9 + "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
10 }
11
12
vllm/transformers_utils/configs/__init__.py
+5
@@ -1,7 +1,12 @@
1 from vllm.transformers_utils.configs.mpt import MPTConfig
2 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3 +# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
4 +# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
5 +# `FalconConfig` class from the official HuggingFace transformers library.
6 +from vllm.transformers_utils.configs.falcon import RWConfig
7
8 __all__ = [
9 "MPTConfig",
10 "BaiChuanConfig",
11 + "RWConfig",
12 ]
vllm/transformers_utils/configs/falcon.py new
+87
@@ -0,0 +1,87 @@
1 +# Adapted from
2 +# https://huggingface.co/tiiuae/falcon-7b/blob/main/configuration_RW.py
3 +# Copyright 2023 The vLLM team.
4 +# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team.
5 +# All rights reserved.
6 +#
7 +# Licensed under the Apache License, Version 2.0 (the "License");
8 +# you may not use this file except in compliance with the License.
9 +# You may obtain a copy of the License at
10 +#
11 +# http://www.apache.org/licenses/LICENSE-2.0
12 +#
13 +# Unless required by applicable law or agreed to in writing, software
14 +# distributed under the License is distributed on an "AS IS" BASIS,
15 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 +# See the License for the specific language governing permissions and
17 +# limitations under the License.
18 +"""Falcon configuration"""
19 +from transformers.configuration_utils import PretrainedConfig
20 +
21 +
22 +class RWConfig(PretrainedConfig):
23 + model_type = "falcon"
24 + keys_to_ignore_at_inference = ["past_key_values"]
25 + attribute_map = {
26 + "num_hidden_layers": "n_layer",
27 + "num_attention_heads": "n_head",
28 + "num_kv_heads": "n_head_kv",
29 + }
30 +
31 + def __init__(
32 + self,
33 + vocab_size=250880,
34 + hidden_size=64,
35 + n_layer=2,
36 + n_head=8,
37 + layer_norm_epsilon=1e-5,
38 + initializer_range=0.02,
39 + use_cache=True,
40 + bos_token_id=1,
41 + eos_token_id=2,
42 + hidden_dropout=0.0,
43 + attention_dropout=0.0,
44 + multi_query=True,
45 + n_head_kv=None,
46 + alibi=False,
47 + bias=False,
48 + parallel_attn=False,
49 + new_decoder_architecture=False,
50 + **kwargs,
51 + ) -> None:
52 + self.vocab_size = vocab_size
53 + # Backward compatibility with n_embed kwarg
54 + n_embed = kwargs.pop("n_embed", None)
55 + self.hidden_size = hidden_size if n_embed is None else n_embed
56 + self.n_layer = n_layer
57 + self.n_head = n_head
58 + self.layer_norm_epsilon = layer_norm_epsilon
59 + self.initializer_range = initializer_range
60 + self.use_cache = use_cache
61 + self.hidden_dropout = hidden_dropout
62 + self.attention_dropout = attention_dropout
63 +
64 + self.bos_token_id = bos_token_id
65 + self.eos_token_id = eos_token_id
66 + self.multi_query = multi_query
67 + self.n_head_kv = 1 if n_head_kv is None else n_head_kv
68 + self.alibi = alibi
69 + self.bias = bias
70 + self.parallel_attn = parallel_attn
71 + self.new_decoder_architecture = new_decoder_architecture
72 +
73 + if self.hidden_size == 8192:
74 + # Hack for falcon-40b
75 + self.new_decoder_architecture = True
76 +
77 + super().__init__(bos_token_id=bos_token_id,
78 + eos_token_id=eos_token_id,
79 + **kwargs)
80 +
81 + @property
82 + def head_dim(self):
83 + return self.hidden_size // self.n_head
84 +
85 + @property
86 + def rotary(self):
87 + return not self.alibi
vllm/worker/worker.py
+1 -6
@@ -9,7 +9,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig)
10 from vllm.model_executor import get_model, InputMetadata, set_random_seed
11 from vllm.model_executor.parallel_utils.parallel_state import (
12 - initialize_model_parallel, initialize_all_reduce_launcher)
12 + initialize_model_parallel)
13 from vllm.sampling_params import SamplingParams
14 from vllm.sequence import SequenceData, SequenceGroupMetadata, SequenceOutputs
15 from vllm.worker.cache_engine import CacheEngine
@@ -65,11 +65,6 @@ class Worker:
65 # Initialize the model.
66 set_random_seed(self.model_config.seed)
67 self.model = get_model(self.model_config)
68 - initialize_all_reduce_launcher(
69 - self.scheduler_config.max_num_batched_tokens,
70 - self.model_config.get_hidden_size(),
71 - self.model_config.dtype,
72 - )
68
69 @torch.inference_mode()
70 def profile_num_available_blocks(