A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add support for BLOOM (#331)
Woosuk Kwon committed
Jul 3, 2023 at 13:12 UTC
e41f06702cb6d6787ac4474832264108a6e28780
11 files changed
+479
-18
README.md
+1
@@ -41,6 +41,7 @@ vLLM is flexible and easy to use with:
41
42
vLLM seamlessly supports many Huggingface models, including the following architectures:
43
44
+- BLOOM (`bigscience/bloom`, `bigscience/bloomz`, etc.)
45
- GPT-2 (`gpt2`, `gpt2-xl`, etc.)
46
- GPT BigCode (`bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, etc.)
47
- GPT-NeoX (`EleutherAI/gpt-neox-20b`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc.)
csrc/attention.cpp
+3
-1
@@ -1,4 +1,5 @@
1
#include <torch/extension.h>
2
+#include <c10/util/Optional.h>
3
4
void single_query_cached_kv_attention(
5
torch::Tensor& out,
@@ -9,7 +10,8 @@ void single_query_cached_kv_attention(
10
torch::Tensor& block_tables,
11
torch::Tensor& context_lens,
12
int block_size,
12
- int max_context_len);
13
+ int max_context_len,
14
+ const c10::optional<torch::Tensor>& alibi_slopes);
15
16
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
17
m.def(
csrc/attention/attention_kernels.cu
+19
-6
@@ -80,6 +80,7 @@ __global__ void single_query_cached_kv_attention_kernel(
80
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
81
const int* __restrict__ context_lens, // [num_seqs]
82
const int max_num_blocks_per_seq,
83
+ const float* __restrict__ alibi_slopes, // [num_heads]
84
const int q_stride) {
85
constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
86
constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
@@ -91,6 +92,7 @@ __global__ void single_query_cached_kv_attention_kernel(
92
const int head_idx = blockIdx.x;
93
const int num_heads = gridDim.x;
94
const int seq_idx = blockIdx.y;
95
+ const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
96
97
// A vector type to store a part of a key or a query.
98
// The vector size is configured in such a way that the threads in a thread group
@@ -167,12 +169,14 @@ __global__ void single_query_cached_kv_attention_kernel(
169
170
// Compute dot product.
171
// This includes a reduction across the threads in the same thread group.
170
- const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
171
- const bool mask = token_idx >= context_len;
172
-
172
+ float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
173
+ // Add the ALiBi bias if slopes are given.
174
+ qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len) : 0;
175
+
176
if (thread_group_offset == 0) {
177
// Store the partial reductions to shared memory.
178
// NOTE(woosuk): It is required to zero out the masked logits.
179
+ const bool mask = token_idx >= context_len;
180
logits[token_idx] = mask ? 0.f : qk;
181
// Update the max value.
182
qk_max = mask ? qk_max : fmaxf(qk_max, qk);
@@ -328,6 +332,7 @@ __global__ void single_query_cached_kv_attention_kernel(
332
block_tables_ptr, \
333
context_lens_ptr, \
334
max_num_blocks_per_seq, \
335
+ alibi_slopes_ptr, \
336
query_stride);
337
338
// TODO(woosuk): Tune NUM_THREADS.
@@ -343,7 +348,8 @@ void single_query_cached_kv_attention_launcher(
348
float scale,
349
torch::Tensor& block_tables,
350
torch::Tensor& context_lens,
346
- int max_context_len) {
351
+ int max_context_len,
352
+ const c10::optional<torch::Tensor>& alibi_slopes) {
353
int num_seqs = query.size(0);
354
int num_heads = query.size(1);
355
int head_size = query.size(2);
@@ -353,6 +359,11 @@ void single_query_cached_kv_attention_launcher(
359
int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
360
assert(head_size % thread_group_size == 0);
361
362
+ // NOTE: alibi_slopes is optional.
363
+ const float* alibi_slopes_ptr = alibi_slopes ?
364
+ reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
365
+ : nullptr;
366
+
367
T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
368
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
369
T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
@@ -411,7 +422,8 @@ void single_query_cached_kv_attention_launcher(
422
scale, \
423
block_tables, \
424
context_lens, \
414
- max_context_len);
425
+ max_context_len, \
426
+ alibi_slopes);
427
428
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
429
// 1, 2, 4, 64, 128, 256.
@@ -458,7 +470,8 @@ void single_query_cached_kv_attention(
470
torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
471
torch::Tensor& context_lens, // [num_seqs]
472
int block_size,
461
- int max_context_len) {
473
+ int max_context_len,
474
+ const c10::optional<torch::Tensor>& alibi_slopes) {
475
if (query.dtype() == at::ScalarType::Float) {
476
CALL_KERNEL_LAUNCHER_BLOCK_SIZE(float);
477
} else if (query.dtype() == at::ScalarType::Half) {
docs/source/models/supported_models.rst
+3
@@ -14,6 +14,9 @@ Alongside each architecture, we include some popular models that use it.
14
* - Architecture
15
- Models
16
- Example HuggingFace Models
17
+ * - :code:`BloomForCausalLM`
18
+ - BLOOM, BLOOMZ, BLOOMChat
19
+ - :code:`bigscience/bloom`, :code:`bigscience/bloomz`, etc.
20
* - :code:`GPT2LMHeadModel`
21
- GPT-2
22
- :code:`gpt2`, :code:`gpt2-xl`, etc.
tests/kernels/test_attention.py
+1
@@ -216,6 +216,7 @@ def run_single_query_cached_kv_attention(
216
context_lens,
217
block_size,
218
max_context_len,
219
+ None, # ALiBi slopes.
220
)
221
222
ref_output = torch.empty_like(query)
vllm/model_executor/input_metadata.py
+4
-2
@@ -1,7 +1,7 @@
1
from typing import Dict, List, Tuple
2
3
import torch
4
-from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
4
+from xformers.ops import AttentionBias
5
6
from vllm.sampling_params import SamplingParams
7
from vllm.sequence import SequenceData
@@ -38,7 +38,6 @@ class InputMetadata:
38
self.max_context_len = max_context_len
39
self.block_tables = block_tables
40
41
- self.attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
41
self.num_prompts = len(prompt_lens)
42
self.num_prompt_tokens = sum(prompt_lens)
43
self.num_generation_tokens = context_lens.shape[0]
@@ -50,6 +49,9 @@ class InputMetadata:
49
assert block_tables.shape[0] == self.num_generation_tokens
50
assert context_lens.shape[0] == self.num_generation_tokens
51
52
+ # Set during the execution of the first attention op.
53
+ self.attn_bias: List[AttentionBias] = []
54
+
55
def __repr__(self) -> str:
56
# Print only useful metadata.
57
return (f'InputMetadata('
vllm/model_executor/layers/attention.py
+128
-5
@@ -1,9 +1,11 @@
1
"""Multi-head attention."""
2
-from typing import Optional
2
+from typing import List, Optional
3
4
import torch
5
import torch.nn as nn
6
from xformers import ops as xops
7
+from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8
+ LowerTriangularMaskWithTensorBias)
9
10
from vllm import attention_ops
11
from vllm import cache_ops
@@ -53,13 +55,21 @@ class PagedAttention(nn.Module):
55
raise ValueError(f"head_size ({self.head_size}) is not supported. "
56
f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
57
58
+ def set_attn_bias(self, input_metadata: InputMetadata) -> None:
59
+ if input_metadata.attn_bias:
60
+ # Already set by a previous layer.
61
+ return
62
+ prompt_lens = input_metadata.prompt_lens
63
+ attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
64
+ input_metadata.attn_bias.append(attn_bias)
65
+
66
def multi_query_kv_attention(
67
self,
68
output: torch.Tensor,
69
query: torch.Tensor,
70
key: torch.Tensor,
71
value: torch.Tensor,
62
- attn_bias: xops.AttentionBias,
72
+ input_metadata: InputMetadata,
73
) -> torch.Tensor:
74
"""Normal attention for the prompt tokens.
75
@@ -68,13 +78,14 @@ class PagedAttention(nn.Module):
78
query: shape = [num_prompt_tokens, num_heads, head_size]
79
key: shape = [num_prompt_tokens, num_heads, head_size]
80
value: shape = [num_prompt_tokens, num_heads, head_size]
81
+ input_metadata: metadata for paged attention.
82
"""
83
# TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
84
out = xops.memory_efficient_attention_forward(
85
query.unsqueeze(0),
86
key.unsqueeze(0),
87
value.unsqueeze(0),
77
- attn_bias=attn_bias,
88
+ attn_bias=input_metadata.attn_bias[0],
89
p=0.0,
90
scale=self.scale,
91
op=self.attn_op,
@@ -112,6 +123,7 @@ class PagedAttention(nn.Module):
123
input_metadata.context_lens,
124
block_size,
125
input_metadata.max_context_len,
126
+ None, # alibi_slopes
127
)
128
129
def forward(
@@ -154,12 +166,13 @@ class PagedAttention(nn.Module):
166
# Compute the attention op for prompts.
167
num_prompt_tokens = input_metadata.num_prompt_tokens
168
if num_prompt_tokens > 0:
169
+ self.set_attn_bias(input_metadata)
170
self.multi_query_kv_attention(
171
output[:num_prompt_tokens],
172
query[:num_prompt_tokens],
173
key[:num_prompt_tokens],
174
value[:num_prompt_tokens],
162
- input_metadata.attn_bias,
175
+ input_metadata,
176
)
177
178
# Wait until the cache op is done.
@@ -219,7 +232,8 @@ class PagedAttentionWithRoPE(PagedAttention):
232
cache = torch.cat((cos, sin), dim=-1)
233
234
# FIXME(woosuk): This assumes that we configure the default dtype when
222
- # initializing the model. Make it more robust.
235
+ # initializing the model.
236
+ # TODO(woosuk): Make it more robust.
237
torch_dtype = torch.get_default_dtype()
238
cache = cache.to(torch_dtype)
239
# Embedding size: [max_position, rotary_dim]
@@ -271,3 +285,112 @@ class PagedAttentionWithRoPE(PagedAttention):
285
input_metadata,
286
cache_event,
287
)
288
+
289
+
290
+class PagedAttentionWithALiBi(PagedAttention):
291
+ """PagedAttention with ALiBi attention bias."""
292
+
293
+ def __init__(
294
+ self,
295
+ num_heads: int,
296
+ head_size: int,
297
+ scale: float,
298
+ slopes: List[float],
299
+ ) -> None:
300
+ super().__init__(num_heads, head_size, scale)
301
+ assert len(slopes) == num_heads
302
+
303
+ slopes = torch.tensor(slopes, dtype=torch.float32)
304
+ self.register_buffer("alibi_slopes", slopes, persistent=False)
305
+
306
+ def set_attn_bias(self, input_metadata: InputMetadata) -> None:
307
+ if input_metadata.attn_bias:
308
+ # Already set by a previous layer.
309
+ return
310
+ # Generates ALiBi mask for each prompt.
311
+ for prompt_len in input_metadata.prompt_lens:
312
+ bias = torch.arange(prompt_len)
313
+ bias = bias[None, :] - bias[:, None]
314
+ bias = bias.to(self.alibi_slopes.device)
315
+
316
+ # When using custom attention bias, xformers requires the bias to
317
+ # be sliced from a tensor whose length is a multiple of 8.
318
+ padded_len = (prompt_len + 7) // 8 * 8
319
+ bias = torch.empty(
320
+ self.num_heads,
321
+ padded_len,
322
+ padded_len,
323
+ device=self.alibi_slopes.device,
324
+ )[:, :prompt_len, :prompt_len].copy_(bias)
325
+ bias.mul_(self.alibi_slopes[:, None, None])
326
+ attn_bias = LowerTriangularMaskWithTensorBias(bias)
327
+ input_metadata.attn_bias.append(attn_bias)
328
+
329
+ def multi_query_kv_attention(
330
+ self,
331
+ output: torch.Tensor,
332
+ query: torch.Tensor,
333
+ key: torch.Tensor,
334
+ value: torch.Tensor,
335
+ input_metadata: InputMetadata,
336
+ ) -> torch.Tensor:
337
+ """Attention with ALiBi bias for the prompt tokens.
338
+
339
+ Args:
340
+ output: shape = [num_prompt_tokens, num_heads, head_size]
341
+ query: shape = [num_prompt_tokens, num_heads, head_size]
342
+ key: shape = [num_prompt_tokens, num_heads, head_size]
343
+ value: shape = [num_prompt_tokens, num_heads, head_size]
344
+ input_metadata: metadata for paged attention.
345
+ """
346
+ # FIXME(woosuk): Because xformers does not support dynamic sequence
347
+ # lengths with custom attention bias, we process each prompt one by
348
+ # one. This is inefficient, especially when we have many short prompts.
349
+ start = 0
350
+ for i, prompt_len in enumerate(input_metadata.prompt_lens):
351
+ end = start + prompt_len
352
+ out = xops.memory_efficient_attention_forward(
353
+ query[None, start:end],
354
+ key[None, start:end],
355
+ value[None, start:end],
356
+ attn_bias=input_metadata.attn_bias[i],
357
+ p=0.0,
358
+ scale=self.scale,
359
+ op=self.attn_op,
360
+ )
361
+ # TODO(woosuk): Unnecessary copy. Optimize.
362
+ output[start:end].copy_(out.squeeze(0))
363
+ start += prompt_len
364
+ return output
365
+
366
+ def single_query_cached_kv_attention(
367
+ self,
368
+ output: torch.Tensor,
369
+ query: torch.Tensor,
370
+ key_cache: torch.Tensor,
371
+ value_cache: torch.Tensor,
372
+ input_metadata: InputMetadata,
373
+ ) -> None:
374
+ """PagedAttention with ALiBi bias for the generation tokens.
375
+
376
+ Args:
377
+ output: shape = [num_generation_tokens, num_heads, head_size]
378
+ query: shape = [num_generation_tokens, num_heads, head_size]
379
+ key_cache: shape = [num_blocks, num_heads, head_size/x,
380
+ block_size, x]
381
+ value_cache: shape = [num_blocks, num_heads, head_size, block_size]
382
+ input_metadata: metadata for paged attention.
383
+ """
384
+ block_size = value_cache.shape[3]
385
+ attention_ops.single_query_cached_kv_attention(
386
+ output,
387
+ query,
388
+ key_cache,
389
+ value_cache,
390
+ self.scale,
391
+ input_metadata.block_tables,
392
+ input_metadata.context_lens,
393
+ block_size,
394
+ input_metadata.max_context_len,
395
+ self.alibi_slopes,
396
+ )
vllm/model_executor/model_loader.py
+2
-3
@@ -6,13 +6,12 @@ import torch.nn as nn
6
from transformers import PretrainedConfig
7
8
from vllm.config import ModelConfig
9
-from vllm.model_executor.models import (GPT2LMHeadModel, GPTBigCodeForCausalLM,
10
- GPTNeoXForCausalLM, LlamaForCausalLM,
11
- OPTForCausalLM)
9
+from vllm.model_executor.models import * # pylint: disable=wildcard-import
10
from vllm.model_executor.weight_utils import initialize_dummy_weights
11
12
# TODO(woosuk): Lazy-load the model classes.
13
_MODEL_REGISTRY = {
14
+ "BloomForCausalLM": BloomForCausalLM,
15
"GPT2LMHeadModel": GPT2LMHeadModel,
16
"GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
17
"GPTNeoXForCausalLM": GPTNeoXForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -1,3 +1,4 @@
1
+from vllm.model_executor.models.bloom import BloomForCausalLM
2
from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
3
from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
4
from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
@@ -5,6 +6,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
6
from vllm.model_executor.models.opt import OPTForCausalLM
7
8
__all__ = [
9
+ "BloomForCausalLM",
10
"GPT2LMHeadModel",
11
"GPTBigCodeForCausalLM",
12
"GPTNeoXForCausalLM",
vllm/model_executor/models/bloom.py
new
+316
@@ -0,0 +1,316 @@
1
+# coding=utf-8
2
+# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
3
+# Copyright 2023 The CacheFlow team.
4
+# Copyright 2022 HuggingFace Inc. team and BigScience workshop.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Inference-only BLOOM model compatible with HuggingFace weights.
18
+
19
+The input of the model is flattened to a 1D tensor of tokens. The model uses
20
+InputMetadata to extract the original 2D shape of the input.
21
+"""
22
+import math
23
+from typing import Dict, List, Optional, Tuple
24
+
25
+import torch
26
+from torch import nn
27
+from transformers import BloomConfig
28
+
29
+from vllm.model_executor.input_metadata import InputMetadata
30
+from vllm.model_executor.layers.activation import get_act_fn
31
+from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
32
+from vllm.model_executor.layers.sampler import Sampler
33
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34
+ load_tensor_parallel_weights)
35
+from vllm.model_executor.parallel_utils.parallel_state import (
36
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
+from vllm.model_executor.parallel_utils.tensor_parallel import (
38
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39
+from vllm.sequence import SequenceOutputs
40
+
41
+KVCache = Tuple[torch.Tensor, torch.Tensor]
42
+
43
+
44
+def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
45
+ closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
46
+ base = torch.tensor(
47
+ 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
48
+ dtype=torch.float32,
49
+ )
50
+ powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
51
+ slopes = torch.pow(base, powers)
52
+
53
+ if closest_power_of_2 != total_num_heads:
54
+ extra_base = torch.tensor(
55
+ 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
56
+ dtype=torch.float32,
57
+ )
58
+ num_remaining_heads = min(closest_power_of_2,
59
+ total_num_heads - closest_power_of_2)
60
+ extra_powers = torch.arange(start=1,
61
+ end=1 + 2 * num_remaining_heads,
62
+ step=2,
63
+ dtype=torch.int32)
64
+ slopes = torch.cat(
65
+ [slopes, torch.pow(extra_base, extra_powers)], dim=0)
66
+ return slopes
67
+
68
+
69
+class BloomAttention(nn.Module):
70
+
71
+ def __init__(self, config: BloomConfig):
72
+ super().__init__()
73
+ self.hidden_size = config.hidden_size
74
+ self.total_num_heads = config.n_head
75
+ self.head_dim = self.hidden_size // self.total_num_heads
76
+ assert self.head_dim * self.total_num_heads == self.hidden_size
77
+
78
+ tp_world_size = get_tensor_model_parallel_world_size()
79
+ assert self.total_num_heads % tp_world_size == 0
80
+ self.num_heads = self.total_num_heads // tp_world_size
81
+
82
+ self.query_key_value = ColumnParallelLinear(
83
+ self.hidden_size,
84
+ 3 * self.hidden_size,
85
+ bias=True,
86
+ gather_output=False,
87
+ perform_initialization=False,
88
+ )
89
+ self.dense = RowParallelLinear(
90
+ self.hidden_size,
91
+ self.hidden_size,
92
+ bias=True,
93
+ input_is_parallel=True,
94
+ perform_initialization=False,
95
+ )
96
+
97
+ # Create the alibi slopes and slice them.
98
+ tp_rank = get_tensor_model_parallel_rank()
99
+ head_start = tp_rank * self.num_heads
100
+ head_end = (tp_rank + 1) * self.num_heads
101
+ alibi_slopes = _get_alibi_slopes(self.total_num_heads)
102
+ alibi_slopes = alibi_slopes[head_start:head_end].tolist()
103
+
104
+ scaling = self.head_dim**-0.5
105
+ self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
106
+ scaling, alibi_slopes)
107
+
108
+ def forward(
109
+ self,
110
+ position_ids: torch.Tensor,
111
+ hidden_states: torch.Tensor,
112
+ kv_cache: KVCache,
113
+ input_metadata: InputMetadata,
114
+ cache_event: Optional[torch.cuda.Event],
115
+ ) -> torch.Tensor:
116
+ del position_ids # Unused.
117
+ qkv, _ = self.query_key_value(hidden_states)
118
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
119
+ k_cache, v_cache = kv_cache
120
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
121
+ cache_event)
122
+ output, _ = self.dense(attn_output)
123
+ return output
124
+
125
+
126
+class BloomMLP(nn.Module):
127
+
128
+ def __init__(self, config: BloomConfig):
129
+ super().__init__()
130
+ hidden_size = config.hidden_size
131
+ self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
132
+ 4 * hidden_size,
133
+ gather_output=False,
134
+ perform_initialization=False)
135
+ self.act = get_act_fn("gelu")
136
+ self.dense_4h_to_h = RowParallelLinear(4 * hidden_size,
137
+ hidden_size,
138
+ input_is_parallel=True,
139
+ perform_initialization=False)
140
+
141
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
142
+ x, _ = self.dense_h_to_4h(x)
143
+ x = self.act(x)
144
+ x, _ = self.dense_4h_to_h(x)
145
+ return x
146
+
147
+
148
+class BloomBlock(nn.Module):
149
+
150
+ def __init__(self, config: BloomConfig):
151
+ super().__init__()
152
+ hidden_size = config.hidden_size
153
+
154
+ self.input_layernorm = nn.LayerNorm(hidden_size,
155
+ eps=config.layer_norm_epsilon)
156
+ self.self_attention = BloomAttention(config)
157
+ self.post_attention_layernorm = nn.LayerNorm(
158
+ hidden_size, eps=config.layer_norm_epsilon)
159
+ self.mlp = BloomMLP(config)
160
+ self.apply_residual_connection_post_layernorm = (
161
+ config.apply_residual_connection_post_layernorm)
162
+
163
+ def forward(
164
+ self,
165
+ position_ids: torch.Tensor,
166
+ hidden_states: torch.Tensor,
167
+ kv_cache: KVCache,
168
+ input_metadata: InputMetadata,
169
+ cache_event: Optional[torch.cuda.Event],
170
+ ) -> torch.Tensor:
171
+ # Layer norm at the beginning of the transformer layer.
172
+ layernorm_output = self.input_layernorm(hidden_states)
173
+
174
+ # Layer norm post the self attention.
175
+ if self.apply_residual_connection_post_layernorm:
176
+ residual = layernorm_output
177
+ else:
178
+ residual = hidden_states
179
+
180
+ # Self attention.
181
+ attention_output = self.self_attention(
182
+ position_ids=position_ids,
183
+ hidden_states=layernorm_output,
184
+ kv_cache=kv_cache,
185
+ input_metadata=input_metadata,
186
+ cache_event=cache_event,
187
+ )
188
+ attention_output = attention_output + residual
189
+ layernorm_output = self.post_attention_layernorm(attention_output)
190
+
191
+ # Get residual
192
+ if self.apply_residual_connection_post_layernorm:
193
+ residual = layernorm_output
194
+ else:
195
+ residual = attention_output
196
+
197
+ # MLP.
198
+ output = self.mlp(layernorm_output) + residual
199
+ return output
200
+
201
+
202
+class BloomModel(nn.Module):
203
+
204
+ def __init__(self, config: BloomConfig):
205
+ super().__init__()
206
+ self.embed_dim = config.hidden_size
207
+
208
+ # Embedding + LN Embedding
209
+ self.word_embeddings = VocabParallelEmbedding(
210
+ config.vocab_size, self.embed_dim, perform_initialization=False)
211
+ self.word_embeddings_layernorm = nn.LayerNorm(
212
+ self.embed_dim, eps=config.layer_norm_epsilon)
213
+
214
+ # Transformer blocks
215
+ self.h = nn.ModuleList(
216
+ [BloomBlock(config) for _ in range(config.num_hidden_layers)])
217
+
218
+ # Final Layer Norm
219
+ self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
220
+
221
+ def forward(
222
+ self,
223
+ input_ids: torch.Tensor,
224
+ position_ids: torch.Tensor,
225
+ kv_caches: List[KVCache],
226
+ input_metadata: InputMetadata,
227
+ cache_events: Optional[List[torch.cuda.Event]],
228
+ ) -> torch.Tensor:
229
+ hidden_states = self.word_embeddings(input_ids)
230
+ hidden_states = self.word_embeddings_layernorm(hidden_states)
231
+ for i in range(len(self.h)):
232
+ if cache_events is None:
233
+ cache_event = None
234
+ else:
235
+ cache_event = cache_events[i]
236
+ layer = self.h[i]
237
+ hidden_states = layer(
238
+ position_ids,
239
+ hidden_states,
240
+ kv_caches[i],
241
+ input_metadata,
242
+ cache_event,
243
+ )
244
+ hidden_states = self.ln_f(hidden_states)
245
+ return hidden_states
246
+
247
+
248
+class BloomForCausalLM(nn.Module):
249
+
250
+ def __init__(self, config: BloomConfig):
251
+ super().__init__()
252
+ self.config = config
253
+ self.transformer = BloomModel(config)
254
+ # TODO(zhuohan): create a new weight after implementing pipeline
255
+ # parallelism
256
+ self.lm_head_weight = self.transformer.word_embeddings.weight
257
+ self.sampler = Sampler(config.vocab_size)
258
+
259
+ def forward(
260
+ self,
261
+ input_ids: torch.Tensor,
262
+ positions: torch.Tensor,
263
+ kv_caches: List[KVCache],
264
+ input_metadata: InputMetadata,
265
+ cache_events: Optional[List[torch.cuda.Event]],
266
+ ) -> Dict[int, SequenceOutputs]:
267
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
268
+ input_metadata, cache_events)
269
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
270
+ input_metadata)
271
+ return next_tokens
272
+
273
+ _column_parallel_weights = [
274
+ "word_embeddings.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"
275
+ ]
276
+ _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
277
+
278
+ def load_weights(self,
279
+ model_name_or_path: str,
280
+ cache_dir: Optional[str] = None,
281
+ use_np_cache: bool = False):
282
+ tp_rank = get_tensor_model_parallel_rank()
283
+ state_dict = self.state_dict()
284
+ for name, loaded_weight in hf_model_weights_iterator(
285
+ model_name_or_path, cache_dir, use_np_cache):
286
+ if not name.startswith("transformer."):
287
+ name = "transformer." + name
288
+
289
+ param = state_dict[name]
290
+ if "query_key_value" in name:
291
+ # NOTE(woosuk): BLOOM's fused QKV has the shape of
292
+ # [num_heads * 3 * head_size, hidden_size], while the
293
+ # required shape is [3 * num_heads * head_size, hidden_size].
294
+ # Thus, we need weight conversion.
295
+ shard_size = param.shape[0]
296
+ start = shard_size * tp_rank
297
+ end = shard_size * (tp_rank + 1)
298
+ loaded_weight = loaded_weight[start:end]
299
+
300
+ num_heads = self.config.num_attention_heads
301
+ hidden_size = self.config.hidden_size
302
+ head_size = hidden_size // num_heads
303
+ if "query_key_value.weight" in name:
304
+ loaded_weight = loaded_weight.view(-1, 3, head_size,
305
+ hidden_size)
306
+ loaded_weight = loaded_weight.transpose(0, 1)
307
+ loaded_weight = loaded_weight.reshape(-1, hidden_size)
308
+ elif "query_key_value.bias" in name:
309
+ loaded_weight = loaded_weight.view(-1, 3, head_size)
310
+ loaded_weight = loaded_weight.transpose(0, 1)
311
+ loaded_weight = loaded_weight.reshape(-1)
312
+ else:
313
+ raise ValueError(f"Unexpected weight name: {name}")
314
+ load_tensor_parallel_weights(param, loaded_weight, name,
315
+ self._column_parallel_weights,
316
+ self._row_parallel_weights, tp_rank)
vllm/model_executor/models/gpt_neox.py
-1
@@ -80,7 +80,6 @@ class GPTNeoXAttention(nn.Module):
80
cache_event: Optional[torch.cuda.Event],
81
) -> torch.Tensor:
82
qkv, _ = self.query_key_value(hidden_states)
83
-
83
q, k, v = qkv.chunk(chunks=3, dim=-1)
84
k_cache, v_cache = kv_cache
85
attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,