@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e41f06702c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add support for BLOOM (#331)

Woosuk Kwon committed Jul 3, 2023 at 13:12 UTC e41f06702cb6d6787ac4474832264108a6e28780
11 files changed +479 -18
README.md
+1
@@ -41,6 +41,7 @@ vLLM is flexible and easy to use with:
41
42 vLLM seamlessly supports many Huggingface models, including the following architectures:
43
44 +- BLOOM (`bigscience/bloom`, `bigscience/bloomz`, etc.)
45 - GPT-2 (`gpt2`, `gpt2-xl`, etc.)
46 - GPT BigCode (`bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, etc.)
47 - GPT-NeoX (`EleutherAI/gpt-neox-20b`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc.)
csrc/attention.cpp
+3 -1
@@ -1,4 +1,5 @@
1 #include <torch/extension.h>
2 +#include <c10/util/Optional.h>
3
4 void single_query_cached_kv_attention(
5 torch::Tensor& out,
@@ -9,7 +10,8 @@ void single_query_cached_kv_attention(
10 torch::Tensor& block_tables,
11 torch::Tensor& context_lens,
12 int block_size,
12 - int max_context_len);
13 + int max_context_len,
14 + const c10::optional<torch::Tensor>& alibi_slopes);
15
16 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
17 m.def(
csrc/attention/attention_kernels.cu
+19 -6
@@ -80,6 +80,7 @@ __global__ void single_query_cached_kv_attention_kernel(
80 const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
81 const int* __restrict__ context_lens, // [num_seqs]
82 const int max_num_blocks_per_seq,
83 + const float* __restrict__ alibi_slopes, // [num_heads]
84 const int q_stride) {
85 constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
86 constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
@@ -91,6 +92,7 @@ __global__ void single_query_cached_kv_attention_kernel(
92 const int head_idx = blockIdx.x;
93 const int num_heads = gridDim.x;
94 const int seq_idx = blockIdx.y;
95 + const float alibi_slope = alibi_slopes == nullptr ? 0.f : alibi_slopes[head_idx];
96
97 // A vector type to store a part of a key or a query.
98 // The vector size is configured in such a way that the threads in a thread group
@@ -167,12 +169,14 @@ __global__ void single_query_cached_kv_attention_kernel(
169
170 // Compute dot product.
171 // This includes a reduction across the threads in the same thread group.
170 - const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
171 - const bool mask = token_idx >= context_len;
172 -
172 + float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
173 + // Add the ALiBi bias if slopes are given.
174 + qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len) : 0;
175 +
176 if (thread_group_offset == 0) {
177 // Store the partial reductions to shared memory.
178 // NOTE(woosuk): It is required to zero out the masked logits.
179 + const bool mask = token_idx >= context_len;
180 logits[token_idx] = mask ? 0.f : qk;
181 // Update the max value.
182 qk_max = mask ? qk_max : fmaxf(qk_max, qk);
@@ -328,6 +332,7 @@ __global__ void single_query_cached_kv_attention_kernel(
332 block_tables_ptr, \
333 context_lens_ptr, \
334 max_num_blocks_per_seq, \
335 + alibi_slopes_ptr, \
336 query_stride);
337
338 // TODO(woosuk): Tune NUM_THREADS.
@@ -343,7 +348,8 @@ void single_query_cached_kv_attention_launcher(
348 float scale,
349 torch::Tensor& block_tables,
350 torch::Tensor& context_lens,
346 - int max_context_len) {
351 + int max_context_len,
352 + const c10::optional<torch::Tensor>& alibi_slopes) {
353 int num_seqs = query.size(0);
354 int num_heads = query.size(1);
355 int head_size = query.size(2);
@@ -353,6 +359,11 @@ void single_query_cached_kv_attention_launcher(
359 int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
360 assert(head_size % thread_group_size == 0);
361
362 + // NOTE: alibi_slopes is optional.
363 + const float* alibi_slopes_ptr = alibi_slopes ?
364 + reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
365 + : nullptr;
366 +
367 T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
368 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
369 T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
@@ -411,7 +422,8 @@ void single_query_cached_kv_attention_launcher(
422 scale, \
423 block_tables, \
424 context_lens, \
414 - max_context_len);
425 + max_context_len, \
426 + alibi_slopes);
427
428 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
429 // 1, 2, 4, 64, 128, 256.
@@ -458,7 +470,8 @@ void single_query_cached_kv_attention(
470 torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
471 torch::Tensor& context_lens, // [num_seqs]
472 int block_size,
461 - int max_context_len) {
473 + int max_context_len,
474 + const c10::optional<torch::Tensor>& alibi_slopes) {
475 if (query.dtype() == at::ScalarType::Float) {
476 CALL_KERNEL_LAUNCHER_BLOCK_SIZE(float);
477 } else if (query.dtype() == at::ScalarType::Half) {
docs/source/models/supported_models.rst
+3
@@ -14,6 +14,9 @@ Alongside each architecture, we include some popular models that use it.
14 * - Architecture
15 - Models
16 - Example HuggingFace Models
17 + * - :code:`BloomForCausalLM`
18 + - BLOOM, BLOOMZ, BLOOMChat
19 + - :code:`bigscience/bloom`, :code:`bigscience/bloomz`, etc.
20 * - :code:`GPT2LMHeadModel`
21 - GPT-2
22 - :code:`gpt2`, :code:`gpt2-xl`, etc.
tests/kernels/test_attention.py
+1
@@ -216,6 +216,7 @@ def run_single_query_cached_kv_attention(
216 context_lens,
217 block_size,
218 max_context_len,
219 + None, # ALiBi slopes.
220 )
221
222 ref_output = torch.empty_like(query)
vllm/model_executor/input_metadata.py
+4 -2
@@ -1,7 +1,7 @@
1 from typing import Dict, List, Tuple
2
3 import torch
4 -from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
4 +from xformers.ops import AttentionBias
5
6 from vllm.sampling_params import SamplingParams
7 from vllm.sequence import SequenceData
@@ -38,7 +38,6 @@ class InputMetadata:
38 self.max_context_len = max_context_len
39 self.block_tables = block_tables
40
41 - self.attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
41 self.num_prompts = len(prompt_lens)
42 self.num_prompt_tokens = sum(prompt_lens)
43 self.num_generation_tokens = context_lens.shape[0]
@@ -50,6 +49,9 @@ class InputMetadata:
49 assert block_tables.shape[0] == self.num_generation_tokens
50 assert context_lens.shape[0] == self.num_generation_tokens
51
52 + # Set during the execution of the first attention op.
53 + self.attn_bias: List[AttentionBias] = []
54 +
55 def __repr__(self) -> str:
56 # Print only useful metadata.
57 return (f'InputMetadata('
vllm/model_executor/layers/attention.py
+128 -5
@@ -1,9 +1,11 @@
1 """Multi-head attention."""
2 -from typing import Optional
2 +from typing import List, Optional
3
4 import torch
5 import torch.nn as nn
6 from xformers import ops as xops
7 +from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8 + LowerTriangularMaskWithTensorBias)
9
10 from vllm import attention_ops
11 from vllm import cache_ops
@@ -53,13 +55,21 @@ class PagedAttention(nn.Module):
55 raise ValueError(f"head_size ({self.head_size}) is not supported. "
56 f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
57
58 + def set_attn_bias(self, input_metadata: InputMetadata) -> None:
59 + if input_metadata.attn_bias:
60 + # Already set by a previous layer.
61 + return
62 + prompt_lens = input_metadata.prompt_lens
63 + attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
64 + input_metadata.attn_bias.append(attn_bias)
65 +
66 def multi_query_kv_attention(
67 self,
68 output: torch.Tensor,
69 query: torch.Tensor,
70 key: torch.Tensor,
71 value: torch.Tensor,
62 - attn_bias: xops.AttentionBias,
72 + input_metadata: InputMetadata,
73 ) -> torch.Tensor:
74 """Normal attention for the prompt tokens.
75
@@ -68,13 +78,14 @@ class PagedAttention(nn.Module):
78 query: shape = [num_prompt_tokens, num_heads, head_size]
79 key: shape = [num_prompt_tokens, num_heads, head_size]
80 value: shape = [num_prompt_tokens, num_heads, head_size]
81 + input_metadata: metadata for paged attention.
82 """
83 # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
84 out = xops.memory_efficient_attention_forward(
85 query.unsqueeze(0),
86 key.unsqueeze(0),
87 value.unsqueeze(0),
77 - attn_bias=attn_bias,
88 + attn_bias=input_metadata.attn_bias[0],
89 p=0.0,
90 scale=self.scale,
91 op=self.attn_op,
@@ -112,6 +123,7 @@ class PagedAttention(nn.Module):
123 input_metadata.context_lens,
124 block_size,
125 input_metadata.max_context_len,
126 + None, # alibi_slopes
127 )
128
129 def forward(
@@ -154,12 +166,13 @@ class PagedAttention(nn.Module):
166 # Compute the attention op for prompts.
167 num_prompt_tokens = input_metadata.num_prompt_tokens
168 if num_prompt_tokens > 0:
169 + self.set_attn_bias(input_metadata)
170 self.multi_query_kv_attention(
171 output[:num_prompt_tokens],
172 query[:num_prompt_tokens],
173 key[:num_prompt_tokens],
174 value[:num_prompt_tokens],
162 - input_metadata.attn_bias,
175 + input_metadata,
176 )
177
178 # Wait until the cache op is done.
@@ -219,7 +232,8 @@ class PagedAttentionWithRoPE(PagedAttention):
232 cache = torch.cat((cos, sin), dim=-1)
233
234 # FIXME(woosuk): This assumes that we configure the default dtype when
222 - # initializing the model. Make it more robust.
235 + # initializing the model.
236 + # TODO(woosuk): Make it more robust.
237 torch_dtype = torch.get_default_dtype()
238 cache = cache.to(torch_dtype)
239 # Embedding size: [max_position, rotary_dim]
@@ -271,3 +285,112 @@ class PagedAttentionWithRoPE(PagedAttention):
285 input_metadata,
286 cache_event,
287 )
288 +
289 +
290 +class PagedAttentionWithALiBi(PagedAttention):
291 + """PagedAttention with ALiBi attention bias."""
292 +
293 + def __init__(
294 + self,
295 + num_heads: int,
296 + head_size: int,
297 + scale: float,
298 + slopes: List[float],
299 + ) -> None:
300 + super().__init__(num_heads, head_size, scale)
301 + assert len(slopes) == num_heads
302 +
303 + slopes = torch.tensor(slopes, dtype=torch.float32)
304 + self.register_buffer("alibi_slopes", slopes, persistent=False)
305 +
306 + def set_attn_bias(self, input_metadata: InputMetadata) -> None:
307 + if input_metadata.attn_bias:
308 + # Already set by a previous layer.
309 + return
310 + # Generates ALiBi mask for each prompt.
311 + for prompt_len in input_metadata.prompt_lens:
312 + bias = torch.arange(prompt_len)
313 + bias = bias[None, :] - bias[:, None]
314 + bias = bias.to(self.alibi_slopes.device)
315 +
316 + # When using custom attention bias, xformers requires the bias to
317 + # be sliced from a tensor whose length is a multiple of 8.
318 + padded_len = (prompt_len + 7) // 8 * 8
319 + bias = torch.empty(
320 + self.num_heads,
321 + padded_len,
322 + padded_len,
323 + device=self.alibi_slopes.device,
324 + )[:, :prompt_len, :prompt_len].copy_(bias)
325 + bias.mul_(self.alibi_slopes[:, None, None])
326 + attn_bias = LowerTriangularMaskWithTensorBias(bias)
327 + input_metadata.attn_bias.append(attn_bias)
328 +
329 + def multi_query_kv_attention(
330 + self,
331 + output: torch.Tensor,
332 + query: torch.Tensor,
333 + key: torch.Tensor,
334 + value: torch.Tensor,
335 + input_metadata: InputMetadata,
336 + ) -> torch.Tensor:
337 + """Attention with ALiBi bias for the prompt tokens.
338 +
339 + Args:
340 + output: shape = [num_prompt_tokens, num_heads, head_size]
341 + query: shape = [num_prompt_tokens, num_heads, head_size]
342 + key: shape = [num_prompt_tokens, num_heads, head_size]
343 + value: shape = [num_prompt_tokens, num_heads, head_size]
344 + input_metadata: metadata for paged attention.
345 + """
346 + # FIXME(woosuk): Because xformers does not support dynamic sequence
347 + # lengths with custom attention bias, we process each prompt one by
348 + # one. This is inefficient, especially when we have many short prompts.
349 + start = 0
350 + for i, prompt_len in enumerate(input_metadata.prompt_lens):
351 + end = start + prompt_len
352 + out = xops.memory_efficient_attention_forward(
353 + query[None, start:end],
354 + key[None, start:end],
355 + value[None, start:end],
356 + attn_bias=input_metadata.attn_bias[i],
357 + p=0.0,
358 + scale=self.scale,
359 + op=self.attn_op,
360 + )
361 + # TODO(woosuk): Unnecessary copy. Optimize.
362 + output[start:end].copy_(out.squeeze(0))
363 + start += prompt_len
364 + return output
365 +
366 + def single_query_cached_kv_attention(
367 + self,
368 + output: torch.Tensor,
369 + query: torch.Tensor,
370 + key_cache: torch.Tensor,
371 + value_cache: torch.Tensor,
372 + input_metadata: InputMetadata,
373 + ) -> None:
374 + """PagedAttention with ALiBi bias for the generation tokens.
375 +
376 + Args:
377 + output: shape = [num_generation_tokens, num_heads, head_size]
378 + query: shape = [num_generation_tokens, num_heads, head_size]
379 + key_cache: shape = [num_blocks, num_heads, head_size/x,
380 + block_size, x]
381 + value_cache: shape = [num_blocks, num_heads, head_size, block_size]
382 + input_metadata: metadata for paged attention.
383 + """
384 + block_size = value_cache.shape[3]
385 + attention_ops.single_query_cached_kv_attention(
386 + output,
387 + query,
388 + key_cache,
389 + value_cache,
390 + self.scale,
391 + input_metadata.block_tables,
392 + input_metadata.context_lens,
393 + block_size,
394 + input_metadata.max_context_len,
395 + self.alibi_slopes,
396 + )
vllm/model_executor/model_loader.py
+2 -3
@@ -6,13 +6,12 @@ import torch.nn as nn
6 from transformers import PretrainedConfig
7
8 from vllm.config import ModelConfig
9 -from vllm.model_executor.models import (GPT2LMHeadModel, GPTBigCodeForCausalLM,
10 - GPTNeoXForCausalLM, LlamaForCausalLM,
11 - OPTForCausalLM)
9 +from vllm.model_executor.models import * # pylint: disable=wildcard-import
10 from vllm.model_executor.weight_utils import initialize_dummy_weights
11
12 # TODO(woosuk): Lazy-load the model classes.
13 _MODEL_REGISTRY = {
14 + "BloomForCausalLM": BloomForCausalLM,
15 "GPT2LMHeadModel": GPT2LMHeadModel,
16 "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
17 "GPTNeoXForCausalLM": GPTNeoXForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -1,3 +1,4 @@
1 +from vllm.model_executor.models.bloom import BloomForCausalLM
2 from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
3 from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
4 from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
@@ -5,6 +6,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
6 from vllm.model_executor.models.opt import OPTForCausalLM
7
8 __all__ = [
9 + "BloomForCausalLM",
10 "GPT2LMHeadModel",
11 "GPTBigCodeForCausalLM",
12 "GPTNeoXForCausalLM",
vllm/model_executor/models/bloom.py new
+316
@@ -0,0 +1,316 @@
1 +# coding=utf-8
2 +# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
3 +# Copyright 2023 The CacheFlow team.
4 +# Copyright 2022 HuggingFace Inc. team and BigScience workshop.
5 +#
6 +# Licensed under the Apache License, Version 2.0 (the "License");
7 +# you may not use this file except in compliance with the License.
8 +# You may obtain a copy of the License at
9 +#
10 +# http://www.apache.org/licenses/LICENSE-2.0
11 +#
12 +# Unless required by applicable law or agreed to in writing, software
13 +# distributed under the License is distributed on an "AS IS" BASIS,
14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 +# See the License for the specific language governing permissions and
16 +# limitations under the License.
17 +"""Inference-only BLOOM model compatible with HuggingFace weights.
18 +
19 +The input of the model is flattened to a 1D tensor of tokens. The model uses
20 +InputMetadata to extract the original 2D shape of the input.
21 +"""
22 +import math
23 +from typing import Dict, List, Optional, Tuple
24 +
25 +import torch
26 +from torch import nn
27 +from transformers import BloomConfig
28 +
29 +from vllm.model_executor.input_metadata import InputMetadata
30 +from vllm.model_executor.layers.activation import get_act_fn
31 +from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
32 +from vllm.model_executor.layers.sampler import Sampler
33 +from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34 + load_tensor_parallel_weights)
35 +from vllm.model_executor.parallel_utils.parallel_state import (
36 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 +from vllm.model_executor.parallel_utils.tensor_parallel import (
38 + VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39 +from vllm.sequence import SequenceOutputs
40 +
41 +KVCache = Tuple[torch.Tensor, torch.Tensor]
42 +
43 +
44 +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
45 + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
46 + base = torch.tensor(
47 + 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
48 + dtype=torch.float32,
49 + )
50 + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
51 + slopes = torch.pow(base, powers)
52 +
53 + if closest_power_of_2 != total_num_heads:
54 + extra_base = torch.tensor(
55 + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
56 + dtype=torch.float32,
57 + )
58 + num_remaining_heads = min(closest_power_of_2,
59 + total_num_heads - closest_power_of_2)
60 + extra_powers = torch.arange(start=1,
61 + end=1 + 2 * num_remaining_heads,
62 + step=2,
63 + dtype=torch.int32)
64 + slopes = torch.cat(
65 + [slopes, torch.pow(extra_base, extra_powers)], dim=0)
66 + return slopes
67 +
68 +
69 +class BloomAttention(nn.Module):
70 +
71 + def __init__(self, config: BloomConfig):
72 + super().__init__()
73 + self.hidden_size = config.hidden_size
74 + self.total_num_heads = config.n_head
75 + self.head_dim = self.hidden_size // self.total_num_heads
76 + assert self.head_dim * self.total_num_heads == self.hidden_size
77 +
78 + tp_world_size = get_tensor_model_parallel_world_size()
79 + assert self.total_num_heads % tp_world_size == 0
80 + self.num_heads = self.total_num_heads // tp_world_size
81 +
82 + self.query_key_value = ColumnParallelLinear(
83 + self.hidden_size,
84 + 3 * self.hidden_size,
85 + bias=True,
86 + gather_output=False,
87 + perform_initialization=False,
88 + )
89 + self.dense = RowParallelLinear(
90 + self.hidden_size,
91 + self.hidden_size,
92 + bias=True,
93 + input_is_parallel=True,
94 + perform_initialization=False,
95 + )
96 +
97 + # Create the alibi slopes and slice them.
98 + tp_rank = get_tensor_model_parallel_rank()
99 + head_start = tp_rank * self.num_heads
100 + head_end = (tp_rank + 1) * self.num_heads
101 + alibi_slopes = _get_alibi_slopes(self.total_num_heads)
102 + alibi_slopes = alibi_slopes[head_start:head_end].tolist()
103 +
104 + scaling = self.head_dim**-0.5
105 + self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
106 + scaling, alibi_slopes)
107 +
108 + def forward(
109 + self,
110 + position_ids: torch.Tensor,
111 + hidden_states: torch.Tensor,
112 + kv_cache: KVCache,
113 + input_metadata: InputMetadata,
114 + cache_event: Optional[torch.cuda.Event],
115 + ) -> torch.Tensor:
116 + del position_ids # Unused.
117 + qkv, _ = self.query_key_value(hidden_states)
118 + q, k, v = qkv.chunk(chunks=3, dim=-1)
119 + k_cache, v_cache = kv_cache
120 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
121 + cache_event)
122 + output, _ = self.dense(attn_output)
123 + return output
124 +
125 +
126 +class BloomMLP(nn.Module):
127 +
128 + def __init__(self, config: BloomConfig):
129 + super().__init__()
130 + hidden_size = config.hidden_size
131 + self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
132 + 4 * hidden_size,
133 + gather_output=False,
134 + perform_initialization=False)
135 + self.act = get_act_fn("gelu")
136 + self.dense_4h_to_h = RowParallelLinear(4 * hidden_size,
137 + hidden_size,
138 + input_is_parallel=True,
139 + perform_initialization=False)
140 +
141 + def forward(self, x: torch.Tensor) -> torch.Tensor:
142 + x, _ = self.dense_h_to_4h(x)
143 + x = self.act(x)
144 + x, _ = self.dense_4h_to_h(x)
145 + return x
146 +
147 +
148 +class BloomBlock(nn.Module):
149 +
150 + def __init__(self, config: BloomConfig):
151 + super().__init__()
152 + hidden_size = config.hidden_size
153 +
154 + self.input_layernorm = nn.LayerNorm(hidden_size,
155 + eps=config.layer_norm_epsilon)
156 + self.self_attention = BloomAttention(config)
157 + self.post_attention_layernorm = nn.LayerNorm(
158 + hidden_size, eps=config.layer_norm_epsilon)
159 + self.mlp = BloomMLP(config)
160 + self.apply_residual_connection_post_layernorm = (
161 + config.apply_residual_connection_post_layernorm)
162 +
163 + def forward(
164 + self,
165 + position_ids: torch.Tensor,
166 + hidden_states: torch.Tensor,
167 + kv_cache: KVCache,
168 + input_metadata: InputMetadata,
169 + cache_event: Optional[torch.cuda.Event],
170 + ) -> torch.Tensor:
171 + # Layer norm at the beginning of the transformer layer.
172 + layernorm_output = self.input_layernorm(hidden_states)
173 +
174 + # Layer norm post the self attention.
175 + if self.apply_residual_connection_post_layernorm:
176 + residual = layernorm_output
177 + else:
178 + residual = hidden_states
179 +
180 + # Self attention.
181 + attention_output = self.self_attention(
182 + position_ids=position_ids,
183 + hidden_states=layernorm_output,
184 + kv_cache=kv_cache,
185 + input_metadata=input_metadata,
186 + cache_event=cache_event,
187 + )
188 + attention_output = attention_output + residual
189 + layernorm_output = self.post_attention_layernorm(attention_output)
190 +
191 + # Get residual
192 + if self.apply_residual_connection_post_layernorm:
193 + residual = layernorm_output
194 + else:
195 + residual = attention_output
196 +
197 + # MLP.
198 + output = self.mlp(layernorm_output) + residual
199 + return output
200 +
201 +
202 +class BloomModel(nn.Module):
203 +
204 + def __init__(self, config: BloomConfig):
205 + super().__init__()
206 + self.embed_dim = config.hidden_size
207 +
208 + # Embedding + LN Embedding
209 + self.word_embeddings = VocabParallelEmbedding(
210 + config.vocab_size, self.embed_dim, perform_initialization=False)
211 + self.word_embeddings_layernorm = nn.LayerNorm(
212 + self.embed_dim, eps=config.layer_norm_epsilon)
213 +
214 + # Transformer blocks
215 + self.h = nn.ModuleList(
216 + [BloomBlock(config) for _ in range(config.num_hidden_layers)])
217 +
218 + # Final Layer Norm
219 + self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
220 +
221 + def forward(
222 + self,
223 + input_ids: torch.Tensor,
224 + position_ids: torch.Tensor,
225 + kv_caches: List[KVCache],
226 + input_metadata: InputMetadata,
227 + cache_events: Optional[List[torch.cuda.Event]],
228 + ) -> torch.Tensor:
229 + hidden_states = self.word_embeddings(input_ids)
230 + hidden_states = self.word_embeddings_layernorm(hidden_states)
231 + for i in range(len(self.h)):
232 + if cache_events is None:
233 + cache_event = None
234 + else:
235 + cache_event = cache_events[i]
236 + layer = self.h[i]
237 + hidden_states = layer(
238 + position_ids,
239 + hidden_states,
240 + kv_caches[i],
241 + input_metadata,
242 + cache_event,
243 + )
244 + hidden_states = self.ln_f(hidden_states)
245 + return hidden_states
246 +
247 +
248 +class BloomForCausalLM(nn.Module):
249 +
250 + def __init__(self, config: BloomConfig):
251 + super().__init__()
252 + self.config = config
253 + self.transformer = BloomModel(config)
254 + # TODO(zhuohan): create a new weight after implementing pipeline
255 + # parallelism
256 + self.lm_head_weight = self.transformer.word_embeddings.weight
257 + self.sampler = Sampler(config.vocab_size)
258 +
259 + def forward(
260 + self,
261 + input_ids: torch.Tensor,
262 + positions: torch.Tensor,
263 + kv_caches: List[KVCache],
264 + input_metadata: InputMetadata,
265 + cache_events: Optional[List[torch.cuda.Event]],
266 + ) -> Dict[int, SequenceOutputs]:
267 + hidden_states = self.transformer(input_ids, positions, kv_caches,
268 + input_metadata, cache_events)
269 + next_tokens = self.sampler(self.lm_head_weight, hidden_states,
270 + input_metadata)
271 + return next_tokens
272 +
273 + _column_parallel_weights = [
274 + "word_embeddings.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"
275 + ]
276 + _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
277 +
278 + def load_weights(self,
279 + model_name_or_path: str,
280 + cache_dir: Optional[str] = None,
281 + use_np_cache: bool = False):
282 + tp_rank = get_tensor_model_parallel_rank()
283 + state_dict = self.state_dict()
284 + for name, loaded_weight in hf_model_weights_iterator(
285 + model_name_or_path, cache_dir, use_np_cache):
286 + if not name.startswith("transformer."):
287 + name = "transformer." + name
288 +
289 + param = state_dict[name]
290 + if "query_key_value" in name:
291 + # NOTE(woosuk): BLOOM's fused QKV has the shape of
292 + # [num_heads * 3 * head_size, hidden_size], while the
293 + # required shape is [3 * num_heads * head_size, hidden_size].
294 + # Thus, we need weight conversion.
295 + shard_size = param.shape[0]
296 + start = shard_size * tp_rank
297 + end = shard_size * (tp_rank + 1)
298 + loaded_weight = loaded_weight[start:end]
299 +
300 + num_heads = self.config.num_attention_heads
301 + hidden_size = self.config.hidden_size
302 + head_size = hidden_size // num_heads
303 + if "query_key_value.weight" in name:
304 + loaded_weight = loaded_weight.view(-1, 3, head_size,
305 + hidden_size)
306 + loaded_weight = loaded_weight.transpose(0, 1)
307 + loaded_weight = loaded_weight.reshape(-1, hidden_size)
308 + elif "query_key_value.bias" in name:
309 + loaded_weight = loaded_weight.view(-1, 3, head_size)
310 + loaded_weight = loaded_weight.transpose(0, 1)
311 + loaded_weight = loaded_weight.reshape(-1)
312 + else:
313 + raise ValueError(f"Unexpected weight name: {name}")
314 + load_tensor_parallel_weights(param, loaded_weight, name,
315 + self._column_parallel_weights,
316 + self._row_parallel_weights, tp_rank)
vllm/model_executor/models/gpt_neox.py
-1
@@ -80,7 +80,6 @@ class GPTNeoXAttention(nn.Module):
80 cache_event: Optional[torch.cuda.Event],
81 ) -> torch.Tensor:
82 qkv, _ = self.query_key_value(hidden_states)
83 -
83 q, k, v = qkv.chunk(chunks=3, dim=-1)
84 k_cache, v_cache = kv_cache
85 attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,