A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix baichuan for different position embedding for 7b and 13b models (#643)
Song committed
Aug 2, 2023 at 13:22 UTC
64f23c29000ca832f50d3470a83d7a9bdbe95a0c
3 files changed
+76
-17
vllm/model_executor/model_loader.py
+2
-1
@@ -11,7 +11,8 @@ from vllm.model_executor.weight_utils import initialize_dummy_weights
11
12
# TODO(woosuk): Lazy-load the model classes.
13
_MODEL_REGISTRY = {
14
- "BaiChuanForCausalLM": BaiChuanForCausalLM,
14
+ "BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
15
+ "BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
16
"BloomForCausalLM": BloomForCausalLM,
17
"GPT2LMHeadModel": GPT2LMHeadModel,
18
"GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
vllm/model_executor/models/__init__.py
+2
-1
@@ -1,4 +1,4 @@
1
-from vllm.model_executor.models.baichuan import BaiChuanForCausalLM
1
+from vllm.model_executor.models.baichuan import BaiChuanForCausalLM, BaichuanForCausalLM
2
from vllm.model_executor.models.bloom import BloomForCausalLM
3
from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
4
from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
@@ -10,6 +10,7 @@ from vllm.model_executor.models.opt import OPTForCausalLM
10
11
__all__ = [
12
"BaiChuanForCausalLM",
13
+ "BaichuanForCausalLM",
14
"BloomForCausalLM",
15
"GPT2LMHeadModel",
16
"GPTBigCodeForCausalLM",
vllm/model_executor/models/baichuan.py
+72
-15
@@ -22,6 +22,7 @@
22
The input of the model is flattened to a 1D tensor of tokens. The model uses
23
InputMetadata to extract the original 2D shape of the input.
24
"""
25
+import math
26
from typing import Dict, List, Optional, Tuple
27
28
import torch
@@ -31,7 +32,7 @@ from vllm.sequence import SequenceOutputs
32
from vllm.model_executor.input_metadata import InputMetadata
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.layernorm import RMSNorm
34
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
35
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE, PagedAttentionWithALiBi
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
38
load_tensor_parallel_weights)
@@ -44,6 +45,31 @@ from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
45
KVCache = Tuple[torch.Tensor, torch.Tensor]
46
47
48
+def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
49
+ closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
50
+ base = torch.tensor(
51
+ 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
52
+ dtype=torch.float32,
53
+ )
54
+ powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
55
+ slopes = torch.pow(base, powers)
56
+
57
+ if closest_power_of_2 != total_num_heads:
58
+ extra_base = torch.tensor(
59
+ 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
60
+ dtype=torch.float32,
61
+ )
62
+ num_remaining_heads = min(closest_power_of_2,
63
+ total_num_heads - closest_power_of_2)
64
+ extra_powers = torch.arange(start=1,
65
+ end=1 + 2 * num_remaining_heads,
66
+ step=2,
67
+ dtype=torch.int32)
68
+ slopes = torch.cat(
69
+ [slopes, torch.pow(extra_base, extra_powers)], dim=0)
70
+ return slopes
71
+
72
+
73
class BaiChuanMLP(nn.Module):
74
75
def __init__(
@@ -82,6 +108,7 @@ class BaiChuanAttention(nn.Module):
108
self,
109
hidden_size: int,
110
num_heads: int,
111
+ position_embedding: str,
112
):
113
super().__init__()
114
self.hidden_size = hidden_size
@@ -92,7 +119,7 @@ class BaiChuanAttention(nn.Module):
119
self.num_heads = (self.total_num_heads //
120
tensor_model_parallel_world_size)
121
self.head_dim = hidden_size // self.total_num_heads
95
- self.scaling = self.head_dim**-0.5
122
+ self.postion_embedding = position_embedding
123
124
# pylint: disable=invalid-name
125
self.W_pack = ColumnParallelLinear(
@@ -109,11 +136,23 @@ class BaiChuanAttention(nn.Module):
136
input_is_parallel=True,
137
perform_initialization=False,
138
)
112
-
113
- self.attn = PagedAttentionWithRoPE(self.num_heads,
114
- self.head_dim,
115
- self.scaling,
116
- rotary_dim=self.head_dim)
139
+ # Create the alibi slopes and slice them.
140
+ if self.postion_embedding == "ALIBI":
141
+ tp_rank = get_tensor_model_parallel_rank()
142
+ head_start = tp_rank * self.num_heads
143
+ head_end = (tp_rank + 1) * self.num_heads
144
+ alibi_slopes = _get_alibi_slopes(self.total_num_heads)
145
+ alibi_slopes = alibi_slopes[head_start:head_end].tolist()
146
+
147
+ scaling = self.head_dim**-0.5
148
+ self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
149
+ scaling, alibi_slopes)
150
+ else:
151
+ self.scaling = self.head_dim**-0.5
152
+ self.attn = PagedAttentionWithRoPE(self.num_heads,
153
+ self.head_dim,
154
+ self.scaling,
155
+ rotary_dim=self.head_dim)
156
157
def forward(
158
self,
@@ -126,20 +165,26 @@ class BaiChuanAttention(nn.Module):
165
qkv, _ = self.W_pack(hidden_states)
166
q, k, v = qkv.chunk(chunks=3, dim=-1)
167
k_cache, v_cache = kv_cache
129
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
130
- input_metadata, cache_event)
168
+ if self.postion_embedding == "ALIBI":
169
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
170
+ cache_event)
171
+ else:
172
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
173
+ input_metadata, cache_event)
174
+
175
output, _ = self.o_proj(attn_output)
176
return output
177
178
179
class BaiChuanDecoderLayer(nn.Module):
180
137
- def __init__(self, config: BaiChuanConfig):
181
+ def __init__(self, config: BaiChuanConfig, position_embedding: str):
182
super().__init__()
183
self.hidden_size = config.hidden_size
184
self.self_attn = BaiChuanAttention(
185
hidden_size=self.hidden_size,
186
num_heads=config.num_attention_heads,
187
+ position_embedding=position_embedding,
188
)
189
self.mlp = BaiChuanMLP(
190
hidden_size=self.hidden_size,
@@ -181,7 +226,7 @@ class BaiChuanDecoderLayer(nn.Module):
226
227
class BaiChuanModel(nn.Module):
228
184
- def __init__(self, config: BaiChuanConfig):
229
+ def __init__(self, config: BaiChuanConfig, position_embedding: str):
230
super().__init__()
231
self.config = config
232
self.padding_idx = config.pad_token_id
@@ -192,7 +237,7 @@ class BaiChuanModel(nn.Module):
237
config.hidden_size,
238
perform_initialization=False)
239
self.layers = nn.ModuleList([
195
- BaiChuanDecoderLayer(config)
240
+ BaiChuanDecoderLayer(config, position_embedding)
241
for _ in range(config.num_hidden_layers)
242
])
243
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -223,12 +268,12 @@ class BaiChuanModel(nn.Module):
268
return hidden_states
269
270
226
-class BaiChuanForCausalLM(nn.Module):
271
+class BaiChuanBaseForCausalLM(nn.Module):
272
228
- def __init__(self, config):
273
+ def __init__(self, config, position_embedding: str):
274
super().__init__()
275
self.config = config
231
- self.model = BaiChuanModel(config)
276
+ self.model = BaiChuanModel(config, position_embedding)
277
self.lm_head = ColumnParallelLinear(config.hidden_size,
278
config.vocab_size,
279
bias=False,
@@ -318,3 +363,15 @@ class BaiChuanForCausalLM(nn.Module):
363
self._row_parallel_weights,
364
tp_rank,
365
)
366
+
367
+
368
+class BaichuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 13b
369
+
370
+ def __init__(self, config):
371
+ super().__init__(config, "ALIBI")
372
+
373
+
374
+class BaiChuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 7b
375
+
376
+ def __init__(self, config):
377
+ super().__init__(config, "ROPE")