@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64f23c2900
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix baichuan for different position embedding for 7b and 13b models (#643)

Song committed Aug 2, 2023 at 13:22 UTC 64f23c29000ca832f50d3470a83d7a9bdbe95a0c
3 files changed +76 -17
vllm/model_executor/model_loader.py
+2 -1
@@ -11,7 +11,8 @@ from vllm.model_executor.weight_utils import initialize_dummy_weights
11
12 # TODO(woosuk): Lazy-load the model classes.
13 _MODEL_REGISTRY = {
14 - "BaiChuanForCausalLM": BaiChuanForCausalLM,
14 + "BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
15 + "BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
16 "BloomForCausalLM": BloomForCausalLM,
17 "GPT2LMHeadModel": GPT2LMHeadModel,
18 "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
vllm/model_executor/models/__init__.py
+2 -1
@@ -1,4 +1,4 @@
1 -from vllm.model_executor.models.baichuan import BaiChuanForCausalLM
1 +from vllm.model_executor.models.baichuan import BaiChuanForCausalLM, BaichuanForCausalLM
2 from vllm.model_executor.models.bloom import BloomForCausalLM
3 from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
4 from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
@@ -10,6 +10,7 @@ from vllm.model_executor.models.opt import OPTForCausalLM
10
11 __all__ = [
12 "BaiChuanForCausalLM",
13 + "BaichuanForCausalLM",
14 "BloomForCausalLM",
15 "GPT2LMHeadModel",
16 "GPTBigCodeForCausalLM",
vllm/model_executor/models/baichuan.py
+72 -15
@@ -22,6 +22,7 @@
22 The input of the model is flattened to a 1D tensor of tokens. The model uses
23 InputMetadata to extract the original 2D shape of the input.
24 """
25 +import math
26 from typing import Dict, List, Optional, Tuple
27
28 import torch
@@ -31,7 +32,7 @@ from vllm.sequence import SequenceOutputs
32 from vllm.model_executor.input_metadata import InputMetadata
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.layernorm import RMSNorm
34 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
35 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE, PagedAttentionWithALiBi
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
38 load_tensor_parallel_weights)
@@ -44,6 +45,31 @@ from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
45 KVCache = Tuple[torch.Tensor, torch.Tensor]
46
47
48 +def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
49 + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
50 + base = torch.tensor(
51 + 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
52 + dtype=torch.float32,
53 + )
54 + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
55 + slopes = torch.pow(base, powers)
56 +
57 + if closest_power_of_2 != total_num_heads:
58 + extra_base = torch.tensor(
59 + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
60 + dtype=torch.float32,
61 + )
62 + num_remaining_heads = min(closest_power_of_2,
63 + total_num_heads - closest_power_of_2)
64 + extra_powers = torch.arange(start=1,
65 + end=1 + 2 * num_remaining_heads,
66 + step=2,
67 + dtype=torch.int32)
68 + slopes = torch.cat(
69 + [slopes, torch.pow(extra_base, extra_powers)], dim=0)
70 + return slopes
71 +
72 +
73 class BaiChuanMLP(nn.Module):
74
75 def __init__(
@@ -82,6 +108,7 @@ class BaiChuanAttention(nn.Module):
108 self,
109 hidden_size: int,
110 num_heads: int,
111 + position_embedding: str,
112 ):
113 super().__init__()
114 self.hidden_size = hidden_size
@@ -92,7 +119,7 @@ class BaiChuanAttention(nn.Module):
119 self.num_heads = (self.total_num_heads //
120 tensor_model_parallel_world_size)
121 self.head_dim = hidden_size // self.total_num_heads
95 - self.scaling = self.head_dim**-0.5
122 + self.postion_embedding = position_embedding
123
124 # pylint: disable=invalid-name
125 self.W_pack = ColumnParallelLinear(
@@ -109,11 +136,23 @@ class BaiChuanAttention(nn.Module):
136 input_is_parallel=True,
137 perform_initialization=False,
138 )
112 -
113 - self.attn = PagedAttentionWithRoPE(self.num_heads,
114 - self.head_dim,
115 - self.scaling,
116 - rotary_dim=self.head_dim)
139 + # Create the alibi slopes and slice them.
140 + if self.postion_embedding == "ALIBI":
141 + tp_rank = get_tensor_model_parallel_rank()
142 + head_start = tp_rank * self.num_heads
143 + head_end = (tp_rank + 1) * self.num_heads
144 + alibi_slopes = _get_alibi_slopes(self.total_num_heads)
145 + alibi_slopes = alibi_slopes[head_start:head_end].tolist()
146 +
147 + scaling = self.head_dim**-0.5
148 + self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
149 + scaling, alibi_slopes)
150 + else:
151 + self.scaling = self.head_dim**-0.5
152 + self.attn = PagedAttentionWithRoPE(self.num_heads,
153 + self.head_dim,
154 + self.scaling,
155 + rotary_dim=self.head_dim)
156
157 def forward(
158 self,
@@ -126,20 +165,26 @@ class BaiChuanAttention(nn.Module):
165 qkv, _ = self.W_pack(hidden_states)
166 q, k, v = qkv.chunk(chunks=3, dim=-1)
167 k_cache, v_cache = kv_cache
129 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
130 - input_metadata, cache_event)
168 + if self.postion_embedding == "ALIBI":
169 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
170 + cache_event)
171 + else:
172 + attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
173 + input_metadata, cache_event)
174 +
175 output, _ = self.o_proj(attn_output)
176 return output
177
178
179 class BaiChuanDecoderLayer(nn.Module):
180
137 - def __init__(self, config: BaiChuanConfig):
181 + def __init__(self, config: BaiChuanConfig, position_embedding: str):
182 super().__init__()
183 self.hidden_size = config.hidden_size
184 self.self_attn = BaiChuanAttention(
185 hidden_size=self.hidden_size,
186 num_heads=config.num_attention_heads,
187 + position_embedding=position_embedding,
188 )
189 self.mlp = BaiChuanMLP(
190 hidden_size=self.hidden_size,
@@ -181,7 +226,7 @@ class BaiChuanDecoderLayer(nn.Module):
226
227 class BaiChuanModel(nn.Module):
228
184 - def __init__(self, config: BaiChuanConfig):
229 + def __init__(self, config: BaiChuanConfig, position_embedding: str):
230 super().__init__()
231 self.config = config
232 self.padding_idx = config.pad_token_id
@@ -192,7 +237,7 @@ class BaiChuanModel(nn.Module):
237 config.hidden_size,
238 perform_initialization=False)
239 self.layers = nn.ModuleList([
195 - BaiChuanDecoderLayer(config)
240 + BaiChuanDecoderLayer(config, position_embedding)
241 for _ in range(config.num_hidden_layers)
242 ])
243 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -223,12 +268,12 @@ class BaiChuanModel(nn.Module):
268 return hidden_states
269
270
226 -class BaiChuanForCausalLM(nn.Module):
271 +class BaiChuanBaseForCausalLM(nn.Module):
272
228 - def __init__(self, config):
273 + def __init__(self, config, position_embedding: str):
274 super().__init__()
275 self.config = config
231 - self.model = BaiChuanModel(config)
276 + self.model = BaiChuanModel(config, position_embedding)
277 self.lm_head = ColumnParallelLinear(config.hidden_size,
278 config.vocab_size,
279 bias=False,
@@ -318,3 +363,15 @@ class BaiChuanForCausalLM(nn.Module):
363 self._row_parallel_weights,
364 tp_rank,
365 )
366 +
367 +
368 +class BaichuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 13b
369 +
370 + def __init__(self, config):
371 + super().__init__(config, "ALIBI")
372 +
373 +
374 +class BaiChuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 7b
375 +
376 + def __init__(self, config):
377 + super().__init__(config, "ROPE")