@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 521b35f799
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Support Microsoft Phi 1.5 (#1664)

maximzubkov committed Nov 16, 2023 at 23:28 UTC 521b35f799d8d7e22961a79e41256ff770ab2b95
6 files changed +320
README.md
+1
@@ -59,6 +59,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
59 - Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
60 - MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
61 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
62 +- Phi-1.5 (`microsoft/phi-1_5`, etc.)
63 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
64
65 Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
requirements.txt
+1
@@ -5,6 +5,7 @@ pandas # Required for Ray data.
5 pyarrow # Required for Ray data.
6 sentencepiece # Required for LLaMA tokenizer.
7 numpy
8 +einops # Required for phi-1_5
9 torch >= 2.1.0
10 transformers >= 4.34.0 # Required for Mistral.
11 xformers >= 0.0.22.post7 # Required for CUDA 12.1.
tests/models/test_models.py
+1
@@ -15,6 +15,7 @@ MODELS = [
15 "EleutherAI/pythia-70m",
16 "bigscience/bloom-560m",
17 "mosaicml/mpt-7b",
18 + "microsoft/phi-1_5",
19 ]
20
21
vllm/model_executor/model_loader.py
+1
@@ -32,6 +32,7 @@ _MODEL_REGISTRY = {
32 "MptForCausalLM": MPTForCausalLM,
33 "MPTForCausalLM": MPTForCausalLM,
34 "OPTForCausalLM": OPTForCausalLM,
35 + "PhiForCausalLM": PhiForCausalLM,
36 "QWenLMHeadModel": QWenLMHeadModel,
37 "RWForCausalLM": FalconForCausalLM,
38 "YiForCausalLM": YiForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -12,6 +12,7 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
12 from vllm.model_executor.models.mistral import MistralForCausalLM
13 from vllm.model_executor.models.mpt import MPTForCausalLM
14 from vllm.model_executor.models.opt import OPTForCausalLM
15 +from vllm.model_executor.models.phi_1_5 import PhiForCausalLM
16 from vllm.model_executor.models.qwen import QWenLMHeadModel
17 from vllm.model_executor.models.chatglm import ChatGLMForCausalLM
18 from vllm.model_executor.models.yi import YiForCausalLM
@@ -31,6 +32,7 @@ __all__ = [
32 "LlamaForCausalLM",
33 "MPTForCausalLM",
34 "OPTForCausalLM",
35 + "PhiForCausalLM",
36 "QWenLMHeadModel",
37 "MistralForCausalLM",
38 "YiForCausalLM",
vllm/model_executor/models/phi_1_5.py new
+314
@@ -0,0 +1,314 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright (c) Microsoft Corporation.
6 +# Licensed under the MIT license.
7 +#
8 +# BSD 3-Clause License
9 +#
10 +# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
11 +# All rights reserved.
12 +#
13 +# Redistribution and use in source and binary forms, with or without
14 +# modification, are permitted provided that the following conditions are met:
15 +#
16 +# * Redistributions of source code must retain the above copyright notice, this
17 +# list of conditions and the following disclaimer.
18 +#
19 +# * Redistributions in binary form must reproduce the above copyright notice,
20 +# this list of conditions and the following disclaimer in the documentation
21 +# and/or other materials provided with the distribution.
22 +#
23 +# * Neither the name of the copyright holder nor the names of its
24 +# contributors may be used to endorse or promote products derived from
25 +# this software without specific prior written permission.
26 +#
27 +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
28 +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
30 +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
31 +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
33 +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
34 +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
35 +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 +"""Inference-only Phi-1.5 model compatible with HuggingFace weights.
38 +
39 +The input of the model is flattened to a 1D tensor of tokens. The model uses
40 +InputMetadata to extract the original 2D shape of the input.
41 +"""
42 +from typing import List, Optional, Tuple
43 +
44 +import torch
45 +from torch import nn
46 +from transformers import PretrainedConfig
47 +
48 +from vllm.model_executor.input_metadata import InputMetadata
49 +from vllm.model_executor.layers.activation import get_act_fn
50 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
51 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52 + LinearMethodBase,
53 + QKVParallelLinear,
54 + RowParallelLinear)
55 +from vllm.model_executor.layers.sampler import Sampler
56 +from vllm.model_executor.layers.vocab_parallel_embedding import (
57 + VocabParallelEmbedding, ParallelLMHead)
58 +from vllm.model_executor.parallel_utils.parallel_state import (
59 + get_tensor_model_parallel_world_size)
60 +from vllm.model_executor.weight_utils import (default_weight_loader,
61 + hf_model_weights_iterator)
62 +from vllm.sequence import SamplerOutput
63 +
64 +KVCache = Tuple[torch.Tensor, torch.Tensor]
65 +
66 +
67 +class PhiEmbedding(nn.Module):
68 +
69 + def __init__(self, config: PretrainedConfig):
70 + super().__init__()
71 +
72 + self.wte = VocabParallelEmbedding(
73 + config.vocab_size,
74 + config.hidden_size,
75 + )
76 +
77 + def forward(self, input_ids: torch.LongTensor):
78 + return self.wte(input_ids)
79 +
80 +
81 +class PhiAttention(nn.Module):
82 +
83 + def __init__(self,
84 + config: PretrainedConfig,
85 + linear_method: Optional[LinearMethodBase] = None):
86 + super().__init__()
87 + self.total_num_heads = config.num_attention_heads
88 + self.hidden_size = config.hidden_size
89 + self.head_size = self.hidden_size // self.total_num_heads
90 +
91 + tensor_model_parallel_world_size = (
92 + get_tensor_model_parallel_world_size())
93 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
94 + self.num_heads = (self.total_num_heads //
95 + tensor_model_parallel_world_size)
96 +
97 + # pylint: disable=C0103
98 + self.Wqkv = QKVParallelLinear(
99 + self.hidden_size,
100 + self.head_size,
101 + self.total_num_heads,
102 + linear_method=linear_method,
103 + )
104 + self.qkv_proj = QKVParallelLinear(
105 + config.hidden_size,
106 + self.head_size,
107 + self.total_num_heads,
108 + bias=False,
109 + linear_method=linear_method,
110 + )
111 + self.out_proj = RowParallelLinear(
112 + self.hidden_size,
113 + self.hidden_size,
114 + linear_method=linear_method,
115 + )
116 +
117 + scaling = self.head_size**-0.5
118 + rotary_dim = config.rotary_dim
119 + assert rotary_dim % 2 == 0
120 +
121 + # pylint: disable=C0301
122 + # Refer to:
123 + # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518
124 + rope_theta = 10000
125 + max_position_embeddings = getattr(config, "n_positions", 2048)
126 + self.attn = PagedAttentionWithRoPE(
127 + self.num_heads,
128 + self.head_size,
129 + scaling,
130 + rotary_dim,
131 + base=rope_theta,
132 + max_position=max_position_embeddings)
133 +
134 + def forward(
135 + self,
136 + position_ids: torch.Tensor,
137 + hidden_states: torch.Tensor,
138 + kv_cache: KVCache,
139 + input_metadata: InputMetadata,
140 + cache_event: Optional[torch.cuda.Event],
141 + ) -> torch.Tensor:
142 + qkv, _ = self.Wqkv(hidden_states)
143 + q, k, v = qkv.chunk(chunks=3, dim=-1)
144 + k_cache, v_cache = kv_cache
145 + attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
146 + input_metadata, cache_event)
147 + output, _ = self.out_proj(attn_output)
148 + return output
149 +
150 +
151 +class PhiMLP(nn.Module):
152 +
153 + def __init__(self,
154 + config: PretrainedConfig,
155 + linear_method: Optional[LinearMethodBase] = None):
156 + super().__init__()
157 +
158 + n_inner = getattr(config, "n_inner", None)
159 + n_inner = n_inner if n_inner is not None else 4 * config.hidden_size
160 +
161 + self.fc1 = ColumnParallelLinear(
162 + config.hidden_size,
163 + n_inner,
164 + linear_method=linear_method,
165 + )
166 + self.fc2 = RowParallelLinear(
167 + n_inner,
168 + config.hidden_size,
169 + linear_method=linear_method,
170 + )
171 + self.act = get_act_fn(config.activation_function)
172 +
173 + def forward(self, hidden_states):
174 + hidden_states, _ = self.fc1(hidden_states)
175 + hidden_states = self.act(hidden_states)
176 + hidden_states, _ = self.fc2(hidden_states)
177 + return hidden_states
178 +
179 +
180 +class PhiLayer(nn.Module):
181 +
182 + def __init__(self,
183 + config: PretrainedConfig,
184 + linear_method: Optional[LinearMethodBase] = None):
185 + super().__init__()
186 + self.ln = nn.LayerNorm(config.hidden_size,
187 + eps=config.layer_norm_epsilon)
188 + self.mixer = PhiAttention(config, linear_method)
189 + self.mlp = PhiMLP(config, linear_method)
190 +
191 + def forward(
192 + self,
193 + position_ids: torch.Tensor,
194 + hidden_states: torch.Tensor,
195 + kv_cache: KVCache,
196 + input_metadata: InputMetadata,
197 + cache_event: Optional[torch.cuda.Event],
198 + ) -> torch.Tensor:
199 + residual = hidden_states
200 + hidden_states = self.ln(hidden_states)
201 + attn_outputs = self.mixer(
202 + position_ids=position_ids,
203 + hidden_states=hidden_states,
204 + kv_cache=kv_cache,
205 + input_metadata=input_metadata,
206 + cache_event=cache_event,
207 + )
208 + feed_forward_hidden_states = self.mlp(hidden_states)
209 + hidden_states = attn_outputs + feed_forward_hidden_states + residual
210 + return hidden_states
211 +
212 +
213 +class PhiCausalLMHead(nn.Module):
214 +
215 + def __init__(self, config: PretrainedConfig):
216 + super().__init__()
217 + self.ln = nn.LayerNorm(config.hidden_size,
218 + eps=config.layer_norm_epsilon)
219 + self.linear = ParallelLMHead(config.vocab_size,
220 + config.hidden_size,
221 + bias=True)
222 + self.sampler = Sampler(config.vocab_size)
223 +
224 + def forward(
225 + self,
226 + hidden_states: torch.Tensor,
227 + input_metadata: InputMetadata,
228 + ):
229 + hidden_states = self.ln(hidden_states)
230 + next_tokens = self.sampler(self.linear.weight, hidden_states,
231 + input_metadata, self.linear.bias)
232 + return next_tokens
233 +
234 +
235 +class PhiModel(nn.Module):
236 +
237 + def __init__(self,
238 + config: PretrainedConfig,
239 + linear_method: Optional[LinearMethodBase] = None):
240 + super().__init__()
241 + self.config = config
242 + self.linear_method = linear_method
243 + self.embd = PhiEmbedding(config)
244 + self.h = nn.ModuleList([
245 + PhiLayer(config, linear_method)
246 + for _ in range(config.num_hidden_layers)
247 + ])
248 +
249 + def forward(
250 + self,
251 + input_ids: torch.Tensor,
252 + positions: torch.Tensor,
253 + kv_caches: List[KVCache],
254 + input_metadata: InputMetadata,
255 + cache_events: Optional[List[torch.cuda.Event]],
256 + ) -> SamplerOutput:
257 + hidden_states = self.embd(input_ids)
258 + for i in range(self.config.num_hidden_layers):
259 + if cache_events is None:
260 + cache_event = None
261 + else:
262 + cache_event = cache_events[i]
263 + layer = self.h[i]
264 + hidden_states = layer(
265 + positions,
266 + hidden_states,
267 + kv_caches[i],
268 + input_metadata,
269 + cache_event,
270 + )
271 + return hidden_states
272 +
273 +
274 +class PhiForCausalLM(nn.Module):
275 +
276 + def __init__(self,
277 + config: PretrainedConfig,
278 + linear_method: Optional[LinearMethodBase] = None):
279 + super().__init__()
280 + self.config = config
281 + self.linear_method = linear_method
282 +
283 + self.transformer = PhiModel(config, linear_method)
284 + self.lm_head = PhiCausalLMHead(config)
285 +
286 + def forward(
287 + self,
288 + input_ids: torch.Tensor,
289 + positions: torch.Tensor,
290 + kv_caches: List[KVCache],
291 + input_metadata: InputMetadata,
292 + cache_events: Optional[List[torch.cuda.Event]],
293 + ) -> SamplerOutput:
294 + hidden_states = self.transformer(input_ids, positions, kv_caches,
295 + input_metadata, cache_events)
296 + lm_logits = self.lm_head(hidden_states, input_metadata)
297 + return lm_logits
298 +
299 + def load_weights(self,
300 + model_name_or_path: str,
301 + cache_dir: Optional[str] = None,
302 + load_format: str = "auto",
303 + revision: Optional[str] = None):
304 + params_dict = dict(self.named_parameters())
305 + for name, loaded_weight in hf_model_weights_iterator(
306 + model_name_or_path, cache_dir, load_format, revision):
307 + if "rotary_emb.inv_freq" in name:
308 + continue
309 +
310 + # pylint: disable=E1136
311 + param = params_dict[name]
312 + weight_loader = getattr(param, "weight_loader",
313 + default_weight_loader)
314 + weight_loader(param, loaded_weight)