A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] Adding torch compile annotations to some models (#9641)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Yongzao committed
Oct 25, 2024 at 00:31 UTC
d27cfbf791ef01483db9c45e215f3f299e54a079
7 files changed
+21
-11
tests/distributed/test_pipeline_parallel.py
+2
-1
@@ -171,7 +171,8 @@ TEXT_GENERATION_MODELS = {
171
"stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
172
"bigcode/starcoder2-3b": PPTestSettings.fast(),
173
"upstage/solar-pro-preview-instruct": PPTestSettings.fast(tp_base=2),
174
- # FIXME: Cannot load tokenizer in latest transformers version
174
+ # FIXME: Cannot load tokenizer in latest transformers version.
175
+ # Need to use tokenizer from `meta-llama/Llama-2-7b-chat-hf`
176
# "xverse/XVERSE-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
177
# [Encoder-only]
178
# TODO: Implement PP
vllm/model_executor/models/opt.py
+2
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import OPTConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.compilation.decorators import support_torch_compile
28
from vllm.config import CacheConfig
29
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
@@ -279,6 +280,7 @@ class OPTDecoder(nn.Module):
280
return hidden_states
281
282
283
+@support_torch_compile
284
class OPTModel(nn.Module):
285
286
def __init__(
vllm/model_executor/models/orion.py
+8
-10
@@ -11,6 +11,7 @@ from torch import nn
11
from transformers import PretrainedConfig
12
13
from vllm.attention import Attention, AttentionMetadata
14
+from vllm.compilation.decorators import support_torch_compile
15
from vllm.config import CacheConfig
16
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
17
from vllm.model_executor.layers.activation import SiluAndMul
@@ -184,7 +185,6 @@ class OrionDecoderLayer(nn.Module):
185
hidden_states: torch.Tensor,
186
kv_cache: torch.Tensor,
187
attn_metadata: AttentionMetadata,
187
- residual: Optional[torch.Tensor],
188
) -> Tuple[torch.Tensor, torch.Tensor]:
189
# Self Attention
190
residual = hidden_states
@@ -203,9 +203,10 @@ class OrionDecoderLayer(nn.Module):
203
hidden_states = self.post_attention_layernorm(hidden_states)
204
hidden_states = self.mlp(hidden_states)
205
hidden_states = residual + hidden_states
206
- return hidden_states, None
206
+ return hidden_states
207
208
209
+@support_torch_compile
210
class OrionModel(nn.Module):
211
212
def __init__(
@@ -233,8 +234,9 @@ class OrionModel(nn.Module):
234
prefix=f"{prefix}.layers")
235
self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
236
self.make_empty_intermediate_tensors = (
236
- make_empty_intermediate_tensors_factory(
237
- ["hidden_states", "residual"], config.hidden_size))
237
+ make_empty_intermediate_tensors_factory([
238
+ "hidden_states",
239
+ ], config.hidden_size))
240
241
def forward(
242
self,
@@ -246,24 +248,20 @@ class OrionModel(nn.Module):
248
) -> Union[torch.Tensor, IntermediateTensors]:
249
if get_pp_group().is_first_rank:
250
hidden_states = self.embed_tokens(input_ids)
249
- residual = None
251
else:
251
- assert intermediate_tensors
252
+ assert intermediate_tensors is not None
253
hidden_states = intermediate_tensors["hidden_states"]
253
- residual = intermediate_tensors["residual"]
254
for i in range(self.start_layer, self.end_layer):
255
layer = self.layers[i]
256
- hidden_states, residual = layer(
256
+ hidden_states = layer(
257
positions,
258
hidden_states,
259
kv_caches[i - self.start_layer],
260
attn_metadata,
261
- residual,
261
)
262
if not get_pp_group().is_last_rank:
263
return IntermediateTensors({
264
"hidden_states": hidden_states,
266
- "residual": residual
265
})
266
hidden_states = self.norm(hidden_states)
267
return hidden_states
vllm/model_executor/models/persimmon.py
+2
@@ -27,6 +27,7 @@ from torch import nn
27
from transformers import PersimmonConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
+from vllm.compilation.decorators import support_torch_compile
31
from vllm.config import CacheConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import get_act_fn
@@ -209,6 +210,7 @@ class PersimmonDecoderLayer(nn.Module):
210
return outputs
211
212
213
+@support_torch_compile
214
class PersimmonModel(nn.Module):
215
216
def __init__(self,
vllm/model_executor/models/solar.py
+2
@@ -29,6 +29,7 @@ from torch import nn
29
from transformers import PretrainedConfig
30
31
from vllm.attention import Attention, AttentionMetadata
32
+from vllm.compilation.decorators import support_torch_compile
33
from vllm.config import CacheConfig, LoRAConfig
34
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
35
get_tensor_model_parallel_world_size)
@@ -263,6 +264,7 @@ class SolarDecoderLayer(nn.Module):
264
return hidden_states, residual
265
266
267
+@support_torch_compile
268
class SolarModel(nn.Module):
269
270
def __init__(
vllm/model_executor/models/starcoder2.py
+2
@@ -25,6 +25,7 @@ from torch import nn
25
from transformers import Starcoder2Config
26
27
from vllm.attention import Attention, AttentionMetadata
28
+from vllm.compilation.decorators import support_torch_compile
29
from vllm.config import CacheConfig
30
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
31
from vllm.model_executor.layers.activation import get_act_fn
@@ -193,6 +194,7 @@ class Starcoder2DecoderLayer(nn.Module):
194
return hidden_states
195
196
197
+@support_torch_compile
198
class Starcoder2Model(nn.Module):
199
200
def __init__(self,
vllm/model_executor/models/xverse.py
+3
@@ -27,6 +27,7 @@ from torch import nn
27
from transformers import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
+from vllm.compilation.decorators import support_torch_compile
31
from vllm.config import CacheConfig, LoRAConfig
32
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33
from vllm.model_executor.layers.activation import SiluAndMul
@@ -220,6 +221,7 @@ class XverseDecoderLayer(nn.Module):
221
return hidden_states, residual
222
223
224
+@support_torch_compile
225
class XverseModel(nn.Module):
226
227
def __init__(
@@ -266,6 +268,7 @@ class XverseModel(nn.Module):
268
residual = None
269
else:
270
hidden_states = intermediate_tensors["hidden_states"]
271
+ residual = intermediate_tensors["residual"]
272
for i in range(self.start_layer, self.end_layer):
273
layer = self.layers[i]
274
hidden_states, residual = layer(