@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d27cfbf791
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] Adding torch compile annotations to some models (#9641)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

Yongzao committed Oct 25, 2024 at 00:31 UTC d27cfbf791ef01483db9c45e215f3f299e54a079
7 files changed +21 -11
tests/distributed/test_pipeline_parallel.py
+2 -1
@@ -171,7 +171,8 @@ TEXT_GENERATION_MODELS = {
171 "stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
172 "bigcode/starcoder2-3b": PPTestSettings.fast(),
173 "upstage/solar-pro-preview-instruct": PPTestSettings.fast(tp_base=2),
174 - # FIXME: Cannot load tokenizer in latest transformers version
174 + # FIXME: Cannot load tokenizer in latest transformers version.
175 + # Need to use tokenizer from `meta-llama/Llama-2-7b-chat-hf`
176 # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
177 # [Encoder-only]
178 # TODO: Implement PP
vllm/model_executor/models/opt.py
+2
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import OPTConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.compilation.decorators import support_torch_compile
28 from vllm.config import CacheConfig
29 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
@@ -279,6 +280,7 @@ class OPTDecoder(nn.Module):
280 return hidden_states
281
282
283 +@support_torch_compile
284 class OPTModel(nn.Module):
285
286 def __init__(
vllm/model_executor/models/orion.py
+8 -10
@@ -11,6 +11,7 @@ from torch import nn
11 from transformers import PretrainedConfig
12
13 from vllm.attention import Attention, AttentionMetadata
14 +from vllm.compilation.decorators import support_torch_compile
15 from vllm.config import CacheConfig
16 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
17 from vllm.model_executor.layers.activation import SiluAndMul
@@ -184,7 +185,6 @@ class OrionDecoderLayer(nn.Module):
185 hidden_states: torch.Tensor,
186 kv_cache: torch.Tensor,
187 attn_metadata: AttentionMetadata,
187 - residual: Optional[torch.Tensor],
188 ) -> Tuple[torch.Tensor, torch.Tensor]:
189 # Self Attention
190 residual = hidden_states
@@ -203,9 +203,10 @@ class OrionDecoderLayer(nn.Module):
203 hidden_states = self.post_attention_layernorm(hidden_states)
204 hidden_states = self.mlp(hidden_states)
205 hidden_states = residual + hidden_states
206 - return hidden_states, None
206 + return hidden_states
207
208
209 +@support_torch_compile
210 class OrionModel(nn.Module):
211
212 def __init__(
@@ -233,8 +234,9 @@ class OrionModel(nn.Module):
234 prefix=f"{prefix}.layers")
235 self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
236 self.make_empty_intermediate_tensors = (
236 - make_empty_intermediate_tensors_factory(
237 - ["hidden_states", "residual"], config.hidden_size))
237 + make_empty_intermediate_tensors_factory([
238 + "hidden_states",
239 + ], config.hidden_size))
240
241 def forward(
242 self,
@@ -246,24 +248,20 @@ class OrionModel(nn.Module):
248 ) -> Union[torch.Tensor, IntermediateTensors]:
249 if get_pp_group().is_first_rank:
250 hidden_states = self.embed_tokens(input_ids)
249 - residual = None
251 else:
251 - assert intermediate_tensors
252 + assert intermediate_tensors is not None
253 hidden_states = intermediate_tensors["hidden_states"]
253 - residual = intermediate_tensors["residual"]
254 for i in range(self.start_layer, self.end_layer):
255 layer = self.layers[i]
256 - hidden_states, residual = layer(
256 + hidden_states = layer(
257 positions,
258 hidden_states,
259 kv_caches[i - self.start_layer],
260 attn_metadata,
261 - residual,
261 )
262 if not get_pp_group().is_last_rank:
263 return IntermediateTensors({
264 "hidden_states": hidden_states,
266 - "residual": residual
265 })
266 hidden_states = self.norm(hidden_states)
267 return hidden_states
vllm/model_executor/models/persimmon.py
+2
@@ -27,6 +27,7 @@ from torch import nn
27 from transformers import PersimmonConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 +from vllm.compilation.decorators import support_torch_compile
31 from vllm.config import CacheConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import get_act_fn
@@ -209,6 +210,7 @@ class PersimmonDecoderLayer(nn.Module):
210 return outputs
211
212
213 +@support_torch_compile
214 class PersimmonModel(nn.Module):
215
216 def __init__(self,
vllm/model_executor/models/solar.py
+2
@@ -29,6 +29,7 @@ from torch import nn
29 from transformers import PretrainedConfig
30
31 from vllm.attention import Attention, AttentionMetadata
32 +from vllm.compilation.decorators import support_torch_compile
33 from vllm.config import CacheConfig, LoRAConfig
34 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
35 get_tensor_model_parallel_world_size)
@@ -263,6 +264,7 @@ class SolarDecoderLayer(nn.Module):
264 return hidden_states, residual
265
266
267 +@support_torch_compile
268 class SolarModel(nn.Module):
269
270 def __init__(
vllm/model_executor/models/starcoder2.py
+2
@@ -25,6 +25,7 @@ from torch import nn
25 from transformers import Starcoder2Config
26
27 from vllm.attention import Attention, AttentionMetadata
28 +from vllm.compilation.decorators import support_torch_compile
29 from vllm.config import CacheConfig
30 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
31 from vllm.model_executor.layers.activation import get_act_fn
@@ -193,6 +194,7 @@ class Starcoder2DecoderLayer(nn.Module):
194 return hidden_states
195
196
197 +@support_torch_compile
198 class Starcoder2Model(nn.Module):
199
200 def __init__(self,
vllm/model_executor/models/xverse.py
+3
@@ -27,6 +27,7 @@ from torch import nn
27 from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 +from vllm.compilation.decorators import support_torch_compile
31 from vllm.config import CacheConfig, LoRAConfig
32 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
33 from vllm.model_executor.layers.activation import SiluAndMul
@@ -220,6 +221,7 @@ class XverseDecoderLayer(nn.Module):
221 return hidden_states, residual
222
223
224 +@support_torch_compile
225 class XverseModel(nn.Module):
226
227 def __init__(
@@ -266,6 +268,7 @@ class XverseModel(nn.Module):
268 residual = None
269 else:
270 hidden_states = intermediate_tensors["hidden_states"]
271 + residual = intermediate_tensors["residual"]
272 for i in range(self.start_layer, self.end_layer):
273 layer = self.layers[i]
274 hidden_states, residual = layer(