@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9013e24f7b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] Adding torch compile annotations to some models (#9614)

Yongzao committed Oct 24, 2024 at 01:07 UTC 9013e24f7b09a19405c6856b88c004afd4e3fc57
6 files changed +12
vllm/model_executor/models/baichuan.py
+2
@@ -26,6 +26,7 @@ from torch import nn
26 from transformers import PretrainedConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 +from vllm.compilation.decorators import support_torch_compile
30 from vllm.config import CacheConfig, LoRAConfig
31 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32 get_tensor_model_parallel_world_size)
@@ -250,6 +251,7 @@ class BaiChuanDecoderLayer(nn.Module):
251 return hidden_states, residual
252
253
254 +@support_torch_compile
255 class BaiChuanModel(nn.Module):
256
257 def __init__(self,
vllm/model_executor/models/bloom.py
+2
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import BloomConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.compilation.decorators import support_torch_compile
28 from vllm.config import CacheConfig
29 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
30 get_tensor_model_parallel_world_size)
@@ -218,6 +219,7 @@ class BloomBlock(nn.Module):
219 return output
220
221
222 +@support_torch_compile
223 class BloomModel(nn.Module):
224
225 def __init__(
vllm/model_executor/models/commandr.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers import CohereConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
@@ -250,6 +251,7 @@ class CohereDecoderLayer(nn.Module):
251 return hidden_states, residual
252
253
254 +@support_torch_compile
255 class CohereModel(nn.Module):
256
257 def __init__(
vllm/model_executor/models/exaone.py
+2
@@ -29,6 +29,7 @@ import torch
29 from torch import nn
30
31 from vllm.attention import Attention, AttentionMetadata
32 +from vllm.compilation.decorators import support_torch_compile
33 from vllm.config import CacheConfig, LoRAConfig
34 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
35 get_tensor_model_parallel_world_size)
@@ -311,6 +312,7 @@ class ExaoneDecoderLayer(nn.Module):
312 return hidden_states, residual
313
314
315 +@support_torch_compile
316 class ExaoneModel(nn.Module):
317
318 def __init__(
vllm/model_executor/models/gemma.py
+2
@@ -22,6 +22,7 @@ from torch import nn
22 from transformers import GemmaConfig
23
24 from vllm.attention import Attention, AttentionMetadata
25 +from vllm.compilation.decorators import support_torch_compile
26 from vllm.config import CacheConfig, LoRAConfig
27 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
28 from vllm.logger import init_logger
@@ -239,6 +240,7 @@ class GemmaDecoderLayer(nn.Module):
240 return hidden_states, residual
241
242
243 +@support_torch_compile
244 class GemmaModel(nn.Module):
245
246 def __init__(
vllm/model_executor/models/gpt2.py
+2
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import GPT2Config
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.compilation.decorators import support_torch_compile
28 from vllm.config import CacheConfig
29 from vllm.distributed.parallel_state import (
30 get_pp_group, get_tensor_model_parallel_world_size)
@@ -182,6 +183,7 @@ class GPT2Block(nn.Module):
183 return hidden_states
184
185
186 +@support_torch_compile
187 class GPT2Model(nn.Module):
188
189 def __init__(