A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] Adding torch compile annotations to some models (#9614)
Yongzao committed
Oct 24, 2024 at 01:07 UTC
9013e24f7b09a19405c6856b88c004afd4e3fc57
6 files changed
+12
vllm/model_executor/models/baichuan.py
+2
@@ -26,6 +26,7 @@ from torch import nn
26
from transformers import PretrainedConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
+from vllm.compilation.decorators import support_torch_compile
30
from vllm.config import CacheConfig, LoRAConfig
31
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
32
get_tensor_model_parallel_world_size)
@@ -250,6 +251,7 @@ class BaiChuanDecoderLayer(nn.Module):
251
return hidden_states, residual
252
253
254
+@support_torch_compile
255
class BaiChuanModel(nn.Module):
256
257
def __init__(self,
vllm/model_executor/models/bloom.py
+2
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import BloomConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.compilation.decorators import support_torch_compile
28
from vllm.config import CacheConfig
29
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
30
get_tensor_model_parallel_world_size)
@@ -218,6 +219,7 @@ class BloomBlock(nn.Module):
219
return output
220
221
222
+@support_torch_compile
223
class BloomModel(nn.Module):
224
225
def __init__(
vllm/model_executor/models/commandr.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers import CohereConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
@@ -250,6 +251,7 @@ class CohereDecoderLayer(nn.Module):
251
return hidden_states, residual
252
253
254
+@support_torch_compile
255
class CohereModel(nn.Module):
256
257
def __init__(
vllm/model_executor/models/exaone.py
+2
@@ -29,6 +29,7 @@ import torch
29
from torch import nn
30
31
from vllm.attention import Attention, AttentionMetadata
32
+from vllm.compilation.decorators import support_torch_compile
33
from vllm.config import CacheConfig, LoRAConfig
34
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
35
get_tensor_model_parallel_world_size)
@@ -311,6 +312,7 @@ class ExaoneDecoderLayer(nn.Module):
312
return hidden_states, residual
313
314
315
+@support_torch_compile
316
class ExaoneModel(nn.Module):
317
318
def __init__(
vllm/model_executor/models/gemma.py
+2
@@ -22,6 +22,7 @@ from torch import nn
22
from transformers import GemmaConfig
23
24
from vllm.attention import Attention, AttentionMetadata
25
+from vllm.compilation.decorators import support_torch_compile
26
from vllm.config import CacheConfig, LoRAConfig
27
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
28
from vllm.logger import init_logger
@@ -239,6 +240,7 @@ class GemmaDecoderLayer(nn.Module):
240
return hidden_states, residual
241
242
243
+@support_torch_compile
244
class GemmaModel(nn.Module):
245
246
def __init__(
vllm/model_executor/models/gpt2.py
+2
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import GPT2Config
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.compilation.decorators import support_torch_compile
28
from vllm.config import CacheConfig
29
from vllm.distributed.parallel_state import (
30
get_pp_group, get_tensor_model_parallel_world_size)
@@ -182,6 +183,7 @@ class GPT2Block(nn.Module):
183
return hidden_states
184
185
186
+@support_torch_compile
187
class GPT2Model(nn.Module):
188
189
def __init__(