A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
Adding "torch compile" annotations to moe models (#9758)
Yongzao committed
Oct 29, 2024 at 04:49 UTC
aa0addb39726b685522e7cf154b564b4159759ad
4 files changed
+8
vllm/model_executor/models/arctic.py
+2
@@ -5,6 +5,7 @@ import torch
5
from torch import nn
6
7
from vllm.attention import Attention, AttentionMetadata
8
+from vllm.compilation.decorators import support_torch_compile
9
from vllm.config import CacheConfig
10
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
11
get_tensor_model_parallel_world_size,
@@ -360,6 +361,7 @@ class ArcticDecoderLayer(nn.Module):
361
return hidden_states
362
363
364
+@support_torch_compile
365
class ArcticModel(nn.Module):
366
367
def __init__(
vllm/model_executor/models/mixtral.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers import MixtralConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -245,6 +246,7 @@ class MixtralDecoderLayer(nn.Module):
246
return hidden_states, residual
247
248
249
+@support_torch_compile
250
class MixtralModel(nn.Module):
251
252
def __init__(
vllm/model_executor/models/olmoe.py
+2
@@ -17,6 +17,7 @@ from torch import nn
17
from transformers import PretrainedConfig
18
19
from vllm.attention import Attention, AttentionMetadata
20
+from vllm.compilation.decorators import support_torch_compile
21
from vllm.config import CacheConfig
22
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
23
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -239,6 +240,7 @@ class OlmoeDecoderLayer(nn.Module):
240
return hidden_states, residual
241
242
243
+@support_torch_compile
244
class OlmoeModel(nn.Module):
245
246
def __init__(
vllm/model_executor/models/phimoe.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers.configuration_utils import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -429,6 +430,7 @@ class PhiMoEDecoderLayer(nn.Module):
430
return hidden_states, residual
431
432
433
+@support_torch_compile
434
class PhiMoEModel(nn.Module):
435
436
def __init__(