@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa0addb397
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

Adding "torch compile" annotations to moe models (#9758)

Yongzao committed Oct 29, 2024 at 04:49 UTC aa0addb39726b685522e7cf154b564b4159759ad
4 files changed +8
vllm/model_executor/models/arctic.py
+2
@@ -5,6 +5,7 @@ import torch
5 from torch import nn
6
7 from vllm.attention import Attention, AttentionMetadata
8 +from vllm.compilation.decorators import support_torch_compile
9 from vllm.config import CacheConfig
10 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
11 get_tensor_model_parallel_world_size,
@@ -360,6 +361,7 @@ class ArcticDecoderLayer(nn.Module):
361 return hidden_states
362
363
364 +@support_torch_compile
365 class ArcticModel(nn.Module):
366
367 def __init__(
vllm/model_executor/models/mixtral.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers import MixtralConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -245,6 +246,7 @@ class MixtralDecoderLayer(nn.Module):
246 return hidden_states, residual
247
248
249 +@support_torch_compile
250 class MixtralModel(nn.Module):
251
252 def __init__(
vllm/model_executor/models/olmoe.py
+2
@@ -17,6 +17,7 @@ from torch import nn
17 from transformers import PretrainedConfig
18
19 from vllm.attention import Attention, AttentionMetadata
20 +from vllm.compilation.decorators import support_torch_compile
21 from vllm.config import CacheConfig
22 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
23 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -239,6 +240,7 @@ class OlmoeDecoderLayer(nn.Module):
240 return hidden_states, residual
241
242
243 +@support_torch_compile
244 class OlmoeModel(nn.Module):
245
246 def __init__(
vllm/model_executor/models/phimoe.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers.configuration_utils import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -429,6 +430,7 @@ class PhiMoEDecoderLayer(nn.Module):
430 return hidden_states, residual
431
432
433 +@support_torch_compile
434 class PhiMoEModel(nn.Module):
435
436 def __init__(