A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Model] torch.compile for layernorm in commandr (#3985)
[Core][Model] Use torch.compile to accelerate layernorm in commandr (#3985)
youkaichao committed
Apr 10, 2024 at 18:48 UTC
caada5e50aa16cd5f59bd7889128a83588ca1f99
1 file changed
+15
-8
vllm/model_executor/models/commandr.py
+15
-8
@@ -48,6 +48,18 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
48
from vllm.sequence import SamplerOutput
49
50
51
+@torch.compile
52
+def layer_norm_func(hidden_states, weight, variance_epsilon):
53
+ input_dtype = hidden_states.dtype
54
+ hidden_states = hidden_states.to(torch.float32)
55
+ mean = hidden_states.mean(-1, keepdim=True)
56
+ variance = (hidden_states - mean).pow(2).mean(-1, keepdim=True)
57
+ hidden_states = (hidden_states - mean) * torch.rsqrt(variance +
58
+ variance_epsilon)
59
+ hidden_states = weight.to(torch.float32) * hidden_states
60
+ return hidden_states.to(input_dtype)
61
+
62
+
63
class LayerNorm(nn.Module):
64
65
def __init__(self, param_shape=None, eps=1e-5):
@@ -57,14 +69,9 @@ class LayerNorm(nn.Module):
69
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
70
71
def forward(self, hidden_states, residuals=None):
60
- input_dtype = hidden_states.dtype
61
- hidden_states = hidden_states.to(torch.float32)
62
- mean = hidden_states.mean(-1, keepdim=True)
63
- variance = (hidden_states - mean).pow(2).mean(-1, keepdim=True)
64
- hidden_states = (hidden_states -
65
- mean) * torch.rsqrt(variance + self.variance_epsilon)
66
- hidden_states = self.weight.to(torch.float32) * hidden_states
67
- return hidden_states.to(input_dtype), residuals
72
+ hidden_states = layer_norm_func(hidden_states, self.weight,
73
+ self.variance_epsilon)
74
+ return hidden_states, residuals
75
76
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
77
tp_rank = get_tensor_model_parallel_rank()