A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Use torch.compile for GemmaRMSNorm (#7642)
Woosuk Kwon committed
Aug 22, 2024 at 01:14 UTC
b3856bef7df6e2f86e54a766eb83b4d77f98ee90
1 file changed
+23
-6
vllm/model_executor/layers/layernorm.py
+23
-6
@@ -114,10 +114,12 @@ class GemmaRMSNorm(CustomOp):
114
self.weight = nn.Parameter(torch.zeros(hidden_size))
115
self.variance_epsilon = eps
116
117
- def forward_native(
118
- self,
117
+ @staticmethod
118
+ def forward_static(
119
+ weight: torch.Tensor,
120
+ variance_epsilon: float,
121
x: torch.Tensor,
120
- residual: Optional[torch.Tensor] = None,
122
+ residual: Optional[torch.Tensor],
123
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
124
"""PyTorch-native implementation equivalent to forward()."""
125
orig_dtype = x.dtype
@@ -127,17 +129,32 @@ class GemmaRMSNorm(CustomOp):
129
130
x = x.float()
131
variance = x.pow(2).mean(dim=-1, keepdim=True)
130
- x = x * torch.rsqrt(variance + self.variance_epsilon)
132
+ x = x * torch.rsqrt(variance + variance_epsilon)
133
# Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16)
134
# See https://github.com/huggingface/transformers/pull/29402
133
- x = x * (1.0 + self.weight.float())
135
+ x = x * (1.0 + weight.float())
136
x = x.to(orig_dtype)
137
return x if residual is None else (x, residual)
138
139
+ def forward_native(
140
+ self,
141
+ x: torch.Tensor,
142
+ residual: Optional[torch.Tensor] = None,
143
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
144
+ """PyTorch-native implementation equivalent to forward()."""
145
+ return self.forward_static(self.weight.data, self.variance_epsilon, x,
146
+ residual)
147
+
148
def forward_cuda(
149
self,
150
x: torch.Tensor,
151
residual: Optional[torch.Tensor] = None,
152
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
142
- # TODO(woosuk): Implement an optimized kernel for GemmaRMSNorm.
153
+ if torch.compiler.is_compiling():
154
+ return self.forward_native(x, residual)
155
+
156
+ if not getattr(self, "_is_compiled", False):
157
+ self.forward_static = torch.compile( # type: ignore
158
+ self.forward_static)
159
+ self._is_compiled = True
160
return self.forward_native(x, residual)