@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b3856bef7d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Use torch.compile for GemmaRMSNorm (#7642)

Woosuk Kwon committed Aug 22, 2024 at 01:14 UTC b3856bef7df6e2f86e54a766eb83b4d77f98ee90
1 file changed +23 -6
vllm/model_executor/layers/layernorm.py
+23 -6
@@ -114,10 +114,12 @@ class GemmaRMSNorm(CustomOp):
114 self.weight = nn.Parameter(torch.zeros(hidden_size))
115 self.variance_epsilon = eps
116
117 - def forward_native(
118 - self,
117 + @staticmethod
118 + def forward_static(
119 + weight: torch.Tensor,
120 + variance_epsilon: float,
121 x: torch.Tensor,
120 - residual: Optional[torch.Tensor] = None,
122 + residual: Optional[torch.Tensor],
123 ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
124 """PyTorch-native implementation equivalent to forward()."""
125 orig_dtype = x.dtype
@@ -127,17 +129,32 @@ class GemmaRMSNorm(CustomOp):
129
130 x = x.float()
131 variance = x.pow(2).mean(dim=-1, keepdim=True)
130 - x = x * torch.rsqrt(variance + self.variance_epsilon)
132 + x = x * torch.rsqrt(variance + variance_epsilon)
133 # Llama does x.to(float16) * w whilst Gemma is (x * w).to(float16)
134 # See https://github.com/huggingface/transformers/pull/29402
133 - x = x * (1.0 + self.weight.float())
135 + x = x * (1.0 + weight.float())
136 x = x.to(orig_dtype)
137 return x if residual is None else (x, residual)
138
139 + def forward_native(
140 + self,
141 + x: torch.Tensor,
142 + residual: Optional[torch.Tensor] = None,
143 + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
144 + """PyTorch-native implementation equivalent to forward()."""
145 + return self.forward_static(self.weight.data, self.variance_epsilon, x,
146 + residual)
147 +
148 def forward_cuda(
149 self,
150 x: torch.Tensor,
151 residual: Optional[torch.Tensor] = None,
152 ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
142 - # TODO(woosuk): Implement an optimized kernel for GemmaRMSNorm.
153 + if torch.compiler.is_compiling():
154 + return self.forward_native(x, residual)
155 +
156 + if not getattr(self, "_is_compiled", False):
157 + self.forward_static = torch.compile( # type: ignore
158 + self.forward_static)
159 + self._is_compiled = True
160 return self.forward_native(x, residual)