@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 41ca62cf03
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] Add CustomOp interface for device portability (#5255)

Woosuk Kwon committed Jun 5, 2024 at 09:18 UTC 41ca62cf03b31deb68dbc14e4a92a1d4579de08b
7 files changed +100 -27
tests/kernels/test_activation.py
+2 -2
@@ -44,7 +44,7 @@ def test_act_and_mul(
44 elif activation == "gelu_tanh":
45 layer = GeluAndMul(approximate="tanh")
46 out = layer(x)
47 - ref_out = layer._forward(x)
47 + ref_out = layer.forward_native(x)
48 # The SiLU and GELU implementations are equivalent to the native PyTorch
49 # implementations, so we can do exact comparison.
50 assert torch.allclose(out, ref_out, atol=0.0, rtol=0.0)
@@ -72,7 +72,7 @@ def test_activation(
72 x = torch.randn(num_tokens, d, dtype=dtype)
73 layer = activation()
74 out = layer(x)
75 - ref_out = layer._forward(x)
75 + ref_out = layer.forward_native(x)
76 assert torch.allclose(out,
77 ref_out,
78 atol=get_default_atol(out),
tests/kernels/test_layernorm.py
+1 -1
@@ -42,7 +42,7 @@ def test_rms_norm(
42
43 # NOTE(woosuk): The reference implementation should be executed first
44 # because the custom kernel is in-place.
45 - ref_out = layer._forward(x, residual)
45 + ref_out = layer.forward_native(x, residual)
46 out = layer(x, residual)
47 # NOTE(woosuk): LayerNorm operators (including RMS) typically have larger
48 # numerical errors than other operators because they involve reductions.
tests/kernels/test_pos_encoding.py
+4 -3
@@ -64,7 +64,7 @@ def test_rotary_embedding(
64
65 # NOTE(woosuk): The reference implementation should be executed first
66 # because the custom kernel is in-place.
67 - ref_query, ref_key = rope._forward(positions, query, key)
67 + ref_query, ref_key = rope.forward_native(positions, query, key)
68 out_query, out_key = rope.forward(positions, query, key)
69 # Compare the results.
70 assert torch.allclose(out_query,
@@ -121,7 +121,7 @@ def test_batched_rotary_embedding(
121
122 # NOTE(woosuk): The reference implementation should be executed first
123 # because the custom kernel is in-place.
124 - ref_query, ref_key = rope._forward(positions, query, key)
124 + ref_query, ref_key = rope.forward_native(positions, query, key)
125 out_query, out_key = rope.forward(positions,
126 query,
127 key,
@@ -195,7 +195,8 @@ def test_batched_rotary_embedding_multi_lora(
195
196 # NOTE(woosuk): The reference implementation should be executed first
197 # because the custom kernel is in-place.
198 - ref_query, ref_key = rope._forward(positions, query, key, query_offsets)
198 + ref_query, ref_key = rope.forward_native(positions, query, key,
199 + query_offsets)
200 out_query, out_key = rope.forward(positions, query, key,
201 query_offsets.flatten())
202 # Compare the results.
vllm/model_executor/custom_op.py new
+60
@@ -0,0 +1,60 @@
1 +import torch.nn as nn
2 +
3 +from vllm.utils import is_cpu, is_hip
4 +
5 +
6 +class CustomOp(nn.Module):
7 +
8 + def __init__(self, *args, **kwargs):
9 + super().__init__()
10 + self._forward_method = self.dispatch_forward()
11 +
12 + def forward(self, *args, **kwargs):
13 + return self._forward_method(*args, **kwargs)
14 +
15 + def forward_native(self, *args, **kwargs):
16 + """PyTorch-native implementation of the forward method.
17 +
18 + This method is optional. If implemented, it can be used with compilers
19 + such as torch.compile or PyTorch XLA. Also, it can be used for testing
20 + purposes.
21 + """
22 + raise NotImplementedError
23 +
24 + def forward_cuda(self, *args, **kwargs):
25 + raise NotImplementedError
26 +
27 + def forward_hip(self, *args, **kwargs):
28 + # By default, we assume that HIP ops are compatible with CUDA ops.
29 + return self.forward_cuda(*args, **kwargs)
30 +
31 + def forward_xpu(self, *args, **kwargs):
32 + # By default, we assume that XPU ops are compatible with CUDA ops.
33 + # NOTE(woosuk): This is a placeholder for future extensions.
34 + return self.forward_cuda(*args, **kwargs)
35 +
36 + def forward_cpu(self, *args, **kwargs):
37 + # By default, we assume that CPU ops are compatible with CUDA ops.
38 + return self.forward_cuda(*args, **kwargs)
39 +
40 + def forward_tpu(self, *args, **kwargs):
41 + # By default, we assume that TPU ops are compatible with the
42 + # PyTorch-native implementation.
43 + # NOTE(woosuk): This is a placeholder for future extensions.
44 + return self.forward_native(*args, **kwargs)
45 +
46 + def forward_gaudi(self, *args, **kwargs):
47 + # By default, we assume that Gaudi ops are compatible with the
48 + # PyTorch-native implementation.
49 + # NOTE(woosuk): This is a placeholder for future extensions.
50 + return self.forward_native(*args, **kwargs)
51 +
52 + def dispatch_forward(self):
53 + # NOTE(woosuk): Here we assume that vLLM was built for only one
54 + # specific backend. Currently, we do not support dynamic dispatching.
55 + if is_hip():
56 + return self.forward_hip
57 + elif is_cpu():
58 + return self.forward_cpu
59 + else:
60 + return self.forward_cuda
vllm/model_executor/layers/activation.py
+21 -13
@@ -6,14 +6,14 @@ import torch
6 import torch.nn as nn
7 import torch.nn.functional as F
8
9 -from vllm import _custom_ops as ops
9 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10 get_tensor_model_parallel_world_size)
11 +from vllm.model_executor.custom_op import CustomOp
12 from vllm.model_executor.layers.quantization import QuantizationConfig
13 from vllm.model_executor.utils import set_weight_attrs
14
15
16 -class SiluAndMul(nn.Module):
16 +class SiluAndMul(CustomOp):
17 """An activation function for SwiGLU.
18
19 The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.
@@ -23,12 +23,14 @@ class SiluAndMul(nn.Module):
23 return: (num_tokens, d) or (batch_size, seq_len, d)
24 """
25
26 - def _forward(self, x: torch.Tensor) -> torch.Tensor:
26 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
27 """PyTorch-native implementation equivalent to forward()."""
28 d = x.shape[-1] // 2
29 return F.silu(x[..., :d]) * x[..., d:]
30
31 - def forward(self, x: torch.Tensor) -> torch.Tensor:
31 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
32 + from vllm import _custom_ops as ops
33 +
34 d = x.shape[-1] // 2
35 output_shape = (x.shape[:-1] + (d, ))
36 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
@@ -36,7 +38,7 @@ class SiluAndMul(nn.Module):
38 return out
39
40
39 -class GeluAndMul(nn.Module):
41 +class GeluAndMul(CustomOp):
42 """An activation function for GeGLU.
43
44 The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2.
@@ -52,12 +54,14 @@ class GeluAndMul(nn.Module):
54 if approximate not in ("none", "tanh"):
55 raise ValueError(f"Unknown approximate mode: {approximate}")
56
55 - def _forward(self, x: torch.Tensor) -> torch.Tensor:
57 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
58 """PyTorch-native implementation equivalent to forward()."""
59 d = x.shape[-1] // 2
60 return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:]
61
60 - def forward(self, x: torch.Tensor) -> torch.Tensor:
62 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
63 + from vllm import _custom_ops as ops
64 +
65 d = x.shape[-1] // 2
66 output_shape = (x.shape[:-1] + (d, ))
67 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
@@ -71,28 +75,32 @@ class GeluAndMul(nn.Module):
75 return f'approximate={repr(self.approximate)}'
76
77
74 -class NewGELU(nn.Module):
78 +class NewGELU(CustomOp):
79
76 - def _forward(self, x: torch.Tensor) -> torch.Tensor:
80 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
81 """PyTorch-native implementation equivalent to forward()."""
82 c = math.sqrt(2.0 / math.pi)
83 return 0.5 * x * (1.0 + torch.tanh(c *
84 (x + 0.044715 * torch.pow(x, 3.0))))
85
82 - def forward(self, x: torch.Tensor) -> torch.Tensor:
86 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
87 + from vllm import _custom_ops as ops
88 +
89 out = torch.empty_like(x)
90 ops.gelu_new(out, x)
91 return out
92
93
88 -class FastGELU(nn.Module):
94 +class FastGELU(CustomOp):
95
90 - def _forward(self, x: torch.Tensor) -> torch.Tensor:
96 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
97 """PyTorch-native implementation equivalent to forward()."""
98 return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 *
99 (1.0 + 0.044715 * x * x)))
100
95 - def forward(self, x: torch.Tensor) -> torch.Tensor:
101 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
102 + from vllm import _custom_ops as ops
103 +
104 out = torch.empty_like(x)
105 ops.gelu_fast(out, x)
106 return out
vllm/model_executor/layers/layernorm.py
+6 -4
@@ -4,10 +4,10 @@ from typing import Optional, Tuple, Union
4 import torch
5 import torch.nn as nn
6
7 -from vllm import _custom_ops as ops
7 +from vllm.model_executor.custom_op import CustomOp
8
9
10 -class RMSNorm(nn.Module):
10 +class RMSNorm(CustomOp):
11 """Root mean square normalization.
12
13 Computes x -> w * x / sqrt(E[x^2] + eps) where w is the learned weight.
@@ -23,7 +23,7 @@ class RMSNorm(nn.Module):
23 self.weight = nn.Parameter(torch.ones(hidden_size))
24 self.variance_epsilon = eps
25
26 - def _forward(
26 + def forward_native(
27 self,
28 x: torch.Tensor,
29 residual: Optional[torch.Tensor] = None,
@@ -43,11 +43,13 @@ class RMSNorm(nn.Module):
43 else:
44 return x, residual
45
46 - def forward(
46 + def forward_cuda(
47 self,
48 x: torch.Tensor,
49 residual: Optional[torch.Tensor] = None,
50 ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
51 + from vllm import _custom_ops as ops
52 +
53 if residual is not None:
54 ops.fused_add_rms_norm(
55 x,
vllm/model_executor/layers/rotary_embedding.py
+6 -4
@@ -27,7 +27,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
27 import torch
28 import torch.nn as nn
29
30 -from vllm import _custom_ops as ops
30 +from vllm.model_executor.custom_op import CustomOp
31
32
33 def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
@@ -43,7 +43,7 @@ def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
43 return x.flatten(-2)
44
45
46 -class RotaryEmbedding(nn.Module):
46 +class RotaryEmbedding(CustomOp):
47 """Original rotary positional embedding."""
48
49 def __init__(
@@ -93,7 +93,7 @@ class RotaryEmbedding(nn.Module):
93 cache = torch.cat((cos, sin), dim=-1)
94 return cache
95
96 - def _forward(
96 + def forward_native(
97 self,
98 positions: torch.Tensor,
99 query: torch.Tensor,
@@ -138,13 +138,15 @@ class RotaryEmbedding(nn.Module):
138 key = key.flatten(-2)
139 return query, key
140
141 - def forward(
141 + def forward_cuda(
142 self,
143 positions: torch.Tensor,
144 query: torch.Tensor,
145 key: torch.Tensor,
146 offsets: Optional[torch.Tensor] = None,
147 ) -> Tuple[torch.Tensor, torch.Tensor]:
148 + from vllm import _custom_ops as ops
149 +
150 self.cos_sin_cache = self.cos_sin_cache.to(positions.device,
151 dtype=query.dtype)
152 # ops.rotary_embedding()/batched_rotary_embedding()