A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] Add CustomOp interface for device portability (#5255)
Woosuk Kwon committed
Jun 5, 2024 at 09:18 UTC
41ca62cf03b31deb68dbc14e4a92a1d4579de08b
7 files changed
+100
-27
tests/kernels/test_activation.py
+2
-2
@@ -44,7 +44,7 @@ def test_act_and_mul(
44
elif activation == "gelu_tanh":
45
layer = GeluAndMul(approximate="tanh")
46
out = layer(x)
47
- ref_out = layer._forward(x)
47
+ ref_out = layer.forward_native(x)
48
# The SiLU and GELU implementations are equivalent to the native PyTorch
49
# implementations, so we can do exact comparison.
50
assert torch.allclose(out, ref_out, atol=0.0, rtol=0.0)
@@ -72,7 +72,7 @@ def test_activation(
72
x = torch.randn(num_tokens, d, dtype=dtype)
73
layer = activation()
74
out = layer(x)
75
- ref_out = layer._forward(x)
75
+ ref_out = layer.forward_native(x)
76
assert torch.allclose(out,
77
ref_out,
78
atol=get_default_atol(out),
tests/kernels/test_layernorm.py
+1
-1
@@ -42,7 +42,7 @@ def test_rms_norm(
42
43
# NOTE(woosuk): The reference implementation should be executed first
44
# because the custom kernel is in-place.
45
- ref_out = layer._forward(x, residual)
45
+ ref_out = layer.forward_native(x, residual)
46
out = layer(x, residual)
47
# NOTE(woosuk): LayerNorm operators (including RMS) typically have larger
48
# numerical errors than other operators because they involve reductions.
tests/kernels/test_pos_encoding.py
+4
-3
@@ -64,7 +64,7 @@ def test_rotary_embedding(
64
65
# NOTE(woosuk): The reference implementation should be executed first
66
# because the custom kernel is in-place.
67
- ref_query, ref_key = rope._forward(positions, query, key)
67
+ ref_query, ref_key = rope.forward_native(positions, query, key)
68
out_query, out_key = rope.forward(positions, query, key)
69
# Compare the results.
70
assert torch.allclose(out_query,
@@ -121,7 +121,7 @@ def test_batched_rotary_embedding(
121
122
# NOTE(woosuk): The reference implementation should be executed first
123
# because the custom kernel is in-place.
124
- ref_query, ref_key = rope._forward(positions, query, key)
124
+ ref_query, ref_key = rope.forward_native(positions, query, key)
125
out_query, out_key = rope.forward(positions,
126
query,
127
key,
@@ -195,7 +195,8 @@ def test_batched_rotary_embedding_multi_lora(
195
196
# NOTE(woosuk): The reference implementation should be executed first
197
# because the custom kernel is in-place.
198
- ref_query, ref_key = rope._forward(positions, query, key, query_offsets)
198
+ ref_query, ref_key = rope.forward_native(positions, query, key,
199
+ query_offsets)
200
out_query, out_key = rope.forward(positions, query, key,
201
query_offsets.flatten())
202
# Compare the results.
vllm/model_executor/custom_op.py
new
+60
@@ -0,0 +1,60 @@
1
+import torch.nn as nn
2
+
3
+from vllm.utils import is_cpu, is_hip
4
+
5
+
6
+class CustomOp(nn.Module):
7
+
8
+ def __init__(self, *args, **kwargs):
9
+ super().__init__()
10
+ self._forward_method = self.dispatch_forward()
11
+
12
+ def forward(self, *args, **kwargs):
13
+ return self._forward_method(*args, **kwargs)
14
+
15
+ def forward_native(self, *args, **kwargs):
16
+ """PyTorch-native implementation of the forward method.
17
+
18
+ This method is optional. If implemented, it can be used with compilers
19
+ such as torch.compile or PyTorch XLA. Also, it can be used for testing
20
+ purposes.
21
+ """
22
+ raise NotImplementedError
23
+
24
+ def forward_cuda(self, *args, **kwargs):
25
+ raise NotImplementedError
26
+
27
+ def forward_hip(self, *args, **kwargs):
28
+ # By default, we assume that HIP ops are compatible with CUDA ops.
29
+ return self.forward_cuda(*args, **kwargs)
30
+
31
+ def forward_xpu(self, *args, **kwargs):
32
+ # By default, we assume that XPU ops are compatible with CUDA ops.
33
+ # NOTE(woosuk): This is a placeholder for future extensions.
34
+ return self.forward_cuda(*args, **kwargs)
35
+
36
+ def forward_cpu(self, *args, **kwargs):
37
+ # By default, we assume that CPU ops are compatible with CUDA ops.
38
+ return self.forward_cuda(*args, **kwargs)
39
+
40
+ def forward_tpu(self, *args, **kwargs):
41
+ # By default, we assume that TPU ops are compatible with the
42
+ # PyTorch-native implementation.
43
+ # NOTE(woosuk): This is a placeholder for future extensions.
44
+ return self.forward_native(*args, **kwargs)
45
+
46
+ def forward_gaudi(self, *args, **kwargs):
47
+ # By default, we assume that Gaudi ops are compatible with the
48
+ # PyTorch-native implementation.
49
+ # NOTE(woosuk): This is a placeholder for future extensions.
50
+ return self.forward_native(*args, **kwargs)
51
+
52
+ def dispatch_forward(self):
53
+ # NOTE(woosuk): Here we assume that vLLM was built for only one
54
+ # specific backend. Currently, we do not support dynamic dispatching.
55
+ if is_hip():
56
+ return self.forward_hip
57
+ elif is_cpu():
58
+ return self.forward_cpu
59
+ else:
60
+ return self.forward_cuda
vllm/model_executor/layers/activation.py
+21
-13
@@ -6,14 +6,14 @@ import torch
6
import torch.nn as nn
7
import torch.nn.functional as F
8
9
-from vllm import _custom_ops as ops
9
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10
get_tensor_model_parallel_world_size)
11
+from vllm.model_executor.custom_op import CustomOp
12
from vllm.model_executor.layers.quantization import QuantizationConfig
13
from vllm.model_executor.utils import set_weight_attrs
14
15
16
-class SiluAndMul(nn.Module):
16
+class SiluAndMul(CustomOp):
17
"""An activation function for SwiGLU.
18
19
The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2.
@@ -23,12 +23,14 @@ class SiluAndMul(nn.Module):
23
return: (num_tokens, d) or (batch_size, seq_len, d)
24
"""
25
26
- def _forward(self, x: torch.Tensor) -> torch.Tensor:
26
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
27
"""PyTorch-native implementation equivalent to forward()."""
28
d = x.shape[-1] // 2
29
return F.silu(x[..., :d]) * x[..., d:]
30
31
- def forward(self, x: torch.Tensor) -> torch.Tensor:
31
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
32
+ from vllm import _custom_ops as ops
33
+
34
d = x.shape[-1] // 2
35
output_shape = (x.shape[:-1] + (d, ))
36
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
@@ -36,7 +38,7 @@ class SiluAndMul(nn.Module):
38
return out
39
40
39
-class GeluAndMul(nn.Module):
41
+class GeluAndMul(CustomOp):
42
"""An activation function for GeGLU.
43
44
The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2.
@@ -52,12 +54,14 @@ class GeluAndMul(nn.Module):
54
if approximate not in ("none", "tanh"):
55
raise ValueError(f"Unknown approximate mode: {approximate}")
56
55
- def _forward(self, x: torch.Tensor) -> torch.Tensor:
57
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
58
"""PyTorch-native implementation equivalent to forward()."""
59
d = x.shape[-1] // 2
60
return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:]
61
60
- def forward(self, x: torch.Tensor) -> torch.Tensor:
62
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
63
+ from vllm import _custom_ops as ops
64
+
65
d = x.shape[-1] // 2
66
output_shape = (x.shape[:-1] + (d, ))
67
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
@@ -71,28 +75,32 @@ class GeluAndMul(nn.Module):
75
return f'approximate={repr(self.approximate)}'
76
77
74
-class NewGELU(nn.Module):
78
+class NewGELU(CustomOp):
79
76
- def _forward(self, x: torch.Tensor) -> torch.Tensor:
80
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
81
"""PyTorch-native implementation equivalent to forward()."""
82
c = math.sqrt(2.0 / math.pi)
83
return 0.5 * x * (1.0 + torch.tanh(c *
84
(x + 0.044715 * torch.pow(x, 3.0))))
85
82
- def forward(self, x: torch.Tensor) -> torch.Tensor:
86
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
87
+ from vllm import _custom_ops as ops
88
+
89
out = torch.empty_like(x)
90
ops.gelu_new(out, x)
91
return out
92
93
88
-class FastGELU(nn.Module):
94
+class FastGELU(CustomOp):
95
90
- def _forward(self, x: torch.Tensor) -> torch.Tensor:
96
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
97
"""PyTorch-native implementation equivalent to forward()."""
98
return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 *
99
(1.0 + 0.044715 * x * x)))
100
95
- def forward(self, x: torch.Tensor) -> torch.Tensor:
101
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
102
+ from vllm import _custom_ops as ops
103
+
104
out = torch.empty_like(x)
105
ops.gelu_fast(out, x)
106
return out
vllm/model_executor/layers/layernorm.py
+6
-4
@@ -4,10 +4,10 @@ from typing import Optional, Tuple, Union
4
import torch
5
import torch.nn as nn
6
7
-from vllm import _custom_ops as ops
7
+from vllm.model_executor.custom_op import CustomOp
8
9
10
-class RMSNorm(nn.Module):
10
+class RMSNorm(CustomOp):
11
"""Root mean square normalization.
12
13
Computes x -> w * x / sqrt(E[x^2] + eps) where w is the learned weight.
@@ -23,7 +23,7 @@ class RMSNorm(nn.Module):
23
self.weight = nn.Parameter(torch.ones(hidden_size))
24
self.variance_epsilon = eps
25
26
- def _forward(
26
+ def forward_native(
27
self,
28
x: torch.Tensor,
29
residual: Optional[torch.Tensor] = None,
@@ -43,11 +43,13 @@ class RMSNorm(nn.Module):
43
else:
44
return x, residual
45
46
- def forward(
46
+ def forward_cuda(
47
self,
48
x: torch.Tensor,
49
residual: Optional[torch.Tensor] = None,
50
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
51
+ from vllm import _custom_ops as ops
52
+
53
if residual is not None:
54
ops.fused_add_rms_norm(
55
x,
vllm/model_executor/layers/rotary_embedding.py
+6
-4
@@ -27,7 +27,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
27
import torch
28
import torch.nn as nn
29
30
-from vllm import _custom_ops as ops
30
+from vllm.model_executor.custom_op import CustomOp
31
32
33
def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
@@ -43,7 +43,7 @@ def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
43
return x.flatten(-2)
44
45
46
-class RotaryEmbedding(nn.Module):
46
+class RotaryEmbedding(CustomOp):
47
"""Original rotary positional embedding."""
48
49
def __init__(
@@ -93,7 +93,7 @@ class RotaryEmbedding(nn.Module):
93
cache = torch.cat((cos, sin), dim=-1)
94
return cache
95
96
- def _forward(
96
+ def forward_native(
97
self,
98
positions: torch.Tensor,
99
query: torch.Tensor,
@@ -138,13 +138,15 @@ class RotaryEmbedding(nn.Module):
138
key = key.flatten(-2)
139
return query, key
140
141
- def forward(
141
+ def forward_cuda(
142
self,
143
positions: torch.Tensor,
144
query: torch.Tensor,
145
key: torch.Tensor,
146
offsets: Optional[torch.Tensor] = None,
147
) -> Tuple[torch.Tensor, torch.Tensor]:
148
+ from vllm import _custom_ops as ops
149
+
150
self.cos_sin_cache = self.cos_sin_cache.to(positions.device,
151
dtype=query.dtype)
152
# ops.rotary_embedding()/batched_rotary_embedding()