A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add PyTorch-native implementation of custom layers (#1898)
Woosuk Kwon committed
Dec 2, 2023 at 21:18 UTC
9b294976a2373f6fda22c1b2e704c395c8bd0787
6 files changed
+150
-185
tests/kernels/test_activation.py
+10
-17
@@ -1,9 +1,7 @@
1
import pytest
2
import torch
3
-import torch.nn.functional as F
4
-from transformers.activations import get_activation
3
6
-from vllm._C import ops
4
+from vllm.model_executor.layers.activation import FastGELU, NewGELU, SiluAndMul
5
6
DTYPES = [torch.half, torch.bfloat16, torch.float]
7
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
@@ -11,11 +9,6 @@ D = [512, 4096, 5120, 13824] # Arbitrary values for testing
9
SEEDS = [0]
10
11
14
-def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
15
- x1, x2 = x.chunk(chunks=2, dim=1)
16
- return F.silu(x1) * x2
17
-
18
-
12
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
13
@pytest.mark.parametrize("d", D)
14
@pytest.mark.parametrize("dtype", DTYPES)
@@ -30,9 +23,9 @@ def test_silu_and_mul(
23
torch.random.manual_seed(seed)
24
torch.cuda.manual_seed(seed)
25
x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
33
- out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
34
- ops.silu_and_mul(out, x)
35
- ref_out = ref_silu_and_mul(x)
26
+ layer = SiluAndMul()
27
+ out = layer(x)
28
+ ref_out = layer._forward(x)
29
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
30
31
@@ -50,9 +43,9 @@ def test_gelu_new(
43
torch.random.manual_seed(seed)
44
torch.cuda.manual_seed(seed)
45
x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
53
- out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
54
- ops.gelu_new(out, x)
55
- ref_out = get_activation("gelu_new")(x)
46
+ layer = NewGELU()
47
+ out = layer(x)
48
+ ref_out = layer._forward(x)
49
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
50
51
@@ -69,7 +62,7 @@ def test_gelu_fast(
62
torch.random.manual_seed(seed)
63
torch.cuda.manual_seed(seed)
64
x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
72
- out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
73
- ops.gelu_fast(out, x)
74
- ref_out = get_activation("gelu_fast")(x)
65
+ layer = FastGELU()
66
+ out = layer(x)
67
+ ref_out = layer._forward(x)
68
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
tests/kernels/test_layernorm.py
+24
-35
@@ -1,58 +1,47 @@
1
import pytest
2
import torch
3
-import torch.nn as nn
3
5
-from vllm._C import ops
4
+from vllm.model_executor.layers.layernorm import RMSNorm
5
6
DTYPES = [torch.half, torch.bfloat16, torch.float]
8
-HIDDEN_SIZES = [67, 768, 2048, 5120, 8192] # Arbitrary values for testing
7
NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
8
+HIDDEN_SIZES = [768, 5120, 8192] # Arbitrary values for testing
9
+ADD_RESIDUAL = [False, True]
10
SEEDS = [0]
11
12
13
-class RefRMSNorm(nn.Module):
14
-
15
- def __init__(self, hidden_size, eps=1e-6):
16
- super().__init__()
17
- weight = torch.empty(hidden_size)
18
- weight.normal_(mean=1.0, std=0.1)
19
- self.weight = nn.Parameter(weight)
20
- self.variance_epsilon = eps
21
-
22
- def forward(self, hidden_states):
23
- input_dtype = hidden_states.dtype
24
- hidden_states = hidden_states.to(torch.float32)
25
- variance = hidden_states.pow(2).mean(-1, keepdim=True)
26
- hidden_states = hidden_states * torch.rsqrt(variance +
27
- self.variance_epsilon)
28
- return self.weight * hidden_states.to(input_dtype)
29
-
30
-
13
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
14
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
15
+@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
16
@pytest.mark.parametrize("dtype", DTYPES)
17
@pytest.mark.parametrize("seed", SEEDS)
18
@torch.inference_mode()
19
def test_rms_norm(
20
num_tokens: int,
21
hidden_size: int,
22
+ add_residual: bool,
23
dtype: torch.dtype,
24
seed: int,
25
) -> None:
26
torch.random.manual_seed(seed)
27
torch.cuda.manual_seed(seed)
28
45
- scale = float(hidden_size**-0.5)
46
- x = torch.empty(num_tokens, hidden_size, dtype=dtype, device="cuda")
47
- x.uniform_(-scale, scale)
48
- ref = RefRMSNorm(hidden_size).to(dtype).cuda()
49
-
50
- out = torch.empty_like(x)
51
- ops.rms_norm(
52
- out,
53
- x,
54
- ref.weight.data,
55
- ref.variance_epsilon,
56
- )
57
- ref_out = ref(x)
58
- assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-5)
29
+ layer = RMSNorm(hidden_size).to(dtype).cuda()
30
+ layer.weight.data.normal_(mean=1.0, std=0.1)
31
+ scale = 1 / (2 * hidden_size)
32
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda")
33
+ x *= scale
34
+ residual = torch.randn_like(x) * scale if add_residual else None
35
+
36
+ # NOTE(woosuk): The reference implementation should be executed first
37
+ # because the custom kernel is in-place.
38
+ ref_out = layer._forward(x, residual)
39
+ out = layer(x, residual)
40
+ # NOTE(woosuk): LayerNorm operators (including RMS) typically have larger
41
+ # numerical errors than other operators because they involve reductions.
42
+ # Therefore, we use a larger tolerance.
43
+ if add_residual:
44
+ assert torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
45
+ assert torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
46
+ else:
47
+ assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)
tests/kernels/test_pos_encoding.py
+24
-133
@@ -1,105 +1,23 @@
1
-from typing import Optional, Tuple
1
+from typing import Optional
2
3
import pytest
4
import torch
5
-import torch.nn as nn
6
-import torch.nn.functional as F
5
8
-from vllm._C import ops
6
+from vllm.model_executor.layers.rotary_embedding import get_rope
7
8
IS_NEOX_STYLE = [True, False]
9
DTYPES = [torch.half, torch.bfloat16, torch.float]
10
HEAD_SIZES = [64, 80, 96, 112, 128, 256]
11
ROTARY_DIMS = [None, 32] # None means rotary dim == head size
14
-NUM_HEADS = [7, 12, 40, 52] # Arbitrary values for testing
15
-NUM_TOKENS = [11, 83, 2048] # Arbitrary values for testing
12
+NUM_HEADS = [7, 17] # Arbitrary values for testing
13
+BATCH_SIZES = [1, 5] # Arbitrary values for testing
14
+SEQ_LENS = [11, 8192] # Arbitrary values for testing
15
SEEDS = [0]
16
17
19
-def rotate_neox(x: torch.Tensor) -> torch.Tensor:
20
- x1 = x[..., :x.shape[-1] // 2]
21
- x2 = x[..., x.shape[-1] // 2:]
22
- return torch.cat((-x2, x1), dim=-1)
23
-
24
-
25
-def rotate_gptj(x: torch.Tensor) -> torch.Tensor:
26
- x1 = x[..., ::2]
27
- x2 = x[..., 1::2]
28
- x = torch.stack((-x2, x1), dim=-1)
29
- return x.flatten(-2)
30
-
31
-
32
-def apply_rope(
33
- q: torch.Tensor,
34
- k: torch.Tensor,
35
- cos: torch.Tensor,
36
- sin: torch.Tensor,
37
- is_neox_style: bool,
38
-) -> Tuple[torch.Tensor, torch.Tensor]:
39
- rotate_fn = rotate_neox if is_neox_style else rotate_gptj
40
- q_embed = (q * cos) + (rotate_fn(q) * sin)
41
- k_embed = (k * cos) + (rotate_fn(k) * sin)
42
- return q_embed, k_embed
43
-
44
-
45
-class RefRotaryEmbedding(nn.Module):
46
- """Reference implementation of rotary embedding."""
47
-
48
- def __init__(
49
- self,
50
- dim: int,
51
- is_neox_style: bool,
52
- max_position_embeddings: int = 8192,
53
- base: int = 10000,
54
- ) -> None:
55
- super().__init__()
56
- self.rotary_dim = dim
57
- self.is_neox_style = is_neox_style
58
- self.max_position_embeddings = max_position_embeddings
59
-
60
- # Create cos and sin embeddings.
61
- inv_freq = 1.0 / (base**(torch.arange(0, dim, 2) / dim))
62
- t = torch.arange(max_position_embeddings).float()
63
- freqs = torch.einsum("i,j->ij", t, inv_freq.float())
64
- if is_neox_style:
65
- emb = torch.cat((freqs, freqs), dim=-1)
66
- else:
67
- emb = torch.repeat_interleave(freqs, 2, -1)
68
- cos = emb.cos().to(dtype=inv_freq.dtype)
69
- sin = emb.sin().to(dtype=inv_freq.dtype)
70
- self.register_buffer("cos_cached", cos, persistent=False)
71
- self.register_buffer("sin_cached", sin, persistent=False)
72
-
73
- def forward(
74
- self,
75
- positions: torch.Tensor, # [num_tokens]
76
- query: torch.Tensor, # [num_tokens, num_heads, head_size]
77
- key: torch.Tensor, # [num_tokens, num_heads, head_size]
78
- ) -> Tuple[torch.Tensor, torch.Tensor]:
79
- query_rot = query[..., :self.rotary_dim]
80
- query_pass = query[..., self.rotary_dim:]
81
- key_rot = key[..., :self.rotary_dim]
82
- key_pass = key[..., self.rotary_dim:]
83
-
84
- query_rot = query_rot.transpose(0, 1)
85
- key_rot = key_rot.transpose(0, 1)
86
- cos = F.embedding(positions, self.cos_cached)
87
- sin = F.embedding(positions, self.sin_cached)
88
-
89
- query_rot, key_rot = apply_rope(query_rot, key_rot, cos, sin,
90
- self.is_neox_style)
91
- query_rot = query_rot.transpose(0, 1).contiguous()
92
- key_rot = key_rot.transpose(0, 1).contiguous()
93
-
94
- query = torch.cat((query_rot, query_pass), dim=-1)
95
- key = torch.cat((key_rot, key_pass), dim=-1)
96
-
97
- # Output query/key shape: [num_tokens, num_tokens, head_size]
98
- return query, key
99
-
100
-
18
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
102
-@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
19
+@pytest.mark.parametrize("batch_size", BATCH_SIZES)
20
+@pytest.mark.parametrize("seq_len", SEQ_LENS)
21
@pytest.mark.parametrize("num_heads", NUM_HEADS)
22
@pytest.mark.parametrize("head_size", HEAD_SIZES)
23
@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
@@ -108,7 +26,8 @@ class RefRotaryEmbedding(nn.Module):
26
@torch.inference_mode()
27
def test_rotary_embedding(
28
is_neox_style: bool,
111
- num_tokens: int,
29
+ batch_size: int,
30
+ seq_len: int,
31
num_heads: int,
32
head_size: int,
33
rotary_dim: Optional[int],
@@ -122,53 +41,25 @@ def test_rotary_embedding(
41
torch.random.manual_seed(seed)
42
torch.cuda.manual_seed(seed)
43
125
- positions = torch.randint(0, max_position, (num_tokens, ), device="cuda")
126
- query = torch.randn(num_tokens,
44
+ if rotary_dim is None:
45
+ rotary_dim = head_size
46
+ rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
47
+ rope = rope.to(dtype).cuda()
48
+
49
+ positions = torch.randint(0,
50
+ max_position, (batch_size, seq_len),
51
+ device="cuda")
52
+ query = torch.randn(batch_size,
53
+ seq_len,
54
num_heads * head_size,
55
dtype=dtype,
56
device="cuda")
130
- key = torch.randn(num_tokens,
131
- num_heads * head_size,
132
- dtype=dtype,
133
- device="cuda")
134
-
135
- # Create the rotary embedding.
136
- inv_freq = 1.0 / (base**(
137
- torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
138
- t = torch.arange(max_position).float()
139
- freqs = torch.einsum("i,j -> ij", t, inv_freq)
140
- cos = freqs.cos()
141
- sin = freqs.sin()
142
- cos_sin_cache = torch.cat((cos, sin), dim=-1)
143
- cos_sin_cache = cos_sin_cache.to(dtype=dtype, device="cuda")
144
-
145
- # Run the kernel. The kernel is in-place, so we need to clone the inputs.
146
- out_query = query.clone()
147
- out_key = key.clone()
148
- ops.rotary_embedding(
149
- positions,
150
- out_query,
151
- out_key,
152
- head_size,
153
- cos_sin_cache,
154
- is_neox_style,
155
- )
156
-
157
- # Run the reference implementation.
158
- ref_rotary_embedding = RefRotaryEmbedding(
159
- dim=rotary_dim,
160
- is_neox_style=is_neox_style,
161
- max_position_embeddings=max_position,
162
- base=base,
163
- ).to(dtype=dtype, device="cuda")
164
- ref_query, ref_key = ref_rotary_embedding(
165
- positions,
166
- query.view(num_tokens, num_heads, head_size),
167
- key.view(num_tokens, num_heads, head_size),
168
- )
169
- ref_query = ref_query.view(num_tokens, num_heads * head_size)
170
- ref_key = ref_key.view(num_tokens, num_heads * head_size)
57
+ key = torch.randn_like(query)
58
59
+ # NOTE(woosuk): The reference implementation should be executed first
60
+ # because the custom kernel is in-place.
61
+ ref_query, ref_key = rope._forward(positions, query, key)
62
+ out_query, out_key = rope.forward(positions, query, key)
63
# Compare the results.
64
assert torch.allclose(out_query, ref_query, atol=1e-5, rtol=1e-5)
65
assert torch.allclose(out_key, ref_key, atol=1e-5, rtol=1e-5)
vllm/model_executor/layers/activation.py
+18
@@ -1,8 +1,10 @@
1
"""Custom activation functions."""
2
+import math
3
from typing import Optional
4
5
import torch
6
import torch.nn as nn
7
+import torch.nn.functional as F
8
9
from vllm._C import ops
10
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -22,6 +24,11 @@ class SiluAndMul(nn.Module):
24
return: (batch_size, seq_len, d) or (num_tokens, d)
25
"""
26
27
+ def _forward(self, x: torch.Tensor) -> torch.Tensor:
28
+ """PyTorch-native implementation equivalent to forward()."""
29
+ d = x.shape[-1] // 2
30
+ return F.silu(x[..., :d]) * x[..., d:]
31
+
32
def forward(self, x: torch.Tensor) -> torch.Tensor:
33
d = x.shape[-1] // 2
34
output_shape = (x.shape[:-1] + (d, ))
@@ -32,6 +39,12 @@ class SiluAndMul(nn.Module):
39
40
class NewGELU(nn.Module):
41
42
+ def _forward(self, x: torch.Tensor) -> torch.Tensor:
43
+ """PyTorch-native implementation equivalent to forward()."""
44
+ c = math.sqrt(2.0 / math.pi)
45
+ return 0.5 * x * (1.0 + torch.tanh(c *
46
+ (x + 0.044715 * torch.pow(x, 3.0))))
47
+
48
def forward(self, x: torch.Tensor) -> torch.Tensor:
49
out = torch.empty_like(x)
50
ops.gelu_new(out, x)
@@ -40,6 +53,11 @@ class NewGELU(nn.Module):
53
54
class FastGELU(nn.Module):
55
56
+ def _forward(self, x: torch.Tensor) -> torch.Tensor:
57
+ """PyTorch-native implementation equivalent to forward()."""
58
+ return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 *
59
+ (1.0 + 0.044715 * x * x)))
60
+
61
def forward(self, x: torch.Tensor) -> torch.Tensor:
62
out = torch.empty_like(x)
63
ops.gelu_fast(out, x)
vllm/model_executor/layers/layernorm.py
+20
@@ -23,6 +23,26 @@ class RMSNorm(nn.Module):
23
self.weight = nn.Parameter(torch.ones(hidden_size))
24
self.variance_epsilon = eps
25
26
+ def _forward(
27
+ self,
28
+ x: torch.Tensor,
29
+ residual: Optional[torch.Tensor] = None,
30
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
31
+ """PyTorch-native implementation equivalent to forward()."""
32
+ orig_dtype = x.dtype
33
+ x = x.to(torch.float32)
34
+ if residual is not None:
35
+ x = x + residual.to(torch.float32)
36
+ residual = x.to(orig_dtype)
37
+
38
+ variance = x.pow(2).mean(dim=-1, keepdim=True)
39
+ x = x * torch.rsqrt(variance + self.variance_epsilon)
40
+ x = x.to(orig_dtype) * self.weight
41
+ if residual is None:
42
+ return x
43
+ else:
44
+ return x, residual
45
+
46
def forward(
47
self,
48
x: torch.Tensor,
vllm/model_executor/layers/rotary_embedding.py
+54
@@ -30,6 +30,19 @@ import torch.nn as nn
30
from vllm._C import ops
31
32
33
+def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
34
+ x1 = x[..., :x.shape[-1] // 2]
35
+ x2 = x[..., x.shape[-1] // 2:]
36
+ return torch.cat((-x2, x1), dim=-1)
37
+
38
+
39
+def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
40
+ x1 = x[..., ::2]
41
+ x2 = x[..., 1::2]
42
+ x = torch.stack((-x2, x1), dim=-1)
43
+ return x.flatten(-2)
44
+
45
+
46
class RotaryEmbedding(nn.Module):
47
"""Original rotary positional embedding."""
48
@@ -81,6 +94,47 @@ class RotaryEmbedding(nn.Module):
94
cache = torch.cat((cos, sin), dim=-1)
95
return cache
96
97
+ def _forward(
98
+ self,
99
+ positions: torch.Tensor,
100
+ query: torch.Tensor,
101
+ key: torch.Tensor,
102
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
103
+ """PyTorch-native implementation equivalent to forward()."""
104
+ query = query.view(*query.shape[:-1], -1, self.head_size)
105
+ key = key.view(*key.shape[:-1], -1, self.head_size)
106
+
107
+ query_rot = query[..., :self.rotary_dim]
108
+ key_rot = key[..., :self.rotary_dim]
109
+ if self.rotary_dim < self.head_size:
110
+ query_pass = query[..., self.rotary_dim:]
111
+ key_pass = key[..., self.rotary_dim:]
112
+
113
+ cos_sin = self.cos_sin_cache[positions]
114
+ cos, sin = cos_sin.chunk(2, dim=-1)
115
+ if self.is_neox_style:
116
+ # NOTE(woosuk): Here we assume that the positions tensor has the
117
+ # shape [batch_size, seq_len].
118
+ cos = cos.repeat(1, 1, 2).unsqueeze(-2)
119
+ sin = sin.repeat(1, 1, 2).unsqueeze(-2)
120
+ else:
121
+ cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
122
+ sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
123
+
124
+ rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj
125
+ query_rot = query_rot * cos + rotate_fn(query_rot) * sin
126
+ key_rot = key_rot * cos + rotate_fn(key_rot) * sin
127
+
128
+ if self.rotary_dim < self.head_size:
129
+ query = torch.cat((query_rot, query_pass), dim=-1)
130
+ key = torch.cat((key_rot, key_pass), dim=-1)
131
+ else:
132
+ query = query_rot
133
+ key = key_rot
134
+ query = query.flatten(-2)
135
+ key = key.flatten(-2)
136
+ return query, key
137
+
138
def forward(
139
self,
140
positions: torch.Tensor,