@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b294976a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add PyTorch-native implementation of custom layers (#1898)

Woosuk Kwon committed Dec 2, 2023 at 21:18 UTC 9b294976a2373f6fda22c1b2e704c395c8bd0787
6 files changed +150 -185
tests/kernels/test_activation.py
+10 -17
@@ -1,9 +1,7 @@
1 import pytest
2 import torch
3 -import torch.nn.functional as F
4 -from transformers.activations import get_activation
3
6 -from vllm._C import ops
4 +from vllm.model_executor.layers.activation import FastGELU, NewGELU, SiluAndMul
5
6 DTYPES = [torch.half, torch.bfloat16, torch.float]
7 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
@@ -11,11 +9,6 @@ D = [512, 4096, 5120, 13824] # Arbitrary values for testing
9 SEEDS = [0]
10
11
14 -def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
15 - x1, x2 = x.chunk(chunks=2, dim=1)
16 - return F.silu(x1) * x2
17 -
18 -
12 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
13 @pytest.mark.parametrize("d", D)
14 @pytest.mark.parametrize("dtype", DTYPES)
@@ -30,9 +23,9 @@ def test_silu_and_mul(
23 torch.random.manual_seed(seed)
24 torch.cuda.manual_seed(seed)
25 x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
33 - out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
34 - ops.silu_and_mul(out, x)
35 - ref_out = ref_silu_and_mul(x)
26 + layer = SiluAndMul()
27 + out = layer(x)
28 + ref_out = layer._forward(x)
29 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
30
31
@@ -50,9 +43,9 @@ def test_gelu_new(
43 torch.random.manual_seed(seed)
44 torch.cuda.manual_seed(seed)
45 x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
53 - out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
54 - ops.gelu_new(out, x)
55 - ref_out = get_activation("gelu_new")(x)
46 + layer = NewGELU()
47 + out = layer(x)
48 + ref_out = layer._forward(x)
49 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
50
51
@@ -69,7 +62,7 @@ def test_gelu_fast(
62 torch.random.manual_seed(seed)
63 torch.cuda.manual_seed(seed)
64 x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
72 - out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
73 - ops.gelu_fast(out, x)
74 - ref_out = get_activation("gelu_fast")(x)
65 + layer = FastGELU()
66 + out = layer(x)
67 + ref_out = layer._forward(x)
68 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
tests/kernels/test_layernorm.py
+24 -35
@@ -1,58 +1,47 @@
1 import pytest
2 import torch
3 -import torch.nn as nn
3
5 -from vllm._C import ops
4 +from vllm.model_executor.layers.layernorm import RMSNorm
5
6 DTYPES = [torch.half, torch.bfloat16, torch.float]
8 -HIDDEN_SIZES = [67, 768, 2048, 5120, 8192] # Arbitrary values for testing
7 NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
8 +HIDDEN_SIZES = [768, 5120, 8192] # Arbitrary values for testing
9 +ADD_RESIDUAL = [False, True]
10 SEEDS = [0]
11
12
13 -class RefRMSNorm(nn.Module):
14 -
15 - def __init__(self, hidden_size, eps=1e-6):
16 - super().__init__()
17 - weight = torch.empty(hidden_size)
18 - weight.normal_(mean=1.0, std=0.1)
19 - self.weight = nn.Parameter(weight)
20 - self.variance_epsilon = eps
21 -
22 - def forward(self, hidden_states):
23 - input_dtype = hidden_states.dtype
24 - hidden_states = hidden_states.to(torch.float32)
25 - variance = hidden_states.pow(2).mean(-1, keepdim=True)
26 - hidden_states = hidden_states * torch.rsqrt(variance +
27 - self.variance_epsilon)
28 - return self.weight * hidden_states.to(input_dtype)
29 -
30 -
13 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
14 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
15 +@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
16 @pytest.mark.parametrize("dtype", DTYPES)
17 @pytest.mark.parametrize("seed", SEEDS)
18 @torch.inference_mode()
19 def test_rms_norm(
20 num_tokens: int,
21 hidden_size: int,
22 + add_residual: bool,
23 dtype: torch.dtype,
24 seed: int,
25 ) -> None:
26 torch.random.manual_seed(seed)
27 torch.cuda.manual_seed(seed)
28
45 - scale = float(hidden_size**-0.5)
46 - x = torch.empty(num_tokens, hidden_size, dtype=dtype, device="cuda")
47 - x.uniform_(-scale, scale)
48 - ref = RefRMSNorm(hidden_size).to(dtype).cuda()
49 -
50 - out = torch.empty_like(x)
51 - ops.rms_norm(
52 - out,
53 - x,
54 - ref.weight.data,
55 - ref.variance_epsilon,
56 - )
57 - ref_out = ref(x)
58 - assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-5)
29 + layer = RMSNorm(hidden_size).to(dtype).cuda()
30 + layer.weight.data.normal_(mean=1.0, std=0.1)
31 + scale = 1 / (2 * hidden_size)
32 + x = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda")
33 + x *= scale
34 + residual = torch.randn_like(x) * scale if add_residual else None
35 +
36 + # NOTE(woosuk): The reference implementation should be executed first
37 + # because the custom kernel is in-place.
38 + ref_out = layer._forward(x, residual)
39 + out = layer(x, residual)
40 + # NOTE(woosuk): LayerNorm operators (including RMS) typically have larger
41 + # numerical errors than other operators because they involve reductions.
42 + # Therefore, we use a larger tolerance.
43 + if add_residual:
44 + assert torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
45 + assert torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
46 + else:
47 + assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)
tests/kernels/test_pos_encoding.py
+24 -133
@@ -1,105 +1,23 @@
1 -from typing import Optional, Tuple
1 +from typing import Optional
2
3 import pytest
4 import torch
5 -import torch.nn as nn
6 -import torch.nn.functional as F
5
8 -from vllm._C import ops
6 +from vllm.model_executor.layers.rotary_embedding import get_rope
7
8 IS_NEOX_STYLE = [True, False]
9 DTYPES = [torch.half, torch.bfloat16, torch.float]
10 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
11 ROTARY_DIMS = [None, 32] # None means rotary dim == head size
14 -NUM_HEADS = [7, 12, 40, 52] # Arbitrary values for testing
15 -NUM_TOKENS = [11, 83, 2048] # Arbitrary values for testing
12 +NUM_HEADS = [7, 17] # Arbitrary values for testing
13 +BATCH_SIZES = [1, 5] # Arbitrary values for testing
14 +SEQ_LENS = [11, 8192] # Arbitrary values for testing
15 SEEDS = [0]
16
17
19 -def rotate_neox(x: torch.Tensor) -> torch.Tensor:
20 - x1 = x[..., :x.shape[-1] // 2]
21 - x2 = x[..., x.shape[-1] // 2:]
22 - return torch.cat((-x2, x1), dim=-1)
23 -
24 -
25 -def rotate_gptj(x: torch.Tensor) -> torch.Tensor:
26 - x1 = x[..., ::2]
27 - x2 = x[..., 1::2]
28 - x = torch.stack((-x2, x1), dim=-1)
29 - return x.flatten(-2)
30 -
31 -
32 -def apply_rope(
33 - q: torch.Tensor,
34 - k: torch.Tensor,
35 - cos: torch.Tensor,
36 - sin: torch.Tensor,
37 - is_neox_style: bool,
38 -) -> Tuple[torch.Tensor, torch.Tensor]:
39 - rotate_fn = rotate_neox if is_neox_style else rotate_gptj
40 - q_embed = (q * cos) + (rotate_fn(q) * sin)
41 - k_embed = (k * cos) + (rotate_fn(k) * sin)
42 - return q_embed, k_embed
43 -
44 -
45 -class RefRotaryEmbedding(nn.Module):
46 - """Reference implementation of rotary embedding."""
47 -
48 - def __init__(
49 - self,
50 - dim: int,
51 - is_neox_style: bool,
52 - max_position_embeddings: int = 8192,
53 - base: int = 10000,
54 - ) -> None:
55 - super().__init__()
56 - self.rotary_dim = dim
57 - self.is_neox_style = is_neox_style
58 - self.max_position_embeddings = max_position_embeddings
59 -
60 - # Create cos and sin embeddings.
61 - inv_freq = 1.0 / (base**(torch.arange(0, dim, 2) / dim))
62 - t = torch.arange(max_position_embeddings).float()
63 - freqs = torch.einsum("i,j->ij", t, inv_freq.float())
64 - if is_neox_style:
65 - emb = torch.cat((freqs, freqs), dim=-1)
66 - else:
67 - emb = torch.repeat_interleave(freqs, 2, -1)
68 - cos = emb.cos().to(dtype=inv_freq.dtype)
69 - sin = emb.sin().to(dtype=inv_freq.dtype)
70 - self.register_buffer("cos_cached", cos, persistent=False)
71 - self.register_buffer("sin_cached", sin, persistent=False)
72 -
73 - def forward(
74 - self,
75 - positions: torch.Tensor, # [num_tokens]
76 - query: torch.Tensor, # [num_tokens, num_heads, head_size]
77 - key: torch.Tensor, # [num_tokens, num_heads, head_size]
78 - ) -> Tuple[torch.Tensor, torch.Tensor]:
79 - query_rot = query[..., :self.rotary_dim]
80 - query_pass = query[..., self.rotary_dim:]
81 - key_rot = key[..., :self.rotary_dim]
82 - key_pass = key[..., self.rotary_dim:]
83 -
84 - query_rot = query_rot.transpose(0, 1)
85 - key_rot = key_rot.transpose(0, 1)
86 - cos = F.embedding(positions, self.cos_cached)
87 - sin = F.embedding(positions, self.sin_cached)
88 -
89 - query_rot, key_rot = apply_rope(query_rot, key_rot, cos, sin,
90 - self.is_neox_style)
91 - query_rot = query_rot.transpose(0, 1).contiguous()
92 - key_rot = key_rot.transpose(0, 1).contiguous()
93 -
94 - query = torch.cat((query_rot, query_pass), dim=-1)
95 - key = torch.cat((key_rot, key_pass), dim=-1)
96 -
97 - # Output query/key shape: [num_tokens, num_tokens, head_size]
98 - return query, key
99 -
100 -
18 @pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
102 -@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
19 +@pytest.mark.parametrize("batch_size", BATCH_SIZES)
20 +@pytest.mark.parametrize("seq_len", SEQ_LENS)
21 @pytest.mark.parametrize("num_heads", NUM_HEADS)
22 @pytest.mark.parametrize("head_size", HEAD_SIZES)
23 @pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
@@ -108,7 +26,8 @@ class RefRotaryEmbedding(nn.Module):
26 @torch.inference_mode()
27 def test_rotary_embedding(
28 is_neox_style: bool,
111 - num_tokens: int,
29 + batch_size: int,
30 + seq_len: int,
31 num_heads: int,
32 head_size: int,
33 rotary_dim: Optional[int],
@@ -122,53 +41,25 @@ def test_rotary_embedding(
41 torch.random.manual_seed(seed)
42 torch.cuda.manual_seed(seed)
43
125 - positions = torch.randint(0, max_position, (num_tokens, ), device="cuda")
126 - query = torch.randn(num_tokens,
44 + if rotary_dim is None:
45 + rotary_dim = head_size
46 + rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
47 + rope = rope.to(dtype).cuda()
48 +
49 + positions = torch.randint(0,
50 + max_position, (batch_size, seq_len),
51 + device="cuda")
52 + query = torch.randn(batch_size,
53 + seq_len,
54 num_heads * head_size,
55 dtype=dtype,
56 device="cuda")
130 - key = torch.randn(num_tokens,
131 - num_heads * head_size,
132 - dtype=dtype,
133 - device="cuda")
134 -
135 - # Create the rotary embedding.
136 - inv_freq = 1.0 / (base**(
137 - torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim))
138 - t = torch.arange(max_position).float()
139 - freqs = torch.einsum("i,j -> ij", t, inv_freq)
140 - cos = freqs.cos()
141 - sin = freqs.sin()
142 - cos_sin_cache = torch.cat((cos, sin), dim=-1)
143 - cos_sin_cache = cos_sin_cache.to(dtype=dtype, device="cuda")
144 -
145 - # Run the kernel. The kernel is in-place, so we need to clone the inputs.
146 - out_query = query.clone()
147 - out_key = key.clone()
148 - ops.rotary_embedding(
149 - positions,
150 - out_query,
151 - out_key,
152 - head_size,
153 - cos_sin_cache,
154 - is_neox_style,
155 - )
156 -
157 - # Run the reference implementation.
158 - ref_rotary_embedding = RefRotaryEmbedding(
159 - dim=rotary_dim,
160 - is_neox_style=is_neox_style,
161 - max_position_embeddings=max_position,
162 - base=base,
163 - ).to(dtype=dtype, device="cuda")
164 - ref_query, ref_key = ref_rotary_embedding(
165 - positions,
166 - query.view(num_tokens, num_heads, head_size),
167 - key.view(num_tokens, num_heads, head_size),
168 - )
169 - ref_query = ref_query.view(num_tokens, num_heads * head_size)
170 - ref_key = ref_key.view(num_tokens, num_heads * head_size)
57 + key = torch.randn_like(query)
58
59 + # NOTE(woosuk): The reference implementation should be executed first
60 + # because the custom kernel is in-place.
61 + ref_query, ref_key = rope._forward(positions, query, key)
62 + out_query, out_key = rope.forward(positions, query, key)
63 # Compare the results.
64 assert torch.allclose(out_query, ref_query, atol=1e-5, rtol=1e-5)
65 assert torch.allclose(out_key, ref_key, atol=1e-5, rtol=1e-5)
vllm/model_executor/layers/activation.py
+18
@@ -1,8 +1,10 @@
1 """Custom activation functions."""
2 +import math
3 from typing import Optional
4
5 import torch
6 import torch.nn as nn
7 +import torch.nn.functional as F
8
9 from vllm._C import ops
10 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -22,6 +24,11 @@ class SiluAndMul(nn.Module):
24 return: (batch_size, seq_len, d) or (num_tokens, d)
25 """
26
27 + def _forward(self, x: torch.Tensor) -> torch.Tensor:
28 + """PyTorch-native implementation equivalent to forward()."""
29 + d = x.shape[-1] // 2
30 + return F.silu(x[..., :d]) * x[..., d:]
31 +
32 def forward(self, x: torch.Tensor) -> torch.Tensor:
33 d = x.shape[-1] // 2
34 output_shape = (x.shape[:-1] + (d, ))
@@ -32,6 +39,12 @@ class SiluAndMul(nn.Module):
39
40 class NewGELU(nn.Module):
41
42 + def _forward(self, x: torch.Tensor) -> torch.Tensor:
43 + """PyTorch-native implementation equivalent to forward()."""
44 + c = math.sqrt(2.0 / math.pi)
45 + return 0.5 * x * (1.0 + torch.tanh(c *
46 + (x + 0.044715 * torch.pow(x, 3.0))))
47 +
48 def forward(self, x: torch.Tensor) -> torch.Tensor:
49 out = torch.empty_like(x)
50 ops.gelu_new(out, x)
@@ -40,6 +53,11 @@ class NewGELU(nn.Module):
53
54 class FastGELU(nn.Module):
55
56 + def _forward(self, x: torch.Tensor) -> torch.Tensor:
57 + """PyTorch-native implementation equivalent to forward()."""
58 + return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 *
59 + (1.0 + 0.044715 * x * x)))
60 +
61 def forward(self, x: torch.Tensor) -> torch.Tensor:
62 out = torch.empty_like(x)
63 ops.gelu_fast(out, x)
vllm/model_executor/layers/layernorm.py
+20
@@ -23,6 +23,26 @@ class RMSNorm(nn.Module):
23 self.weight = nn.Parameter(torch.ones(hidden_size))
24 self.variance_epsilon = eps
25
26 + def _forward(
27 + self,
28 + x: torch.Tensor,
29 + residual: Optional[torch.Tensor] = None,
30 + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
31 + """PyTorch-native implementation equivalent to forward()."""
32 + orig_dtype = x.dtype
33 + x = x.to(torch.float32)
34 + if residual is not None:
35 + x = x + residual.to(torch.float32)
36 + residual = x.to(orig_dtype)
37 +
38 + variance = x.pow(2).mean(dim=-1, keepdim=True)
39 + x = x * torch.rsqrt(variance + self.variance_epsilon)
40 + x = x.to(orig_dtype) * self.weight
41 + if residual is None:
42 + return x
43 + else:
44 + return x, residual
45 +
46 def forward(
47 self,
48 x: torch.Tensor,
vllm/model_executor/layers/rotary_embedding.py
+54
@@ -30,6 +30,19 @@ import torch.nn as nn
30 from vllm._C import ops
31
32
33 +def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
34 + x1 = x[..., :x.shape[-1] // 2]
35 + x2 = x[..., x.shape[-1] // 2:]
36 + return torch.cat((-x2, x1), dim=-1)
37 +
38 +
39 +def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
40 + x1 = x[..., ::2]
41 + x2 = x[..., 1::2]
42 + x = torch.stack((-x2, x1), dim=-1)
43 + return x.flatten(-2)
44 +
45 +
46 class RotaryEmbedding(nn.Module):
47 """Original rotary positional embedding."""
48
@@ -81,6 +94,47 @@ class RotaryEmbedding(nn.Module):
94 cache = torch.cat((cos, sin), dim=-1)
95 return cache
96
97 + def _forward(
98 + self,
99 + positions: torch.Tensor,
100 + query: torch.Tensor,
101 + key: torch.Tensor,
102 + ) -> Tuple[torch.Tensor, torch.Tensor]:
103 + """PyTorch-native implementation equivalent to forward()."""
104 + query = query.view(*query.shape[:-1], -1, self.head_size)
105 + key = key.view(*key.shape[:-1], -1, self.head_size)
106 +
107 + query_rot = query[..., :self.rotary_dim]
108 + key_rot = key[..., :self.rotary_dim]
109 + if self.rotary_dim < self.head_size:
110 + query_pass = query[..., self.rotary_dim:]
111 + key_pass = key[..., self.rotary_dim:]
112 +
113 + cos_sin = self.cos_sin_cache[positions]
114 + cos, sin = cos_sin.chunk(2, dim=-1)
115 + if self.is_neox_style:
116 + # NOTE(woosuk): Here we assume that the positions tensor has the
117 + # shape [batch_size, seq_len].
118 + cos = cos.repeat(1, 1, 2).unsqueeze(-2)
119 + sin = sin.repeat(1, 1, 2).unsqueeze(-2)
120 + else:
121 + cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
122 + sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
123 +
124 + rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj
125 + query_rot = query_rot * cos + rotate_fn(query_rot) * sin
126 + key_rot = key_rot * cos + rotate_fn(key_rot) * sin
127 +
128 + if self.rotary_dim < self.head_size:
129 + query = torch.cat((query_rot, query_pass), dim=-1)
130 + key = torch.cat((key_rot, key_pass), dim=-1)
131 + else:
132 + query = query_rot
133 + key = key_rot
134 + query = query.flatten(-2)
135 + key = key.flatten(-2)
136 + return query, key
137 +
138 def forward(
139 self,
140 positions: torch.Tensor,