A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] Add indirection layer for custom ops (#3913)
Kunshang Ji committed
Apr 11, 2024 at 03:26 UTC
e9da5a40c63ce7f8a85438d3c7d919b46e7939f5
14 files changed
+224
-32
benchmarks/kernels/benchmark_paged_attention.py
+1
-1
@@ -5,7 +5,7 @@ from typing import Optional
5
6
import torch
7
8
-from vllm._C import ops
8
+from vllm import _custom_ops as ops
9
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
10
11
NUM_BLOCKS = 1024
tests/kernels/test_attention.py
+3
-3
@@ -7,7 +7,7 @@ from allclose_default import get_default_atol, get_default_rtol
7
from xformers import ops as xops
8
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
9
10
-from vllm._C import cache_ops, ops
10
+from vllm import _custom_ops as ops
11
from vllm.utils import get_max_shared_memory_bytes, is_hip
12
13
FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
@@ -237,14 +237,14 @@ def test_paged_attention(
237
dequantized_key_cache = torch.empty(size=key_cache_shape,
238
dtype=dtype,
239
device=device)
240
- cache_ops.convert_fp8(key_cache, dequantized_key_cache)
240
+ ops.convert_fp8(key_cache, dequantized_key_cache)
241
key_cache = dequantized_key_cache
242
243
value_cache_shape = value_cache.shape
244
dequantized_value_cache = torch.empty(size=value_cache_shape,
245
dtype=dtype,
246
device=device)
247
- cache_ops.convert_fp8(value_cache, dequantized_value_cache)
247
+ ops.convert_fp8(value_cache, dequantized_value_cache)
248
value_cache = dequantized_value_cache
249
250
ref_output = torch.empty_like(query)
tests/kernels/test_cache.py
+12
-13
@@ -4,7 +4,7 @@ from typing import Tuple
4
import pytest
5
import torch
6
7
-from vllm._C import cache_ops
7
+from vllm import _custom_ops as ops
8
from vllm.utils import is_hip
9
10
COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
@@ -80,7 +80,7 @@ def test_copy_blocks(
80
cloned_value_caches = [value_cache.clone() for value_cache in value_caches]
81
82
# Call the copy blocks kernel.
83
- cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
83
+ ops.copy_blocks(key_caches, value_caches, block_mapping)
84
85
# Run the reference implementation.
86
for src, dsts in block_mapping.items():
@@ -145,9 +145,9 @@ def test_reshape_and_cache(
145
# Clone the KV caches.
146
if kv_cache_dtype == "fp8":
147
cloned_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
148
- cache_ops.convert_fp8(key_cache, cloned_key_cache)
148
+ ops.convert_fp8(key_cache, cloned_key_cache)
149
cloned_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
150
- cache_ops.convert_fp8(value_cache, cloned_value_cache)
150
+ ops.convert_fp8(value_cache, cloned_value_cache)
151
else:
152
cloned_key_cache = key_cache.clone()
153
cloned_value_cache = value_cache.clone()
@@ -156,14 +156,14 @@ def test_reshape_and_cache(
156
kv_scale = 1.0
157
158
# Call the reshape_and_cache kernel.
159
- cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
160
- slot_mapping, kv_cache_dtype, kv_scale)
159
+ ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping,
160
+ kv_cache_dtype, kv_scale)
161
162
if kv_cache_dtype == "fp8":
163
result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
164
- cache_ops.convert_fp8(key_cache, result_key_cache)
164
+ ops.convert_fp8(key_cache, result_key_cache)
165
result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
166
- cache_ops.convert_fp8(value_cache, result_value_cache)
166
+ ops.convert_fp8(value_cache, result_value_cache)
167
168
# Run the reference implementation.
169
reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
@@ -251,9 +251,8 @@ def test_swap_blocks(
251
src_value_caches_clone = src_value_caches[0].clone()
252
253
# Call the swap_blocks kernel.
254
- cache_ops.swap_blocks(src_key_caches[0], dist_key_caches[0], block_mapping)
255
- cache_ops.swap_blocks(src_value_caches[0], dist_value_caches[0],
256
- block_mapping)
254
+ ops.swap_blocks(src_key_caches[0], dist_key_caches[0], block_mapping)
255
+ ops.swap_blocks(src_value_caches[0], dist_value_caches[0], block_mapping)
256
257
for src, dst in block_mapping.items():
258
assert torch.allclose(src_key_caches_clone[src].cpu(),
@@ -291,9 +290,9 @@ def test_fp8_conversion(
290
cache.uniform_(low, high)
291
292
cache_fp8 = torch.empty_like(cache, dtype=torch.uint8)
294
- cache_ops.convert_fp8(cache, cache_fp8)
293
+ ops.convert_fp8(cache, cache_fp8)
294
295
converted_cache = torch.empty_like(cache)
297
- cache_ops.convert_fp8(cache_fp8, converted_cache)
296
+ ops.convert_fp8(cache_fp8, converted_cache)
297
298
assert torch.allclose(cache, converted_cache, atol=0.001, rtol=0.1)
vllm/_custom_ops.py
new
+193
@@ -0,0 +1,193 @@
1
+from typing import Dict, Optional
2
+
3
+import torch
4
+
5
+try:
6
+ from vllm._C import cache_ops as vllm_cache_ops
7
+ from vllm._C import ops as vllm_ops
8
+except ImportError:
9
+ pass
10
+
11
+
12
+# activation ops
13
+def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
14
+ vllm_ops.silu_and_mul(out, x)
15
+
16
+
17
+def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
18
+ vllm_ops.gelu_and_mul(out, x)
19
+
20
+
21
+def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
22
+ vllm_ops.gelu_tanh_and_mul(out, x)
23
+
24
+
25
+def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
26
+ vllm_ops.gelu_fast(out, x)
27
+
28
+
29
+def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
30
+ vllm_ops.gelu_new(out, x)
31
+
32
+
33
+# page attention ops
34
+def paged_attention_v1(
35
+ out: torch.Tensor,
36
+ query: torch.Tensor,
37
+ key_cache: torch.Tensor,
38
+ value_cache: torch.Tensor,
39
+ num_kv_heads: int,
40
+ scale: float,
41
+ block_tables: torch.Tensor,
42
+ context_lens: torch.Tensor,
43
+ block_size: int,
44
+ max_context_len: int,
45
+ alibi_slopes: Optional[torch.Tensor],
46
+ kv_cache_dtype: str,
47
+ kv_scale: float,
48
+) -> None:
49
+ vllm_ops.paged_attention_v1(out, query, key_cache, value_cache,
50
+ num_kv_heads, scale, block_tables,
51
+ context_lens, block_size, max_context_len,
52
+ alibi_slopes, kv_cache_dtype, kv_scale)
53
+
54
+
55
+def paged_attention_v2(
56
+ out: torch.Tensor,
57
+ exp_sum: torch.Tensor,
58
+ max_logits: torch.Tensor,
59
+ tmp_out: torch.Tensor,
60
+ query: torch.Tensor,
61
+ key_cache: torch.Tensor,
62
+ value_cache: torch.Tensor,
63
+ num_kv_heads: int,
64
+ scale: float,
65
+ block_tables: torch.Tensor,
66
+ context_lens: torch.Tensor,
67
+ block_size: int,
68
+ max_context_len: int,
69
+ alibi_slopes: Optional[torch.Tensor],
70
+ kv_cache_dtype: str,
71
+ kv_scale: float,
72
+) -> None:
73
+ vllm_ops.paged_attention_v2(out, exp_sum, max_logits, tmp_out, query,
74
+ key_cache, value_cache, num_kv_heads, scale,
75
+ block_tables, context_lens, block_size,
76
+ max_context_len, alibi_slopes, kv_cache_dtype,
77
+ kv_scale)
78
+
79
+
80
+# pos encoding ops
81
+def rotary_embedding(
82
+ positions: torch.Tensor,
83
+ query: torch.Tensor,
84
+ key: torch.Tensor,
85
+ head_size: int,
86
+ cos_sin_cache: torch.Tensor,
87
+ is_neox: bool,
88
+) -> None:
89
+ vllm_ops.rotary_embedding(positions, query, key, head_size, cos_sin_cache,
90
+ is_neox)
91
+
92
+
93
+def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
94
+ key: torch.Tensor, head_size: int,
95
+ cos_sin_cache: torch.Tensor, is_neox: bool,
96
+ rot_dim: int,
97
+ cos_sin_cache_offsets: torch.Tensor) -> None:
98
+ vllm_ops.batched_rotary_embedding(positions, query, key, head_size,
99
+ cos_sin_cache, is_neox, rot_dim,
100
+ cos_sin_cache_offsets)
101
+
102
+
103
+# layer norm ops
104
+def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor,
105
+ epsilon: float) -> None:
106
+ vllm_ops.rms_norm(out, input, weight, epsilon)
107
+
108
+
109
+def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
110
+ weight: torch.Tensor, epsilon: float) -> None:
111
+ vllm_ops.fused_add_rms_norm(input, residual, weight, epsilon)
112
+
113
+
114
+# quantization ops
115
+# awq
116
+def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor,
117
+ zeros: torch.Tensor, split_k_iters: int, thx: int,
118
+ thy: int) -> torch.Tensor:
119
+ return vllm_ops.awq_dequantize(qweight, scales, zeros, split_k_iters, thx,
120
+ thy)
121
+
122
+
123
+def awq_gemm(input: torch.Tensor, qweight: torch.Tensor, qzeros: torch.Tensor,
124
+ scales: torch.Tensor, split_k_iters: int) -> torch.Tensor:
125
+ return vllm_ops.awq_gemm(input, qweight, qzeros, scales, split_k_iters)
126
+
127
+
128
+# gptq
129
+def gptq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
130
+ b_gptq_qzeros: torch.Tensor, b_gptq_scales: torch.Tensor,
131
+ b_g_idx: torch.Tensor, use_exllama: bool,
132
+ bit: int) -> torch.Tensor:
133
+ return vllm_ops.gptq_gemm(a, b_q_weight, b_gptq_qzeros, b_gptq_scales,
134
+ b_g_idx, use_exllama, bit)
135
+
136
+
137
+def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor,
138
+ bit: int) -> None:
139
+ vllm_ops.gptq_shuffle(q_weight, q_perm, bit)
140
+
141
+
142
+# squeezellm
143
+def squeezellm_gemm(vec: torch.Tensor, mat: torch.Tensor, mul: torch.Tensor,
144
+ lookup_table: torch.Tensor) -> None:
145
+ vllm_ops.squeezellm_gemm(vec, mat, mul, lookup_table)
146
+
147
+
148
+# marlin
149
+def marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
150
+ b_scales: torch.Tensor, workspace: torch.Tensor, size_m: int,
151
+ size_n: int, size_k: int) -> torch.Tensor:
152
+ return vllm_ops.marlin_gemm(a, b_q_weight, b_scales, workspace, size_m,
153
+ size_n, size_k)
154
+
155
+
156
+# moe
157
+def moe_align_block_size(topk_ids: torch.Tensor, num_experts: int,
158
+ block_size: int, sorted_token_ids: torch.Tensor,
159
+ experts_ids: torch.Tensor,
160
+ num_tokens_post_pad: torch.Tensor) -> None:
161
+ vllm_ops.moe_align_block_size(topk_ids, num_experts, block_size,
162
+ sorted_token_ids, experts_ids,
163
+ num_tokens_post_pad)
164
+
165
+
166
+def reshape_and_cache(
167
+ key: torch.Tensor,
168
+ value: torch.Tensor,
169
+ key_cache: torch.Tensor,
170
+ value_cache: torch.Tensor,
171
+ slot_mapping: torch.Tensor,
172
+ kv_cache_dtype: str,
173
+ kv_scale: float,
174
+) -> None:
175
+ vllm_cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
176
+ slot_mapping, kv_cache_dtype, kv_scale)
177
+
178
+
179
+def copy_blocks(key_caches: torch.Tensor, value_caches: torch.Tensor,
180
+ block_mapping: torch.Tensor) -> None:
181
+ vllm_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
182
+
183
+
184
+def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
185
+ block_mapping: Dict[int, int]) -> None:
186
+ vllm_cache_ops.swap_blocks(src, dst, block_mapping)
187
+
188
+
189
+def convert_fp8(output: torch.Tensor, input: torch.Tensor) -> None:
190
+ vllm_cache_ops.convert_fp8(output, input)
191
+
192
+
193
+#TODO: cuda_utils, custom_ar
vllm/attention/ops/paged_attn.py
+5
-5
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional, Tuple
3
4
import torch
5
6
-from vllm._C import cache_ops, ops
6
+from vllm import _custom_ops as ops
7
from vllm.attention.ops.prefix_prefill import context_attention_fwd
8
9
# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -69,7 +69,7 @@ class PagedAttention:
69
kv_cache_dtype: str,
70
kv_scale: float,
71
) -> None:
72
- cache_ops.reshape_and_cache(
72
+ ops.reshape_and_cache(
73
key,
74
value,
75
key_cache,
@@ -199,11 +199,11 @@ class PagedAttention:
199
) -> None:
200
src_key_cache = src_kv_cache[0]
201
dst_key_cache = dst_kv_cache[0]
202
- cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
202
+ ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
203
204
src_value_cache = src_kv_cache[1]
205
dst_value_cache = dst_kv_cache[1]
206
- cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
206
+ ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
207
208
@staticmethod
209
def copy_blocks(
@@ -212,4 +212,4 @@ class PagedAttention:
212
) -> None:
213
key_caches = [kv_cache[0] for kv_cache in kv_caches]
214
value_caches = [kv_cache[1] for kv_cache in kv_caches]
215
- cache_ops.copy_blocks(key_caches, value_caches, src_to_dists)
215
+ ops.copy_blocks(key_caches, value_caches, src_to_dists)
vllm/model_executor/layers/activation.py
+1
-1
@@ -6,7 +6,7 @@ import torch
6
import torch.nn as nn
7
import torch.nn.functional as F
8
9
-from vllm._C import ops
9
+from vllm import _custom_ops as ops
10
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
11
get_tensor_model_parallel_world_size)
12
from vllm.model_executor.layers.quantization import QuantizationConfig
vllm/model_executor/layers/fused_moe/fused_moe.py
+1
-1
@@ -8,7 +8,7 @@ import torch
8
import triton
9
import triton.language as tl
10
11
-from vllm._C import ops
11
+from vllm import _custom_ops as ops
12
from vllm.logger import init_logger
13
from vllm.utils import is_hip
14
vllm/model_executor/layers/layernorm.py
+1
-1
@@ -4,7 +4,7 @@ from typing import Optional, Tuple, Union
4
import torch
5
import torch.nn as nn
6
7
-from vllm._C import ops
7
+from vllm import _custom_ops as ops
8
9
10
class RMSNorm(nn.Module):
vllm/model_executor/layers/quantization/awq.py
+1
-1
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
3
import torch
4
from torch.nn.parameter import Parameter
5
6
-from vllm._C import ops
6
+from vllm import _custom_ops as ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
from vllm.model_executor.layers.quantization.base_config import (
vllm/model_executor/layers/quantization/gptq.py
+1
-1
@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional
6
import torch
7
from torch.nn.parameter import Parameter
8
9
-from vllm._C import ops
9
+from vllm import _custom_ops as ops
10
from vllm.model_executor.layers.linear import (LinearMethodBase,
11
set_weight_attrs)
12
from vllm.model_executor.layers.quantization.base_config import (
vllm/model_executor/layers/quantization/marlin.py
+1
-1
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
3
import torch
4
from torch.nn.parameter import Parameter
5
6
-from vllm._C import ops
6
+from vllm import _custom_ops as ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
from vllm.model_executor.layers.quantization.base_config import (
vllm/model_executor/layers/quantization/squeezellm.py
+1
-1
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
3
import torch
4
from torch.nn.parameter import Parameter
5
6
-from vllm._C import ops
6
+from vllm import _custom_ops as ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
from vllm.model_executor.layers.quantization.base_config import (
vllm/model_executor/layers/rotary_embedding.py
+1
-1
@@ -27,7 +27,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
27
import torch
28
import torch.nn as nn
29
30
-from vllm._C import ops
30
+from vllm import _custom_ops as ops
31
32
33
def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
vllm/utils.py
+2
-2
@@ -279,10 +279,10 @@ def _generate_random_fp8(
279
#-----|-------------|-------------------
280
# Inf | N/A | s.11111.00
281
# NaN | s.1111.111 | s.11111.{01,10,11}
282
- from vllm._C import cache_ops
282
+ from vllm import _custom_ops as ops
283
tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
284
tensor_tmp.uniform_(low, high)
285
- cache_ops.convert_fp8(tensor_tmp, tensor)
285
+ ops.convert_fp8(tensor_tmp, tensor)
286
del tensor_tmp
287
288