A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Build] Avoid building too many extensions (#1624)
Yanming W committed
Nov 23, 2023 at 16:31 UTC
e0c6f556e85053059c74ab6b5cee396baf3b4316
25 files changed
+206
-272
benchmarks/kernels/benchmark_paged_attention.py
+3
-3
@@ -4,7 +4,7 @@ import time
4
5
import torch
6
7
-from vllm import attention_ops
7
+from vllm._C import ops
8
9
NUM_BLOCKS = 1024
10
PARTITION_SIZE = 512
@@ -98,7 +98,7 @@ def main(
98
99
for _ in range(num_iters):
100
if version == "v1":
101
- attention_ops.paged_attention_v1(
101
+ ops.paged_attention_v1(
102
output,
103
query,
104
key_cache,
@@ -112,7 +112,7 @@ def main(
112
alibi_slopes,
113
)
114
elif version == "v2":
115
- attention_ops.paged_attention_v2(
115
+ ops.paged_attention_v2(
116
output,
117
exp_sums,
118
max_logits,
csrc/activation.cpp
deleted
-28
@@ -1,28 +0,0 @@
1
-#include <torch/extension.h>
2
-
3
-void silu_and_mul(
4
- torch::Tensor& out,
5
- torch::Tensor& input);
6
-
7
-void gelu_new(
8
- torch::Tensor& out,
9
- torch::Tensor& input);
10
-
11
-void gelu_fast(
12
- torch::Tensor& out,
13
- torch::Tensor& input);
14
-
15
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16
- m.def(
17
- "silu_and_mul",
18
- &silu_and_mul,
19
- "Activation function used in SwiGLU.");
20
- m.def(
21
- "gelu_new",
22
- &gelu_new,
23
- "GELU implementation used in GPT-2.");
24
- m.def(
25
- "gelu_fast",
26
- &gelu_fast,
27
- "Approximate GELU implementation.");
28
-}
csrc/attention.cpp
deleted
-42
@@ -1,42 +0,0 @@
1
-#include <torch/extension.h>
2
-#include <c10/util/Optional.h>
3
-
4
-void paged_attention_v1(
5
- torch::Tensor& out,
6
- torch::Tensor& query,
7
- torch::Tensor& key_cache,
8
- torch::Tensor& value_cache,
9
- torch::Tensor& head_mapping,
10
- float scale,
11
- torch::Tensor& block_tables,
12
- torch::Tensor& context_lens,
13
- int block_size,
14
- int max_context_len,
15
- const c10::optional<torch::Tensor>& alibi_slopes);
16
-
17
-void paged_attention_v2(
18
- torch::Tensor& out,
19
- torch::Tensor& exp_sums,
20
- torch::Tensor& max_logits,
21
- torch::Tensor& tmp_out,
22
- torch::Tensor& query,
23
- torch::Tensor& key_cache,
24
- torch::Tensor& value_cache,
25
- torch::Tensor& head_mapping,
26
- float scale,
27
- torch::Tensor& block_tables,
28
- torch::Tensor& context_lens,
29
- int block_size,
30
- int max_context_len,
31
- const c10::optional<torch::Tensor>& alibi_slopes);
32
-
33
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
34
- m.def(
35
- "paged_attention_v1",
36
- &paged_attention_v1,
37
- "Compute the attention between an input query and the cached keys/values using PagedAttention.");
38
- m.def(
39
- "paged_attention_v2",
40
- &paged_attention_v2,
41
- "PagedAttention V2.");
42
-}
csrc/cache.h
renamed
-19
@@ -26,22 +26,3 @@ void gather_cached_kv(
26
torch::Tensor& key_cache,
27
torch::Tensor& value_cache,
28
torch::Tensor& slot_mapping);
29
-
30
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
31
- m.def(
32
- "swap_blocks",
33
- &swap_blocks,
34
- "Swap in (out) the cache blocks from src to dst");
35
- m.def(
36
- "copy_blocks",
37
- ©_blocks,
38
- "Copy the cache blocks from src to dst");
39
- m.def(
40
- "reshape_and_cache",
41
- &reshape_and_cache,
42
- "Reshape the key and value tensors and cache them");
43
- m.def(
44
- "gather_cached_kv",
45
- &gather_cached_kv,
46
- "Gather key and value from the cache into contiguous QKV tensors");
47
-}
csrc/cuda_utils.cpp
deleted
-13
@@ -1,13 +0,0 @@
1
-#include <torch/extension.h>
2
-
3
-int get_device_attribute(
4
- int attribute,
5
- int device_id);
6
-
7
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8
- m.def(
9
- "get_device_attribute",
10
- &get_device_attribute,
11
- "Gets the specified device attribute.");
12
-}
13
-
csrc/cuda_utils.h
new
+5
@@ -0,0 +1,5 @@
1
+#include <torch/extension.h>
2
+
3
+int get_device_attribute(
4
+ int attribute,
5
+ int device_id);
csrc/layernorm.cpp
deleted
-24
@@ -1,24 +0,0 @@
1
-#include <torch/extension.h>
2
-
3
-void rms_norm(
4
- torch::Tensor& out,
5
- torch::Tensor& input,
6
- torch::Tensor& weight,
7
- float epsilon);
8
-
9
-void fused_add_rms_norm(
10
- torch::Tensor& input,
11
- torch::Tensor& residual,
12
- torch::Tensor& weight,
13
- float epsilon);
14
-
15
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16
- m.def(
17
- "rms_norm",
18
- &rms_norm,
19
- "Apply Root Mean Square (RMS) Normalization to the input tensor.");
20
- m.def(
21
- "fused_add_rms_norm",
22
- &fused_add_rms_norm,
23
- "In-place fused Add and RMS Normalization");
24
-}
csrc/ops.h
new
+75
@@ -0,0 +1,75 @@
1
+#include <torch/extension.h>
2
+
3
+void paged_attention_v1(
4
+ torch::Tensor& out,
5
+ torch::Tensor& query,
6
+ torch::Tensor& key_cache,
7
+ torch::Tensor& value_cache,
8
+ torch::Tensor& head_mapping,
9
+ float scale,
10
+ torch::Tensor& block_tables,
11
+ torch::Tensor& context_lens,
12
+ int block_size,
13
+ int max_context_len,
14
+ const c10::optional<torch::Tensor>& alibi_slopes);
15
+
16
+void paged_attention_v2(
17
+ torch::Tensor& out,
18
+ torch::Tensor& exp_sums,
19
+ torch::Tensor& max_logits,
20
+ torch::Tensor& tmp_out,
21
+ torch::Tensor& query,
22
+ torch::Tensor& key_cache,
23
+ torch::Tensor& value_cache,
24
+ torch::Tensor& head_mapping,
25
+ float scale,
26
+ torch::Tensor& block_tables,
27
+ torch::Tensor& context_lens,
28
+ int block_size,
29
+ int max_context_len,
30
+ const c10::optional<torch::Tensor>& alibi_slopes);
31
+
32
+void rms_norm(
33
+ torch::Tensor& out,
34
+ torch::Tensor& input,
35
+ torch::Tensor& weight,
36
+ float epsilon);
37
+
38
+void fused_add_rms_norm(
39
+ torch::Tensor& input,
40
+ torch::Tensor& residual,
41
+ torch::Tensor& weight,
42
+ float epsilon);
43
+
44
+void rotary_embedding(
45
+ torch::Tensor& positions,
46
+ torch::Tensor& query,
47
+ torch::Tensor& key,
48
+ int head_size,
49
+ torch::Tensor& cos_sin_cache,
50
+ bool is_neox);
51
+
52
+void silu_and_mul(
53
+ torch::Tensor& out,
54
+ torch::Tensor& input);
55
+
56
+void gelu_new(
57
+ torch::Tensor& out,
58
+ torch::Tensor& input);
59
+
60
+void gelu_fast(
61
+ torch::Tensor& out,
62
+ torch::Tensor& input);
63
+
64
+torch::Tensor awq_gemm(
65
+ torch::Tensor _in_feats,
66
+ torch::Tensor _kernel,
67
+ torch::Tensor _scaling_factors,
68
+ torch::Tensor _zeros,
69
+ int split_k_iters);
70
+
71
+void squeezellm_gemm(
72
+ torch::Tensor vec,
73
+ torch::Tensor mat,
74
+ torch::Tensor mul,
75
+ torch::Tensor lookup_table);
csrc/pos_encoding.cpp
deleted
-16
@@ -1,16 +0,0 @@
1
-#include <torch/extension.h>
2
-
3
-void rotary_embedding(
4
- torch::Tensor& positions,
5
- torch::Tensor& query,
6
- torch::Tensor& key,
7
- int head_size,
8
- torch::Tensor& cos_sin_cache,
9
- bool is_neox);
10
-
11
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
12
- m.def(
13
- "rotary_embedding",
14
- &rotary_embedding,
15
- "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
16
-}
csrc/pybind.cpp
new
+80
@@ -0,0 +1,80 @@
1
+#include "cache.h"
2
+#include "cuda_utils.h"
3
+#include "ops.h"
4
+#include <torch/extension.h>
5
+
6
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
7
+ // vLLM custom ops
8
+ pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9
+
10
+ // Attention ops
11
+ ops.def(
12
+ "paged_attention_v1",
13
+ &paged_attention_v1,
14
+ "Compute the attention between an input query and the cached keys/values using PagedAttention.");
15
+ ops.def(
16
+ "paged_attention_v2",
17
+ &paged_attention_v2,
18
+ "PagedAttention V2.");
19
+
20
+ // Activation ops
21
+ ops.def(
22
+ "silu_and_mul",
23
+ &silu_and_mul,
24
+ "Activation function used in SwiGLU.");
25
+ ops.def(
26
+ "gelu_new",
27
+ &gelu_new,
28
+ "GELU implementation used in GPT-2.");
29
+ ops.def(
30
+ "gelu_fast",
31
+ &gelu_fast,
32
+ "Approximate GELU implementation.");
33
+
34
+ // Layernorm
35
+ ops.def(
36
+ "rms_norm",
37
+ &rms_norm,
38
+ "Apply Root Mean Square (RMS) Normalization to the input tensor.");
39
+
40
+ ops.def(
41
+ "fused_add_rms_norm",
42
+ &fused_add_rms_norm,
43
+ "In-place fused Add and RMS Normalization");
44
+
45
+ // Rotary embedding
46
+ ops.def(
47
+ "rotary_embedding",
48
+ &rotary_embedding,
49
+ "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
50
+
51
+ // Quantization ops
52
+ ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
53
+ ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
54
+
55
+ // Cache ops
56
+ pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
57
+ cache_ops.def(
58
+ "swap_blocks",
59
+ &swap_blocks,
60
+ "Swap in (out) the cache blocks from src to dst");
61
+ cache_ops.def(
62
+ "copy_blocks",
63
+ ©_blocks,
64
+ "Copy the cache blocks from src to dst");
65
+ cache_ops.def(
66
+ "reshape_and_cache",
67
+ &reshape_and_cache,
68
+ "Reshape the key and value tensors and cache them");
69
+ cache_ops.def(
70
+ "gather_cached_kv",
71
+ &gather_cached_kv,
72
+ "Gather key and value from the cache into contiguous QKV tensors");
73
+
74
+ // Cuda utils
75
+ pybind11::module cuda_utils = m.def_submodule("cuda_utils", "vLLM cuda utils");
76
+ cuda_utils.def(
77
+ "get_device_attribute",
78
+ &get_device_attribute,
79
+ "Gets the specified device attribute.");
80
+}
csrc/quantization.cpp
deleted
-19
@@ -1,19 +0,0 @@
1
-#include <torch/extension.h>
2
-
3
-torch::Tensor awq_gemm(
4
- torch::Tensor _in_feats,
5
- torch::Tensor _kernel,
6
- torch::Tensor _scaling_factors,
7
- torch::Tensor _zeros,
8
- int split_k_iters);
9
-
10
-void squeezellm_gemm(
11
- torch::Tensor vec,
12
- torch::Tensor mat,
13
- torch::Tensor mul,
14
- torch::Tensor lookup_table);
15
-
16
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
17
- m.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
18
- m.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
19
-}
setup.py
+10
-72
@@ -142,87 +142,25 @@ if nvcc_cuda_version >= Version("11.2"):
142
NVCC_FLAGS += ["--threads", str(num_threads)]
143
144
ext_modules = []
145
-
146
-# Cache operations.
147
-cache_extension = CUDAExtension(
148
- name="vllm.cache_ops",
149
- sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
150
- extra_compile_args={
151
- "cxx": CXX_FLAGS,
152
- "nvcc": NVCC_FLAGS,
153
- },
154
-)
155
-ext_modules.append(cache_extension)
156
-
157
-# Attention kernels.
158
-attention_extension = CUDAExtension(
159
- name="vllm.attention_ops",
160
- sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
161
- extra_compile_args={
162
- "cxx": CXX_FLAGS,
163
- "nvcc": NVCC_FLAGS,
164
- },
165
-)
166
-ext_modules.append(attention_extension)
167
-
168
-# Positional encoding kernels.
169
-positional_encoding_extension = CUDAExtension(
170
- name="vllm.pos_encoding_ops",
171
- sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
172
- extra_compile_args={
173
- "cxx": CXX_FLAGS,
174
- "nvcc": NVCC_FLAGS,
175
- },
176
-)
177
-ext_modules.append(positional_encoding_extension)
178
-
179
-# Layer normalization kernels.
180
-layernorm_extension = CUDAExtension(
181
- name="vllm.layernorm_ops",
182
- sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
183
- extra_compile_args={
184
- "cxx": CXX_FLAGS,
185
- "nvcc": NVCC_FLAGS,
186
- },
187
-)
188
-ext_modules.append(layernorm_extension)
189
-
190
-# Activation kernels.
191
-activation_extension = CUDAExtension(
192
- name="vllm.activation_ops",
193
- sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
194
- extra_compile_args={
195
- "cxx": CXX_FLAGS,
196
- "nvcc": NVCC_FLAGS,
197
- },
198
-)
199
-ext_modules.append(activation_extension)
200
-
201
-# Quantization kernels.
202
-quantization_extension = CUDAExtension(
203
- name="vllm.quantization_ops",
145
+vllm_extension = CUDAExtension(
146
+ name="vllm._C",
147
sources=[
205
- "csrc/quantization.cpp",
148
+ "csrc/cache_kernels.cu",
149
+ "csrc/attention/attention_kernels.cu",
150
+ "csrc/pos_encoding_kernels.cu",
151
+ "csrc/activation_kernels.cu",
152
+ "csrc/layernorm_kernels.cu",
153
"csrc/quantization/awq/gemm_kernels.cu",
154
"csrc/quantization/squeezellm/quant_cuda_kernel.cu",
155
+ "csrc/cuda_utils_kernels.cu",
156
+ "csrc/pybind.cpp",
157
],
158
extra_compile_args={
159
"cxx": CXX_FLAGS,
160
"nvcc": NVCC_FLAGS,
161
},
162
)
214
-ext_modules.append(quantization_extension)
215
-
216
-# Misc. CUDA utils.
217
-cuda_utils_extension = CUDAExtension(
218
- name="vllm.cuda_utils",
219
- sources=["csrc/cuda_utils.cpp", "csrc/cuda_utils_kernels.cu"],
220
- extra_compile_args={
221
- "cxx": CXX_FLAGS,
222
- "nvcc": NVCC_FLAGS,
223
- },
224
-)
225
-ext_modules.append(cuda_utils_extension)
163
+ext_modules.append(vllm_extension)
164
165
166
def get_path(*filepath) -> str:
tests/kernels/test_activation.py
+4
-4
@@ -3,7 +3,7 @@ import torch
3
import torch.nn.functional as F
4
from transformers.activations import get_activation
5
6
-from vllm import activation_ops
6
+from vllm._C import ops
7
8
DTYPES = [torch.half, torch.bfloat16, torch.float]
9
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
@@ -31,7 +31,7 @@ def test_silu_and_mul(
31
torch.cuda.manual_seed(seed)
32
x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
33
out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
34
- activation_ops.silu_and_mul(out, x)
34
+ ops.silu_and_mul(out, x)
35
ref_out = ref_silu_and_mul(x)
36
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
37
@@ -51,7 +51,7 @@ def test_gelu_new(
51
torch.cuda.manual_seed(seed)
52
x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
53
out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
54
- activation_ops.gelu_new(out, x)
54
+ ops.gelu_new(out, x)
55
ref_out = get_activation("gelu_new")(x)
56
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
57
@@ -70,6 +70,6 @@ def test_gelu_fast(
70
torch.cuda.manual_seed(seed)
71
x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
72
out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
73
- activation_ops.gelu_fast(out, x)
73
+ ops.gelu_fast(out, x)
74
ref_out = get_activation("gelu_fast")(x)
75
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
tests/kernels/test_attention.py
+3
-3
@@ -6,7 +6,7 @@ import torch
6
from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9
-from vllm import attention_ops
9
+from vllm._C import ops
10
from vllm.utils import get_max_shared_memory_bytes
11
12
FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
@@ -165,7 +165,7 @@ def test_paged_attention(
165
# Call the paged attention kernel.
166
output = torch.empty_like(query)
167
if version == "v1":
168
- attention_ops.paged_attention_v1(
168
+ ops.paged_attention_v1(
169
output,
170
query,
171
key_cache,
@@ -194,7 +194,7 @@ def test_paged_attention(
194
device=output.device,
195
)
196
max_logits = torch.empty_like(exp_sums)
197
- attention_ops.paged_attention_v2(
197
+ ops.paged_attention_v2(
198
output,
199
exp_sums,
200
max_logits,
tests/kernels/test_cache.py
+1
-1
@@ -3,7 +3,7 @@ import random
3
import pytest
4
import torch
5
6
-from vllm import cache_ops
6
+from vllm._C import cache_ops
7
8
DTYPES = [torch.half, torch.bfloat16, torch.float]
9
NUM_TOKENS = [83] # Arbitrary values for testing
tests/kernels/test_layernorm.py
+2
-2
@@ -2,7 +2,7 @@ import pytest
2
import torch
3
import torch.nn as nn
4
5
-from vllm import layernorm_ops
5
+from vllm._C import ops
6
7
DTYPES = [torch.half, torch.bfloat16, torch.float]
8
HIDDEN_SIZES = [67, 768, 2048, 5120, 8192] # Arbitrary values for testing
@@ -48,7 +48,7 @@ def test_rms_norm(
48
ref = RefRMSNorm(hidden_size).to(dtype).cuda()
49
50
out = torch.empty_like(x)
51
- layernorm_ops.rms_norm(
51
+ ops.rms_norm(
52
out,
53
x,
54
ref.weight.data,
tests/kernels/test_pos_encoding.py
+2
-2
@@ -5,7 +5,7 @@ import torch
5
import torch.nn as nn
6
import torch.nn.functional as F
7
8
-from vllm import pos_encoding_ops
8
+from vllm._C import ops
9
10
IS_NEOX_STYLE = [True, False]
11
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -145,7 +145,7 @@ def test_rotary_embedding(
145
# Run the kernel. The kernel is in-place, so we need to clone the inputs.
146
out_query = query.clone()
147
out_key = key.clone()
148
- pos_encoding_ops.rotary_embedding(
148
+ ops.rotary_embedding(
149
positions,
150
out_query,
151
out_key,
vllm/model_executor/layers/activation.py
+4
-4
@@ -4,7 +4,7 @@ from typing import Optional
4
import torch
5
import torch.nn as nn
6
7
-from vllm import activation_ops
7
+from vllm._C import ops
8
from vllm.model_executor.layers.quantization import QuantizationConfig
9
from vllm.model_executor.parallel_utils.parallel_state import (
10
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -26,7 +26,7 @@ class SiluAndMul(nn.Module):
26
d = x.shape[-1] // 2
27
output_shape = (x.shape[:-1] + (d, ))
28
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
29
- activation_ops.silu_and_mul(out, x)
29
+ ops.silu_and_mul(out, x)
30
return out
31
32
@@ -34,7 +34,7 @@ class NewGELU(nn.Module):
34
35
def forward(self, x: torch.Tensor) -> torch.Tensor:
36
out = torch.empty_like(x)
37
- activation_ops.gelu_new(out, x)
37
+ ops.gelu_new(out, x)
38
return out
39
40
@@ -42,7 +42,7 @@ class FastGELU(nn.Module):
42
43
def forward(self, x: torch.Tensor) -> torch.Tensor:
44
out = torch.empty_like(x)
45
- activation_ops.gelu_fast(out, x)
45
+ ops.gelu_fast(out, x)
46
return out
47
48
vllm/model_executor/layers/attention.py
+4
-4
@@ -7,8 +7,8 @@ from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8
LowerTriangularMaskWithTensorBias)
9
10
-from vllm import attention_ops
11
-from vllm import cache_ops
10
+from vllm._C import ops
11
+from vllm._C import cache_ops
12
from vllm.model_executor.input_metadata import InputMetadata
13
from vllm.model_executor.layers.rotary_embedding import get_rope
14
@@ -163,7 +163,7 @@ class PagedAttention(nn.Module):
163
max_num_partitions == 1 or num_seqs * num_heads > 512)
164
if use_v1:
165
# Run PagedAttention V1.
166
- attention_ops.paged_attention_v1(
166
+ ops.paged_attention_v1(
167
output,
168
query,
169
key_cache,
@@ -190,7 +190,7 @@ class PagedAttention(nn.Module):
190
device=output.device,
191
)
192
max_logits = torch.empty_like(exp_sums)
193
- attention_ops.paged_attention_v2(
193
+ ops.paged_attention_v2(
194
output,
195
exp_sums,
196
max_logits,
vllm/model_executor/layers/layernorm.py
+3
-3
@@ -4,7 +4,7 @@ from typing import Optional, Tuple, Union
4
import torch
5
import torch.nn as nn
6
7
-from vllm import layernorm_ops
7
+from vllm._C import ops
8
9
10
class RMSNorm(nn.Module):
@@ -29,7 +29,7 @@ class RMSNorm(nn.Module):
29
residual: Optional[torch.Tensor] = None,
30
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
31
if residual is not None:
32
- layernorm_ops.fused_add_rms_norm(
32
+ ops.fused_add_rms_norm(
33
x,
34
residual,
35
self.weight.data,
@@ -37,7 +37,7 @@ class RMSNorm(nn.Module):
37
)
38
return x, residual
39
out = torch.empty_like(x)
40
- layernorm_ops.rms_norm(
40
+ ops.rms_norm(
41
out,
42
x,
43
self.weight.data,
vllm/model_executor/layers/quantization/awq.py
+2
-3
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
3
import torch
4
from torch.nn.parameter import Parameter
5
6
-from vllm import quantization_ops
6
+from vllm._C import ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -151,8 +151,7 @@ class AWQLinearMethod(LinearMethodBase):
151
pack_factor = self.quant_config.pack_factor
152
out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
153
reshaped_x = x.reshape(-1, x.shape[-1])
154
- out = quantization_ops.awq_gemm(reshaped_x, qweight, scales, qzeros,
155
- pack_factor)
154
+ out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor)
155
if bias is not None:
156
out = out + bias
157
return out.reshape(out_shape)
vllm/model_executor/layers/quantization/squeezellm.py
+2
-3
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
3
import torch
4
from torch.nn.parameter import Parameter
5
6
-from vllm import quantization_ops
6
+from vllm._C import ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -116,8 +116,7 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
116
reshaped_x = x.reshape(-1, x.shape[-1])
117
# NOTE: The output tensor should be zero-initialized.
118
out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
119
- quantization_ops.squeezellm_gemm(reshaped_x, qweight, out,
120
- lookup_table)
119
+ ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
120
121
if bias is not None:
122
out = out + bias
vllm/model_executor/layers/rotary_embedding.py
+4
-5
@@ -27,7 +27,7 @@ from typing import Any, Dict, Optional, Tuple, Union
27
import torch
28
import torch.nn as nn
29
30
-from vllm import pos_encoding_ops
30
+from vllm._C import ops
31
32
33
class RotaryEmbedding(nn.Module):
@@ -87,11 +87,10 @@ class RotaryEmbedding(nn.Module):
87
query: torch.Tensor,
88
key: torch.Tensor,
89
) -> Tuple[torch.Tensor, torch.Tensor]:
90
- # pos_encoding_ops.rotary_embedding() is an in-place operation that
90
+ # ops.rotary_embedding() is an in-place operation that
91
# updates the query and key tensors.
92
- pos_encoding_ops.rotary_embedding(positions, query, key,
93
- self.head_size, self.cos_sin_cache,
94
- self.is_neox_style)
92
+ ops.rotary_embedding(positions, query, key, self.head_size,
93
+ self.cos_sin_cache, self.is_neox_style)
94
return query, key
95
96
vllm/utils.py
+1
-1
@@ -5,7 +5,7 @@ from platform import uname
5
import psutil
6
import torch
7
8
-from vllm import cuda_utils
8
+from vllm._C import cuda_utils
9
10
11
class Device(enum.Enum):
vllm/worker/cache_engine.py
+1
-1
@@ -3,7 +3,7 @@ from typing import Dict, List, Tuple
3
4
import torch
5
6
-from vllm import cache_ops
6
+from vllm._C import cache_ops
7
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
from vllm.logger import init_logger
9
from vllm.utils import in_wsl