@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e0c6f556e8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Build] Avoid building too many extensions (#1624)

Yanming W committed Nov 23, 2023 at 16:31 UTC e0c6f556e85053059c74ab6b5cee396baf3b4316
25 files changed +206 -272
benchmarks/kernels/benchmark_paged_attention.py
+3 -3
@@ -4,7 +4,7 @@ import time
4
5 import torch
6
7 -from vllm import attention_ops
7 +from vllm._C import ops
8
9 NUM_BLOCKS = 1024
10 PARTITION_SIZE = 512
@@ -98,7 +98,7 @@ def main(
98
99 for _ in range(num_iters):
100 if version == "v1":
101 - attention_ops.paged_attention_v1(
101 + ops.paged_attention_v1(
102 output,
103 query,
104 key_cache,
@@ -112,7 +112,7 @@ def main(
112 alibi_slopes,
113 )
114 elif version == "v2":
115 - attention_ops.paged_attention_v2(
115 + ops.paged_attention_v2(
116 output,
117 exp_sums,
118 max_logits,
csrc/activation.cpp deleted
-28
@@ -1,28 +0,0 @@
1 -#include <torch/extension.h>
2 -
3 -void silu_and_mul(
4 - torch::Tensor& out,
5 - torch::Tensor& input);
6 -
7 -void gelu_new(
8 - torch::Tensor& out,
9 - torch::Tensor& input);
10 -
11 -void gelu_fast(
12 - torch::Tensor& out,
13 - torch::Tensor& input);
14 -
15 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16 - m.def(
17 - "silu_and_mul",
18 - &silu_and_mul,
19 - "Activation function used in SwiGLU.");
20 - m.def(
21 - "gelu_new",
22 - &gelu_new,
23 - "GELU implementation used in GPT-2.");
24 - m.def(
25 - "gelu_fast",
26 - &gelu_fast,
27 - "Approximate GELU implementation.");
28 -}
csrc/attention.cpp deleted
-42
@@ -1,42 +0,0 @@
1 -#include <torch/extension.h>
2 -#include <c10/util/Optional.h>
3 -
4 -void paged_attention_v1(
5 - torch::Tensor& out,
6 - torch::Tensor& query,
7 - torch::Tensor& key_cache,
8 - torch::Tensor& value_cache,
9 - torch::Tensor& head_mapping,
10 - float scale,
11 - torch::Tensor& block_tables,
12 - torch::Tensor& context_lens,
13 - int block_size,
14 - int max_context_len,
15 - const c10::optional<torch::Tensor>& alibi_slopes);
16 -
17 -void paged_attention_v2(
18 - torch::Tensor& out,
19 - torch::Tensor& exp_sums,
20 - torch::Tensor& max_logits,
21 - torch::Tensor& tmp_out,
22 - torch::Tensor& query,
23 - torch::Tensor& key_cache,
24 - torch::Tensor& value_cache,
25 - torch::Tensor& head_mapping,
26 - float scale,
27 - torch::Tensor& block_tables,
28 - torch::Tensor& context_lens,
29 - int block_size,
30 - int max_context_len,
31 - const c10::optional<torch::Tensor>& alibi_slopes);
32 -
33 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
34 - m.def(
35 - "paged_attention_v1",
36 - &paged_attention_v1,
37 - "Compute the attention between an input query and the cached keys/values using PagedAttention.");
38 - m.def(
39 - "paged_attention_v2",
40 - &paged_attention_v2,
41 - "PagedAttention V2.");
42 -}
csrc/cache.h renamed
-19
@@ -26,22 +26,3 @@ void gather_cached_kv(
26 torch::Tensor& key_cache,
27 torch::Tensor& value_cache,
28 torch::Tensor& slot_mapping);
29 -
30 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
31 - m.def(
32 - "swap_blocks",
33 - &swap_blocks,
34 - "Swap in (out) the cache blocks from src to dst");
35 - m.def(
36 - "copy_blocks",
37 - &copy_blocks,
38 - "Copy the cache blocks from src to dst");
39 - m.def(
40 - "reshape_and_cache",
41 - &reshape_and_cache,
42 - "Reshape the key and value tensors and cache them");
43 - m.def(
44 - "gather_cached_kv",
45 - &gather_cached_kv,
46 - "Gather key and value from the cache into contiguous QKV tensors");
47 -}
csrc/cuda_utils.cpp deleted
-13
@@ -1,13 +0,0 @@
1 -#include <torch/extension.h>
2 -
3 -int get_device_attribute(
4 - int attribute,
5 - int device_id);
6 -
7 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8 - m.def(
9 - "get_device_attribute",
10 - &get_device_attribute,
11 - "Gets the specified device attribute.");
12 -}
13 -
csrc/cuda_utils.h new
+5
@@ -0,0 +1,5 @@
1 +#include <torch/extension.h>
2 +
3 +int get_device_attribute(
4 + int attribute,
5 + int device_id);
csrc/layernorm.cpp deleted
-24
@@ -1,24 +0,0 @@
1 -#include <torch/extension.h>
2 -
3 -void rms_norm(
4 - torch::Tensor& out,
5 - torch::Tensor& input,
6 - torch::Tensor& weight,
7 - float epsilon);
8 -
9 -void fused_add_rms_norm(
10 - torch::Tensor& input,
11 - torch::Tensor& residual,
12 - torch::Tensor& weight,
13 - float epsilon);
14 -
15 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16 - m.def(
17 - "rms_norm",
18 - &rms_norm,
19 - "Apply Root Mean Square (RMS) Normalization to the input tensor.");
20 - m.def(
21 - "fused_add_rms_norm",
22 - &fused_add_rms_norm,
23 - "In-place fused Add and RMS Normalization");
24 -}
csrc/ops.h new
+75
@@ -0,0 +1,75 @@
1 +#include <torch/extension.h>
2 +
3 +void paged_attention_v1(
4 + torch::Tensor& out,
5 + torch::Tensor& query,
6 + torch::Tensor& key_cache,
7 + torch::Tensor& value_cache,
8 + torch::Tensor& head_mapping,
9 + float scale,
10 + torch::Tensor& block_tables,
11 + torch::Tensor& context_lens,
12 + int block_size,
13 + int max_context_len,
14 + const c10::optional<torch::Tensor>& alibi_slopes);
15 +
16 +void paged_attention_v2(
17 + torch::Tensor& out,
18 + torch::Tensor& exp_sums,
19 + torch::Tensor& max_logits,
20 + torch::Tensor& tmp_out,
21 + torch::Tensor& query,
22 + torch::Tensor& key_cache,
23 + torch::Tensor& value_cache,
24 + torch::Tensor& head_mapping,
25 + float scale,
26 + torch::Tensor& block_tables,
27 + torch::Tensor& context_lens,
28 + int block_size,
29 + int max_context_len,
30 + const c10::optional<torch::Tensor>& alibi_slopes);
31 +
32 +void rms_norm(
33 + torch::Tensor& out,
34 + torch::Tensor& input,
35 + torch::Tensor& weight,
36 + float epsilon);
37 +
38 +void fused_add_rms_norm(
39 + torch::Tensor& input,
40 + torch::Tensor& residual,
41 + torch::Tensor& weight,
42 + float epsilon);
43 +
44 +void rotary_embedding(
45 + torch::Tensor& positions,
46 + torch::Tensor& query,
47 + torch::Tensor& key,
48 + int head_size,
49 + torch::Tensor& cos_sin_cache,
50 + bool is_neox);
51 +
52 +void silu_and_mul(
53 + torch::Tensor& out,
54 + torch::Tensor& input);
55 +
56 +void gelu_new(
57 + torch::Tensor& out,
58 + torch::Tensor& input);
59 +
60 +void gelu_fast(
61 + torch::Tensor& out,
62 + torch::Tensor& input);
63 +
64 +torch::Tensor awq_gemm(
65 + torch::Tensor _in_feats,
66 + torch::Tensor _kernel,
67 + torch::Tensor _scaling_factors,
68 + torch::Tensor _zeros,
69 + int split_k_iters);
70 +
71 +void squeezellm_gemm(
72 + torch::Tensor vec,
73 + torch::Tensor mat,
74 + torch::Tensor mul,
75 + torch::Tensor lookup_table);
csrc/pos_encoding.cpp deleted
-16
@@ -1,16 +0,0 @@
1 -#include <torch/extension.h>
2 -
3 -void rotary_embedding(
4 - torch::Tensor& positions,
5 - torch::Tensor& query,
6 - torch::Tensor& key,
7 - int head_size,
8 - torch::Tensor& cos_sin_cache,
9 - bool is_neox);
10 -
11 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
12 - m.def(
13 - "rotary_embedding",
14 - &rotary_embedding,
15 - "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
16 -}
csrc/pybind.cpp new
+80
@@ -0,0 +1,80 @@
1 +#include "cache.h"
2 +#include "cuda_utils.h"
3 +#include "ops.h"
4 +#include <torch/extension.h>
5 +
6 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
7 + // vLLM custom ops
8 + pybind11::module ops = m.def_submodule("ops", "vLLM custom operators");
9 +
10 + // Attention ops
11 + ops.def(
12 + "paged_attention_v1",
13 + &paged_attention_v1,
14 + "Compute the attention between an input query and the cached keys/values using PagedAttention.");
15 + ops.def(
16 + "paged_attention_v2",
17 + &paged_attention_v2,
18 + "PagedAttention V2.");
19 +
20 + // Activation ops
21 + ops.def(
22 + "silu_and_mul",
23 + &silu_and_mul,
24 + "Activation function used in SwiGLU.");
25 + ops.def(
26 + "gelu_new",
27 + &gelu_new,
28 + "GELU implementation used in GPT-2.");
29 + ops.def(
30 + "gelu_fast",
31 + &gelu_fast,
32 + "Approximate GELU implementation.");
33 +
34 + // Layernorm
35 + ops.def(
36 + "rms_norm",
37 + &rms_norm,
38 + "Apply Root Mean Square (RMS) Normalization to the input tensor.");
39 +
40 + ops.def(
41 + "fused_add_rms_norm",
42 + &fused_add_rms_norm,
43 + "In-place fused Add and RMS Normalization");
44 +
45 + // Rotary embedding
46 + ops.def(
47 + "rotary_embedding",
48 + &rotary_embedding,
49 + "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
50 +
51 + // Quantization ops
52 + ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
53 + ops.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
54 +
55 + // Cache ops
56 + pybind11::module cache_ops = m.def_submodule("cache_ops", "vLLM cache ops");
57 + cache_ops.def(
58 + "swap_blocks",
59 + &swap_blocks,
60 + "Swap in (out) the cache blocks from src to dst");
61 + cache_ops.def(
62 + "copy_blocks",
63 + &copy_blocks,
64 + "Copy the cache blocks from src to dst");
65 + cache_ops.def(
66 + "reshape_and_cache",
67 + &reshape_and_cache,
68 + "Reshape the key and value tensors and cache them");
69 + cache_ops.def(
70 + "gather_cached_kv",
71 + &gather_cached_kv,
72 + "Gather key and value from the cache into contiguous QKV tensors");
73 +
74 + // Cuda utils
75 + pybind11::module cuda_utils = m.def_submodule("cuda_utils", "vLLM cuda utils");
76 + cuda_utils.def(
77 + "get_device_attribute",
78 + &get_device_attribute,
79 + "Gets the specified device attribute.");
80 +}
csrc/quantization.cpp deleted
-19
@@ -1,19 +0,0 @@
1 -#include <torch/extension.h>
2 -
3 -torch::Tensor awq_gemm(
4 - torch::Tensor _in_feats,
5 - torch::Tensor _kernel,
6 - torch::Tensor _scaling_factors,
7 - torch::Tensor _zeros,
8 - int split_k_iters);
9 -
10 -void squeezellm_gemm(
11 - torch::Tensor vec,
12 - torch::Tensor mat,
13 - torch::Tensor mul,
14 - torch::Tensor lookup_table);
15 -
16 -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
17 - m.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
18 - m.def("squeezellm_gemm", &squeezellm_gemm, "Quantized GEMM for SqueezeLLM");
19 -}
setup.py
+10 -72
@@ -142,87 +142,25 @@ if nvcc_cuda_version >= Version("11.2"):
142 NVCC_FLAGS += ["--threads", str(num_threads)]
143
144 ext_modules = []
145 -
146 -# Cache operations.
147 -cache_extension = CUDAExtension(
148 - name="vllm.cache_ops",
149 - sources=["csrc/cache.cpp", "csrc/cache_kernels.cu"],
150 - extra_compile_args={
151 - "cxx": CXX_FLAGS,
152 - "nvcc": NVCC_FLAGS,
153 - },
154 -)
155 -ext_modules.append(cache_extension)
156 -
157 -# Attention kernels.
158 -attention_extension = CUDAExtension(
159 - name="vllm.attention_ops",
160 - sources=["csrc/attention.cpp", "csrc/attention/attention_kernels.cu"],
161 - extra_compile_args={
162 - "cxx": CXX_FLAGS,
163 - "nvcc": NVCC_FLAGS,
164 - },
165 -)
166 -ext_modules.append(attention_extension)
167 -
168 -# Positional encoding kernels.
169 -positional_encoding_extension = CUDAExtension(
170 - name="vllm.pos_encoding_ops",
171 - sources=["csrc/pos_encoding.cpp", "csrc/pos_encoding_kernels.cu"],
172 - extra_compile_args={
173 - "cxx": CXX_FLAGS,
174 - "nvcc": NVCC_FLAGS,
175 - },
176 -)
177 -ext_modules.append(positional_encoding_extension)
178 -
179 -# Layer normalization kernels.
180 -layernorm_extension = CUDAExtension(
181 - name="vllm.layernorm_ops",
182 - sources=["csrc/layernorm.cpp", "csrc/layernorm_kernels.cu"],
183 - extra_compile_args={
184 - "cxx": CXX_FLAGS,
185 - "nvcc": NVCC_FLAGS,
186 - },
187 -)
188 -ext_modules.append(layernorm_extension)
189 -
190 -# Activation kernels.
191 -activation_extension = CUDAExtension(
192 - name="vllm.activation_ops",
193 - sources=["csrc/activation.cpp", "csrc/activation_kernels.cu"],
194 - extra_compile_args={
195 - "cxx": CXX_FLAGS,
196 - "nvcc": NVCC_FLAGS,
197 - },
198 -)
199 -ext_modules.append(activation_extension)
200 -
201 -# Quantization kernels.
202 -quantization_extension = CUDAExtension(
203 - name="vllm.quantization_ops",
145 +vllm_extension = CUDAExtension(
146 + name="vllm._C",
147 sources=[
205 - "csrc/quantization.cpp",
148 + "csrc/cache_kernels.cu",
149 + "csrc/attention/attention_kernels.cu",
150 + "csrc/pos_encoding_kernels.cu",
151 + "csrc/activation_kernels.cu",
152 + "csrc/layernorm_kernels.cu",
153 "csrc/quantization/awq/gemm_kernels.cu",
154 "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
155 + "csrc/cuda_utils_kernels.cu",
156 + "csrc/pybind.cpp",
157 ],
158 extra_compile_args={
159 "cxx": CXX_FLAGS,
160 "nvcc": NVCC_FLAGS,
161 },
162 )
214 -ext_modules.append(quantization_extension)
215 -
216 -# Misc. CUDA utils.
217 -cuda_utils_extension = CUDAExtension(
218 - name="vllm.cuda_utils",
219 - sources=["csrc/cuda_utils.cpp", "csrc/cuda_utils_kernels.cu"],
220 - extra_compile_args={
221 - "cxx": CXX_FLAGS,
222 - "nvcc": NVCC_FLAGS,
223 - },
224 -)
225 -ext_modules.append(cuda_utils_extension)
163 +ext_modules.append(vllm_extension)
164
165
166 def get_path(*filepath) -> str:
tests/kernels/test_activation.py
+4 -4
@@ -3,7 +3,7 @@ import torch
3 import torch.nn.functional as F
4 from transformers.activations import get_activation
5
6 -from vllm import activation_ops
6 +from vllm._C import ops
7
8 DTYPES = [torch.half, torch.bfloat16, torch.float]
9 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
@@ -31,7 +31,7 @@ def test_silu_and_mul(
31 torch.cuda.manual_seed(seed)
32 x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
33 out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
34 - activation_ops.silu_and_mul(out, x)
34 + ops.silu_and_mul(out, x)
35 ref_out = ref_silu_and_mul(x)
36 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
37
@@ -51,7 +51,7 @@ def test_gelu_new(
51 torch.cuda.manual_seed(seed)
52 x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
53 out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
54 - activation_ops.gelu_new(out, x)
54 + ops.gelu_new(out, x)
55 ref_out = get_activation("gelu_new")(x)
56 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
57
@@ -70,6 +70,6 @@ def test_gelu_fast(
70 torch.cuda.manual_seed(seed)
71 x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
72 out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
73 - activation_ops.gelu_fast(out, x)
73 + ops.gelu_fast(out, x)
74 ref_out = get_activation("gelu_fast")(x)
75 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
tests/kernels/test_attention.py
+3 -3
@@ -6,7 +6,7 @@ import torch
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9 -from vllm import attention_ops
9 +from vllm._C import ops
10 from vllm.utils import get_max_shared_memory_bytes
11
12 FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
@@ -165,7 +165,7 @@ def test_paged_attention(
165 # Call the paged attention kernel.
166 output = torch.empty_like(query)
167 if version == "v1":
168 - attention_ops.paged_attention_v1(
168 + ops.paged_attention_v1(
169 output,
170 query,
171 key_cache,
@@ -194,7 +194,7 @@ def test_paged_attention(
194 device=output.device,
195 )
196 max_logits = torch.empty_like(exp_sums)
197 - attention_ops.paged_attention_v2(
197 + ops.paged_attention_v2(
198 output,
199 exp_sums,
200 max_logits,
tests/kernels/test_cache.py
+1 -1
@@ -3,7 +3,7 @@ import random
3 import pytest
4 import torch
5
6 -from vllm import cache_ops
6 +from vllm._C import cache_ops
7
8 DTYPES = [torch.half, torch.bfloat16, torch.float]
9 NUM_TOKENS = [83] # Arbitrary values for testing
tests/kernels/test_layernorm.py
+2 -2
@@ -2,7 +2,7 @@ import pytest
2 import torch
3 import torch.nn as nn
4
5 -from vllm import layernorm_ops
5 +from vllm._C import ops
6
7 DTYPES = [torch.half, torch.bfloat16, torch.float]
8 HIDDEN_SIZES = [67, 768, 2048, 5120, 8192] # Arbitrary values for testing
@@ -48,7 +48,7 @@ def test_rms_norm(
48 ref = RefRMSNorm(hidden_size).to(dtype).cuda()
49
50 out = torch.empty_like(x)
51 - layernorm_ops.rms_norm(
51 + ops.rms_norm(
52 out,
53 x,
54 ref.weight.data,
tests/kernels/test_pos_encoding.py
+2 -2
@@ -5,7 +5,7 @@ import torch
5 import torch.nn as nn
6 import torch.nn.functional as F
7
8 -from vllm import pos_encoding_ops
8 +from vllm._C import ops
9
10 IS_NEOX_STYLE = [True, False]
11 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -145,7 +145,7 @@ def test_rotary_embedding(
145 # Run the kernel. The kernel is in-place, so we need to clone the inputs.
146 out_query = query.clone()
147 out_key = key.clone()
148 - pos_encoding_ops.rotary_embedding(
148 + ops.rotary_embedding(
149 positions,
150 out_query,
151 out_key,
vllm/model_executor/layers/activation.py
+4 -4
@@ -4,7 +4,7 @@ from typing import Optional
4 import torch
5 import torch.nn as nn
6
7 -from vllm import activation_ops
7 +from vllm._C import ops
8 from vllm.model_executor.layers.quantization import QuantizationConfig
9 from vllm.model_executor.parallel_utils.parallel_state import (
10 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -26,7 +26,7 @@ class SiluAndMul(nn.Module):
26 d = x.shape[-1] // 2
27 output_shape = (x.shape[:-1] + (d, ))
28 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
29 - activation_ops.silu_and_mul(out, x)
29 + ops.silu_and_mul(out, x)
30 return out
31
32
@@ -34,7 +34,7 @@ class NewGELU(nn.Module):
34
35 def forward(self, x: torch.Tensor) -> torch.Tensor:
36 out = torch.empty_like(x)
37 - activation_ops.gelu_new(out, x)
37 + ops.gelu_new(out, x)
38 return out
39
40
@@ -42,7 +42,7 @@ class FastGELU(nn.Module):
42
43 def forward(self, x: torch.Tensor) -> torch.Tensor:
44 out = torch.empty_like(x)
45 - activation_ops.gelu_fast(out, x)
45 + ops.gelu_fast(out, x)
46 return out
47
48
vllm/model_executor/layers/attention.py
+4 -4
@@ -7,8 +7,8 @@ from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8 LowerTriangularMaskWithTensorBias)
9
10 -from vllm import attention_ops
11 -from vllm import cache_ops
10 +from vllm._C import ops
11 +from vllm._C import cache_ops
12 from vllm.model_executor.input_metadata import InputMetadata
13 from vllm.model_executor.layers.rotary_embedding import get_rope
14
@@ -163,7 +163,7 @@ class PagedAttention(nn.Module):
163 max_num_partitions == 1 or num_seqs * num_heads > 512)
164 if use_v1:
165 # Run PagedAttention V1.
166 - attention_ops.paged_attention_v1(
166 + ops.paged_attention_v1(
167 output,
168 query,
169 key_cache,
@@ -190,7 +190,7 @@ class PagedAttention(nn.Module):
190 device=output.device,
191 )
192 max_logits = torch.empty_like(exp_sums)
193 - attention_ops.paged_attention_v2(
193 + ops.paged_attention_v2(
194 output,
195 exp_sums,
196 max_logits,
vllm/model_executor/layers/layernorm.py
+3 -3
@@ -4,7 +4,7 @@ from typing import Optional, Tuple, Union
4 import torch
5 import torch.nn as nn
6
7 -from vllm import layernorm_ops
7 +from vllm._C import ops
8
9
10 class RMSNorm(nn.Module):
@@ -29,7 +29,7 @@ class RMSNorm(nn.Module):
29 residual: Optional[torch.Tensor] = None,
30 ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
31 if residual is not None:
32 - layernorm_ops.fused_add_rms_norm(
32 + ops.fused_add_rms_norm(
33 x,
34 residual,
35 self.weight.data,
@@ -37,7 +37,7 @@ class RMSNorm(nn.Module):
37 )
38 return x, residual
39 out = torch.empty_like(x)
40 - layernorm_ops.rms_norm(
40 + ops.rms_norm(
41 out,
42 x,
43 self.weight.data,
vllm/model_executor/layers/quantization/awq.py
+2 -3
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
3 import torch
4 from torch.nn.parameter import Parameter
5
6 -from vllm import quantization_ops
6 +from vllm._C import ops
7 from vllm.model_executor.layers.linear import (LinearMethodBase,
8 set_weight_attrs)
9 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -151,8 +151,7 @@ class AWQLinearMethod(LinearMethodBase):
151 pack_factor = self.quant_config.pack_factor
152 out_shape = (x.shape[:-1] + (qweight.shape[-1] * pack_factor, ))
153 reshaped_x = x.reshape(-1, x.shape[-1])
154 - out = quantization_ops.awq_gemm(reshaped_x, qweight, scales, qzeros,
155 - pack_factor)
154 + out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor)
155 if bias is not None:
156 out = out + bias
157 return out.reshape(out_shape)
vllm/model_executor/layers/quantization/squeezellm.py
+2 -3
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
3 import torch
4 from torch.nn.parameter import Parameter
5
6 -from vllm import quantization_ops
6 +from vllm._C import ops
7 from vllm.model_executor.layers.linear import (LinearMethodBase,
8 set_weight_attrs)
9 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -116,8 +116,7 @@ class SqueezeLLMLinearMethod(LinearMethodBase):
116 reshaped_x = x.reshape(-1, x.shape[-1])
117 # NOTE: The output tensor should be zero-initialized.
118 out = torch.zeros(out_shape, device="cuda", dtype=torch.float16)
119 - quantization_ops.squeezellm_gemm(reshaped_x, qweight, out,
120 - lookup_table)
119 + ops.squeezellm_gemm(reshaped_x, qweight, out, lookup_table)
120
121 if bias is not None:
122 out = out + bias
vllm/model_executor/layers/rotary_embedding.py
+4 -5
@@ -27,7 +27,7 @@ from typing import Any, Dict, Optional, Tuple, Union
27 import torch
28 import torch.nn as nn
29
30 -from vllm import pos_encoding_ops
30 +from vllm._C import ops
31
32
33 class RotaryEmbedding(nn.Module):
@@ -87,11 +87,10 @@ class RotaryEmbedding(nn.Module):
87 query: torch.Tensor,
88 key: torch.Tensor,
89 ) -> Tuple[torch.Tensor, torch.Tensor]:
90 - # pos_encoding_ops.rotary_embedding() is an in-place operation that
90 + # ops.rotary_embedding() is an in-place operation that
91 # updates the query and key tensors.
92 - pos_encoding_ops.rotary_embedding(positions, query, key,
93 - self.head_size, self.cos_sin_cache,
94 - self.is_neox_style)
92 + ops.rotary_embedding(positions, query, key, self.head_size,
93 + self.cos_sin_cache, self.is_neox_style)
94 return query, key
95
96
vllm/utils.py
+1 -1
@@ -5,7 +5,7 @@ from platform import uname
5 import psutil
6 import torch
7
8 -from vllm import cuda_utils
8 +from vllm._C import cuda_utils
9
10
11 class Device(enum.Enum):
vllm/worker/cache_engine.py
+1 -1
@@ -3,7 +3,7 @@ from typing import Dict, List, Tuple
3
4 import torch
5
6 -from vllm import cache_ops
6 +from vllm._C import cache_ops
7 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8 from vllm.logger import init_logger
9 from vllm.utils import in_wsl