A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[torch.compile] Dynamic fp8 + rms_norm fusion (#10906)
Signed-off-by: luka <luka@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Luka Govedič committed
Dec 12, 2024 at 22:19 UTC
30870b4f66414020645608b81dced94d8a99111c
20 files changed
+1736
-252
CMakeLists.txt
+2
-1
@@ -196,6 +196,7 @@ set(VLLM_EXT_SRC
196
"csrc/quantization/gptq/q_gemm.cu"
197
"csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
198
"csrc/quantization/fp8/common.cu"
199
+ "csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu"
200
"csrc/quantization/gguf/gguf_kernel.cu"
201
"csrc/cuda_utils_kernels.cu"
202
"csrc/prepare_inputs/advance_step.cu"
@@ -300,7 +301,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
301
#
302
# For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
303
# kernels for the remaining archs that are not already built for 3x.
303
- cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
304
+ cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
305
"7.5;8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
306
# subtract out the archs that are already built for 3x
307
list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
benchmarks/fused_kernels/layernorm_rms_benchmarks.py
new
+173
@@ -0,0 +1,173 @@
1
+import pickle as pkl
2
+import time
3
+from dataclasses import dataclass
4
+from itertools import product
5
+from typing import Callable, Iterable, List, Optional
6
+
7
+import torch
8
+import torch.utils.benchmark as TBenchmark
9
+from torch.utils.benchmark import Measurement as TMeasurement
10
+from tqdm import tqdm
11
+
12
+import vllm._custom_ops as ops
13
+from vllm.model_executor.layers.layernorm import RMSNorm
14
+
15
+
16
+@dataclass
17
+class bench_params_t:
18
+ num_tokens: int
19
+ hidden_size: int
20
+ add_residual: bool
21
+ dtype: torch.dtype
22
+
23
+ def description(self):
24
+ return (f'N {self.num_tokens} '
25
+ f'x D {self.hidden_size} '
26
+ f'x R {self.add_residual} '
27
+ f'x DT {self.dtype}')
28
+
29
+
30
+def get_bench_params() -> List[bench_params_t]:
31
+ ## Test Fixtures
32
+ NUM_TOKENS = [2**x for x in range(11)]
33
+ HIDDEN_SIZES = list(range(1024, 8129, 1024))
34
+ ADD_RESIDUAL = [True, False]
35
+ DTYPES = [torch.bfloat16, torch.float]
36
+
37
+ combinations = product(NUM_TOKENS, HIDDEN_SIZES, ADD_RESIDUAL, DTYPES)
38
+ bench_params = list(map(lambda x: \
39
+ bench_params_t(x[0], x[1], x[2], x[3]), combinations))
40
+ return bench_params
41
+
42
+
43
+# Reference impls
44
+def unfused_int8_impl(rms_norm_layer: RMSNorm, x: torch.Tensor,
45
+ residual: Optional[torch.Tensor],
46
+ quant_dtype: torch.dtype):
47
+ # Norm
48
+ torch_out = None
49
+ if residual is None:
50
+ torch_out = rms_norm_layer.forward_cuda(x, residual)
51
+ else:
52
+ torch_out, _ = rms_norm_layer.forward_cuda(x, residual)
53
+
54
+ # Quant
55
+ torch_out, _, _ = ops.scaled_int8_quant(torch_out)
56
+
57
+
58
+def unfused_fp8_impl(rms_norm_layer: RMSNorm, x: torch.Tensor,
59
+ residual: Optional[torch.Tensor],
60
+ quant_dtype: torch.dtype):
61
+ # Norm
62
+ torch_out = None
63
+ if residual is None:
64
+ torch_out = rms_norm_layer.forward_cuda(x, residual)
65
+ else:
66
+ torch_out, _ = rms_norm_layer.forward_cuda(x, residual)
67
+
68
+ # Quant
69
+ torch_out, _ = ops.scaled_fp8_quant(torch_out)
70
+
71
+
72
+def fused_impl(
73
+ rms_norm_layer: RMSNorm, # this stores the weights
74
+ x: torch.Tensor,
75
+ residual: Optional[torch.Tensor],
76
+ quant_dtype: torch.dtype):
77
+ out, _ = ops.rms_norm_dynamic_per_token_quant(x,
78
+ rms_norm_layer.weight,
79
+ 1e-6,
80
+ quant_dtype,
81
+ residual=residual)
82
+
83
+
84
+# Bench functions
85
+def bench_fn(rms_norm_layer: RMSNorm, x: torch.Tensor, residual: torch.Tensor,
86
+ quant_dtype: torch.dtype, label: str, sub_label: str,
87
+ fn: Callable, description: str) -> TMeasurement:
88
+
89
+ min_run_time = 1
90
+
91
+ globals = {
92
+ "rms_norm_layer": rms_norm_layer,
93
+ "x": x,
94
+ "residual": residual,
95
+ "quant_dtype": quant_dtype,
96
+ "fn": fn,
97
+ }
98
+ return TBenchmark.Timer(
99
+ stmt="fn(rms_norm_layer, x, residual, quant_dtype)",
100
+ globals=globals,
101
+ label=label,
102
+ sub_label=sub_label,
103
+ description=description,
104
+ ).blocked_autorange(min_run_time=min_run_time)
105
+
106
+def bench(params: bench_params_t, label: str, sub_label: str) \
107
+ -> Iterable[TMeasurement]:
108
+
109
+ # Make inputs
110
+ layer = RMSNorm(params.hidden_size, 1e-6).to(dtype=params.dtype)
111
+ # Make weights
112
+ layer.weight.data.normal_(mean=1.0, std=0.1)
113
+ # Make inputs
114
+ scale = 1 / params.hidden_size
115
+ x = torch.randn(params.num_tokens,
116
+ params.hidden_size,
117
+ dtype=params.dtype,
118
+ device='cuda') * scale
119
+ residual = (torch.randn_like(x) * scale).to(device='cuda') \
120
+ if params.add_residual else None
121
+
122
+ timers = []
123
+
124
+ # unfused int8 impl.
125
+ timers.append(
126
+ bench_fn(layer, x, residual, torch.int8, label, sub_label,
127
+ unfused_int8_impl, "unfused_int8_impl"))
128
+
129
+ # unfused fp8 impl.
130
+ timers.append(
131
+ bench_fn(layer, x, residual, torch.float8_e4m3fn, label, sub_label,
132
+ unfused_fp8_impl, "unfused_fp8_impl"))
133
+
134
+ # fused int8 impl.
135
+ timers.append(
136
+ bench_fn(layer, x, residual, torch.int8, label, sub_label, fused_impl,
137
+ "fused_int8_impl"))
138
+
139
+ # fused fp8 impl.
140
+ timers.append(
141
+ bench_fn(layer, x, residual, torch.float8_e4m3fn, label, sub_label,
142
+ fused_impl, "fused_fp8_impl"))
143
+
144
+ print_timers(timers)
145
+
146
+ return timers
147
+
148
+
149
+# launch bench
150
+# runner
151
+def print_timers(timers: Iterable[TMeasurement]):
152
+ compare = TBenchmark.Compare(timers)
153
+ compare.print()
154
+
155
+
156
+def main():
157
+ torch.set_default_device('cuda')
158
+ bench_params = get_bench_params()
159
+
160
+ timers = []
161
+ for bp in tqdm(bench_params):
162
+ timers.extend(
163
+ bench(bp, "rms-norm-dynamic-per-token-quant", bp.description()))
164
+ print_timers(timers)
165
+
166
+ # pickle all the results
167
+ timestamp = int(time.time())
168
+ with open(f"rms_norm_dpt_quant-{timestamp}.pkl", "wb") as f:
169
+ pkl.dump(timers, f)
170
+
171
+
172
+if __name__ == '__main__':
173
+ main()
csrc/dispatch_utils.h
+14
@@ -14,6 +14,20 @@
14
#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15
AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
16
17
+// TODO(luka/varun): use FP8_TYPE macro after refactoring
18
+#ifndef USE_ROCM
19
+ #define VLLM_DISPATCH_CASE_QUANT_TYPES(...) \
20
+ AT_DISPATCH_CASE(at::ScalarType::Float8_e4m3fn, __VA_ARGS__) \
21
+ AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__)
22
+#else
23
+ #define VLLM_DISPATCH_CASE_QUANT_TYPES(...) \
24
+ AT_DISPATCH_CASE(at::ScalarType::Float8_e4m3fnuz, __VA_ARGS__) \
25
+ AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__)
26
+#endif
27
+
28
+#define VLLM_DISPATCH_QUANT_TYPES(TYPE, NAME, ...) \
29
+ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_QUANT_TYPES(__VA_ARGS__))
30
+
31
#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
32
AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
33
AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
csrc/ops.h
+8
@@ -66,6 +66,14 @@ void fused_add_rms_norm_static_fp8_quant(torch::Tensor& out,
66
torch::Tensor& weight,
67
torch::Tensor& scale, double epsilon);
68
69
+void rms_norm_dynamic_per_token_quant(torch::Tensor& out,
70
+ torch::Tensor const& input,
71
+ torch::Tensor const& weight,
72
+ torch::Tensor& scales,
73
+ double const epsilon,
74
+ std::optional<torch::Tensor> scale_ub,
75
+ std::optional<torch::Tensor> residual);
76
+
77
void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
78
torch::Tensor& key, int64_t head_size,
79
torch::Tensor& cos_sin_cache, bool is_neox);
csrc/quantization/fp8/common.cuh
+7
-19
@@ -1,6 +1,9 @@
1
#pragma once
2
3
+#include "quantization/vectorization.cuh"
4
+
5
#include <cmath>
6
+#include <c10/core/ScalarType.h>
7
8
#ifndef USE_ROCM
9
#include <c10/util/Float8_e4m3fn.h>
@@ -15,6 +18,7 @@ using FP8_TYPE = c10::Float8_e4m3fnuz;
18
// issue when running dynamic quantization. Here use 224.0f for rocm.
19
constexpr auto FP8_E4M3_MAX = 224.0f;
20
#endif
21
+constexpr static auto kFp8Type = c10::CppTypeToScalarType<FP8_TYPE>::value;
22
23
namespace vllm {
24
@@ -89,22 +93,6 @@ __global__ void segmented_max_reduction(float* __restrict__ scale,
93
}
94
}
95
92
-template <typename scalar_t>
93
-struct __align__(8) vec4_t {
94
- scalar_t x;
95
- scalar_t y;
96
- scalar_t z;
97
- scalar_t w;
98
-};
99
-
100
-typedef struct __align__(4) {
101
- FP8_TYPE x;
102
- FP8_TYPE y;
103
- FP8_TYPE z;
104
- FP8_TYPE w;
105
-}
106
-float8x4_t;
107
-
96
template <typename scalar_t>
97
__device__ float thread_max_vec(scalar_t const* __restrict__ input,
98
int64_t const num_elems, int const tid,
@@ -139,10 +127,10 @@ __device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
127
float const scale,
128
int64_t const num_elems,
129
int const tid, int const step) {
130
+ using float8x4_t = q8x4_t<FP8_TYPE>;
131
// Vectorized input/output to better utilize memory bandwidth.
143
- vec4_t<scalar_t> const* vectorized_in =
144
- reinterpret_cast<vec4_t<scalar_t> const*>(input);
145
- float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
132
+ auto const* vectorized_in = reinterpret_cast<vec4_t<scalar_t> const*>(input);
133
+ auto* vectorized_out = reinterpret_cast<float8x4_t*>(out);
134
135
int64_t const num_vec_elems = num_elems >> 2;
136
csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu
new
+160
@@ -0,0 +1,160 @@
1
+
2
+#include <ATen/cuda/CUDAContext.h>
3
+#include <c10/cuda/CUDAGuard.h>
4
+
5
+#include "../../dispatch_utils.h"
6
+#include "layernorm_utils.cuh"
7
+#include "quant_conversions.cuh"
8
+
9
+namespace vllm {
10
+
11
+template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
12
+__device__ void rms_norm_dynamic_per_token_quant_vec(
13
+ scalar_out_t* __restrict__ out, // [..., hidden_size]
14
+ float* __restrict__ scales, // [num_tokens]
15
+ scalar_t const* __restrict__ input, // [..., hidden_size]
16
+ scalar_t const* __restrict__ weight, // [hidden_size]
17
+ float const* scale_ub, float const var_epsilon,
18
+ float const min_scaling_factor, int32_t const hidden_size,
19
+ scalar_t* __restrict__ residual = nullptr) {
20
+ float rms = 0.0f;
21
+ float token_scale = 0.0f;
22
+
23
+ // Compute rms
24
+ vllm::vectorized::compute_rms<scalar_t, has_residual>(
25
+ &rms, input, hidden_size, var_epsilon, residual);
26
+
27
+ // Compute scale
28
+ vllm::vectorized::compute_dynamic_per_token_scales<scalar_t, scalar_out_t,
29
+ has_residual>(
30
+ &token_scale, scales, input, weight, rms, scale_ub, min_scaling_factor,
31
+ hidden_size, residual);
32
+
33
+ // RMS Norm + Quant
34
+ if constexpr (std::is_same_v<scalar_out_t, int8_t>) {
35
+ vllm::vectorized::norm_and_quant<scalar_t, scalar_out_t, true,
36
+ has_residual>(
37
+ out, input, weight, rms, 1.0f / token_scale, hidden_size, residual);
38
+ } else {
39
+ // FP8 - Do not invert token_scale for exact match with FBGemm
40
+ vllm::vectorized::norm_and_quant<scalar_t, scalar_out_t, false,
41
+ has_residual>(
42
+ out, input, weight, rms, token_scale, hidden_size, residual);
43
+ }
44
+}
45
+
46
+// RMS norm + quant kernel
47
+template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
48
+__global__ void rms_norm_dynamic_per_token_quant_kernel(
49
+ scalar_out_t* __restrict__ out, // [..., hidden_size]
50
+ float* __restrict__ scales, // [num_tokens]
51
+ scalar_t const* __restrict__ input, // [..., hidden_size]
52
+ scalar_t const* __restrict__ weight, // [hidden_size]
53
+ float const* scale_ub, float const var_epsilon,
54
+ float const min_scaling_factor, int32_t const hidden_size,
55
+ scalar_t* __restrict__ residual = nullptr) {
56
+ // For vectorization, token_input and token_output pointers need to be
57
+ // aligned at 8-byte and 4-byte addresses respectively.
58
+ bool const can_vectorize = hidden_size % 4 == 0;
59
+
60
+ if (can_vectorize) {
61
+ return rms_norm_dynamic_per_token_quant_vec<scalar_t, scalar_out_t,
62
+ has_residual>(
63
+ out, scales, input, weight, scale_ub, var_epsilon, min_scaling_factor,
64
+ hidden_size, residual);
65
+ }
66
+
67
+ float rms = 0.0f;
68
+ float token_scale = 0.0f;
69
+
70
+ // Compute RMS
71
+ vllm::compute_rms<scalar_t, has_residual>(&rms, input, hidden_size,
72
+ var_epsilon, residual);
73
+ // Compute Scale
74
+ vllm::compute_dynamic_per_token_scales<scalar_t, scalar_out_t, has_residual>(
75
+ &token_scale, scales, input, weight, rms, scale_ub, min_scaling_factor,
76
+ hidden_size, residual);
77
+
78
+ // RMS Norm + Quant
79
+ if constexpr (std::is_same_v<scalar_out_t, int8_t>) {
80
+ vllm::norm_and_quant<scalar_t, scalar_out_t, true, has_residual>(
81
+ out, input, weight, rms, 1.0f / token_scale, hidden_size, residual);
82
+ } else {
83
+ // FP8 - Do not invert s_token_scale for exact match with FBGemm
84
+ vllm::norm_and_quant<scalar_t, scalar_out_t, false, has_residual>(
85
+ out, input, weight, rms, token_scale, hidden_size, residual);
86
+ }
87
+}
88
+} // namespace vllm
89
+
90
+// Residual add + RMS norm + dynamic per token
91
+template <typename scalar_in_t>
92
+void rms_norm_dynamic_per_token_quant_dispatch(
93
+ torch::Tensor& out, // [..., hidden_size]
94
+ torch::Tensor const& input, // [..., hidden_size]
95
+ torch::Tensor const& weight, // [hidden_size]
96
+ torch::Tensor& scales, // [num_tokens]
97
+ double const var_epsilon, // Variance epsilon used in norm calculation
98
+ std::optional<at::Tensor> const& scale_ub,
99
+ std::optional<at::Tensor>& residual) {
100
+ int32_t hidden_size = input.size(-1);
101
+ int32_t num_tokens = input.numel() / hidden_size;
102
+
103
+ dim3 grid(num_tokens);
104
+ dim3 block(std::min(hidden_size, 1024));
105
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
106
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
107
+
108
+ const float min_scaling_factor =
109
+ out.dtype() == torch::kInt8
110
+ ? std::numeric_limits<float>::epsilon()
111
+ : 1.0f / (std::numeric_limits<c10::Float8_e4m3fn>::max() * 512.f);
112
+
113
+ if (residual.has_value()) {
114
+ VLLM_DISPATCH_QUANT_TYPES(
115
+ out.scalar_type(), "rms_norm_dynamic_per_token_quant_kernel", [&] {
116
+ vllm::rms_norm_dynamic_per_token_quant_kernel<scalar_in_t, scalar_t,
117
+ true>
118
+ <<<grid, block, 0, stream>>>(
119
+ out.data_ptr<scalar_t>(), scales.data_ptr<float>(),
120
+ input.data_ptr<scalar_in_t>(), weight.data_ptr<scalar_in_t>(),
121
+ scale_ub.has_value() ? scale_ub->data_ptr<float>() : nullptr,
122
+ var_epsilon, min_scaling_factor, hidden_size,
123
+ residual->data_ptr<scalar_in_t>());
124
+ });
125
+
126
+ } else {
127
+ VLLM_DISPATCH_QUANT_TYPES(
128
+ out.scalar_type(), "rms_norm_dynamic_per_token_quant_kernel", [&] {
129
+ vllm::rms_norm_dynamic_per_token_quant_kernel<scalar_in_t, scalar_t,
130
+ false>
131
+ <<<grid, block, 0, stream>>>(
132
+ out.data_ptr<scalar_t>(), scales.data_ptr<float>(),
133
+ input.data_ptr<scalar_in_t>(), weight.data_ptr<scalar_in_t>(),
134
+ scale_ub.has_value() ? scale_ub->data_ptr<float>() : nullptr,
135
+ var_epsilon, min_scaling_factor, hidden_size, nullptr);
136
+ });
137
+ }
138
+}
139
+
140
+void rms_norm_dynamic_per_token_quant(
141
+ torch::Tensor& out, // [..., hidden_size]
142
+ torch::Tensor const& input, // [..., hidden_size]
143
+ torch::Tensor const& weight, // [hidden_size]
144
+ torch::Tensor& scales, // [num_tokens]
145
+ double const var_epsilon, // Variance epsilon used in norm calculation
146
+ std::optional<at::Tensor> scale_ub, std::optional<at::Tensor> residual) {
147
+ TORCH_CHECK(out.dtype() == kFp8Type || out.dtype() == torch::kInt8);
148
+ TORCH_CHECK(out.is_contiguous() && input.is_contiguous());
149
+
150
+ if (scale_ub.has_value()) {
151
+ TORCH_CHECK(out.dtype() == kFp8Type);
152
+ }
153
+ TORCH_CHECK(scales.dtype() == torch::kFloat32);
154
+
155
+ VLLM_DISPATCH_FLOATING_TYPES(
156
+ input.scalar_type(), "rms_norm_dynamic_per_token_quant_dispatch", [&] {
157
+ rms_norm_dynamic_per_token_quant_dispatch<scalar_t>(
158
+ out, input, weight, scales, var_epsilon, scale_ub, residual);
159
+ });
160
+}
csrc/quantization/fused_kernels/layernorm_utils.cuh
new
+327
@@ -0,0 +1,327 @@
1
+#pragma once
2
+
3
+/**
4
+ * __device__ layernorm utilities.
5
+ */
6
+
7
+#include "quantization/vectorization.cuh"
8
+#include "quant_conversions.cuh"
9
+
10
+#ifndef USE_ROCM
11
+ #include <cub/cub.cuh>
12
+#else
13
+ #include <hipcub/hipcub.hpp>
14
+#endif
15
+
16
+namespace vllm {
17
+
18
+// has_residual must be true, if residual is not a nullptr
19
+template <typename scalar_t, bool has_residual = false>
20
+__device__ void compute_rms(float* rms, scalar_t const* __restrict__ input,
21
+ int32_t const hidden_size, float const epsilon,
22
+ scalar_t const* __restrict__ residual = nullptr) {
23
+ int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
24
+ // sum of squares
25
+ float ss = 0.0f;
26
+
27
+ for (int32_t i = threadIdx.x; i < hidden_size; i += blockDim.x) {
28
+ float x = static_cast<float>(input[token_offset + i]);
29
+ if constexpr (has_residual) {
30
+ x += static_cast<float>(residual[token_offset + i]);
31
+ }
32
+
33
+ ss += x * x;
34
+ }
35
+
36
+ using BlockReduce = cub::BlockReduce<float, 1024>;
37
+ __shared__ typename BlockReduce::TempStorage reduceStore;
38
+ ss = BlockReduce(reduceStore).Reduce(ss, cub::Sum{}, blockDim.x);
39
+
40
+ __shared__ float s_rms;
41
+ if (threadIdx.x == 0) {
42
+ s_rms = rsqrtf(ss / hidden_size + epsilon);
43
+ }
44
+ __syncthreads();
45
+
46
+ *rms = s_rms;
47
+}
48
+
49
+template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
50
+__device__ void compute_dynamic_per_token_scales(
51
+ float* __restrict__ token_scale, float* __restrict__ all_token_scales,
52
+ scalar_t const* __restrict__ input, scalar_t const* __restrict__ weight,
53
+ float const rms, float const* __restrict__ scale_ub,
54
+ float const min_scaling_factor, int32_t const hidden_size,
55
+ scalar_t const* __restrict__ residual = nullptr) {
56
+ int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
57
+ ;
58
+ constexpr scalar_out_t qmax{std::numeric_limits<scalar_out_t>::max()};
59
+
60
+ float block_absmax_val_maybe = 0.0f;
61
+ for (int32_t i = threadIdx.x; i < hidden_size; i += blockDim.x) {
62
+ float x = static_cast<float>(input[token_offset + i]);
63
+ if constexpr (has_residual) {
64
+ x += static_cast<float>(residual[token_offset + i]);
65
+ }
66
+
67
+ x = static_cast<float>(static_cast<scalar_t>(x * rms) * weight[i]);
68
+ block_absmax_val_maybe = fmaxf(block_absmax_val_maybe, fabsf(x));
69
+ }
70
+
71
+ using BlockReduce = cub::BlockReduce<float, 1024>;
72
+ __shared__ typename BlockReduce::TempStorage reduceStore;
73
+ block_absmax_val_maybe =
74
+ BlockReduce(reduceStore)
75
+ .Reduce(block_absmax_val_maybe, cub::Max{}, blockDim.x);
76
+
77
+ __shared__ float s_token_scale;
78
+ if (threadIdx.x == 0) {
79
+ float scale = 0.0f;
80
+ if (scale_ub) {
81
+ scale = min(block_absmax_val_maybe, *scale_ub);
82
+ } else {
83
+ scale = block_absmax_val_maybe;
84
+ }
85
+ // token scale computation
86
+ scale = max(scale / qmax, min_scaling_factor);
87
+ s_token_scale = scale; // Shared memory store
88
+ all_token_scales[blockIdx.x] = scale; // Global output store
89
+ }
90
+ __syncthreads();
91
+
92
+ *token_scale = s_token_scale;
93
+}
94
+
95
+template <typename scalar_t, typename scalar_out_t, bool is_scale_inverted,
96
+ bool has_residual = false>
97
+__device__ void norm_and_quant(scalar_out_t* __restrict__ output,
98
+ scalar_t const* __restrict__ input,
99
+ scalar_t const* __restrict__ weight,
100
+ float const rms, float const scale,
101
+ int32_t const hidden_size,
102
+ scalar_t* __restrict__ residual = nullptr) {
103
+ int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
104
+ ;
105
+
106
+ for (int32_t i = threadIdx.x; i < hidden_size; i += blockDim.x) {
107
+ float x = static_cast<float>(input[token_offset + i]);
108
+ if constexpr (has_residual) {
109
+ x += static_cast<float>(residual[token_offset + i]);
110
+ residual[token_offset + i] = static_cast<scalar_t>(x);
111
+ }
112
+ // Norm
113
+ x = static_cast<float>(static_cast<scalar_t>(x * rms) * weight[i]);
114
+ // Quant
115
+ output[token_offset + i] =
116
+ ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(x, scale);
117
+ }
118
+}
119
+
120
+namespace vectorized {
121
+
122
+// Compute 1.0/rms(input)
123
+// hidden_size must be a multiple of 4
124
+template <typename scalar_t, bool has_residual = false>
125
+__device__ void compute_rms(float* rms, scalar_t const* __restrict__ input,
126
+ int32_t const hidden_size, float const epsilon,
127
+ scalar_t const* __restrict__ residual = nullptr) {
128
+ int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
129
+
130
+ // Vectorized input/output to better utilize memory bandwidth.
131
+ vec4_t<scalar_t> const* vec_input =
132
+ reinterpret_cast<vec4_t<scalar_t> const*>(&input[token_offset]);
133
+ vec4_t<scalar_t> const* vec_residual = nullptr;
134
+ if constexpr (has_residual) {
135
+ vec_residual =
136
+ reinterpret_cast<vec4_t<scalar_t> const*>(&residual[token_offset]);
137
+ }
138
+
139
+ // sum of squares
140
+ float ss = 0.0f;
141
+
142
+ int32_t const num_vec_elems = hidden_size >> 2;
143
+
144
+#pragma unroll 4
145
+ for (int32_t i = threadIdx.x; i < num_vec_elems; i += blockDim.x) {
146
+ vec4_t<scalar_t> in = vec_input[i];
147
+
148
+ vec4_t<float> x;
149
+ x.x = static_cast<float>(in.x);
150
+ x.y = static_cast<float>(in.y);
151
+ x.z = static_cast<float>(in.z);
152
+ x.w = static_cast<float>(in.w);
153
+ if constexpr (has_residual) {
154
+ vec4_t<scalar_t> r = vec_residual[i];
155
+ x.x += static_cast<float>(r.x);
156
+ x.y += static_cast<float>(r.y);
157
+ x.z += static_cast<float>(r.z);
158
+ x.w += static_cast<float>(r.w);
159
+ }
160
+
161
+ ss += x.x * x.x;
162
+ ss += x.y * x.y;
163
+ ss += x.z * x.z;
164
+ ss += x.w * x.w;
165
+ }
166
+
167
+ using BlockReduce = cub::BlockReduce<float, 1024>;
168
+ __shared__ typename BlockReduce::TempStorage reduceStore;
169
+ ss = BlockReduce(reduceStore).Reduce(ss, cub::Sum{}, blockDim.x);
170
+
171
+ __shared__ float s_rms;
172
+ if (threadIdx.x == 0) {
173
+ s_rms = rsqrtf(ss / hidden_size + epsilon);
174
+ }
175
+ __syncthreads();
176
+
177
+ *rms = s_rms;
178
+}
179
+
180
+// Vectorized version of vllm::compute_dynamic_per_token_scales
181
+// hidden_size must be a multiple of 4
182
+template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
183
+__device__ void compute_dynamic_per_token_scales(
184
+ float* __restrict__ token_scale, float* __restrict__ all_token_scales,
185
+ scalar_t const* __restrict__ input, scalar_t const* __restrict__ weight,
186
+ float const rms, float const* __restrict__ scale_ub,
187
+ float const min_scaling_factor, int32_t const hidden_size,
188
+ scalar_t const* __restrict__ residual = nullptr) {
189
+ int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
190
+ ;
191
+
192
+ // Vectorized input/weight/residual to better utilize memory bandwidth.
193
+ vec4_t<scalar_t> const* vec_input =
194
+ reinterpret_cast<vec4_t<scalar_t> const*>(&input[token_offset]);
195
+ vec4_t<scalar_t> const* vec_weight =
196
+ reinterpret_cast<vec4_t<scalar_t> const*>(weight);
197
+ vec4_t<scalar_t> const* vec_residual = nullptr;
198
+ if constexpr (has_residual) {
199
+ vec_residual =
200
+ reinterpret_cast<vec4_t<scalar_t> const*>(&residual[token_offset]);
201
+ }
202
+
203
+ constexpr scalar_out_t qmax{std::numeric_limits<scalar_out_t>::max()};
204
+
205
+ int32_t const num_vec_elems = hidden_size >> 2;
206
+ float block_absmax_val_maybe = 0.0f;
207
+
208
+#pragma unroll 4
209
+ for (int32_t i = threadIdx.x; i < num_vec_elems; i += blockDim.x) {
210
+ vec4_t<scalar_t> in = vec_input[i];
211
+ vec4_t<scalar_t> const w = vec_weight[i];
212
+
213
+ vec4_t<float> x;
214
+ x.x = static_cast<float>(in.x);
215
+ x.y = static_cast<float>(in.y);
216
+ x.z = static_cast<float>(in.z);
217
+ x.w = static_cast<float>(in.w);
218
+ if constexpr (has_residual) {
219
+ vec4_t<scalar_t> r = vec_residual[i];
220
+ x.x += static_cast<float>(r.x);
221
+ x.y += static_cast<float>(r.y);
222
+ x.z += static_cast<float>(r.z);
223
+ x.w += static_cast<float>(r.w);
224
+ }
225
+
226
+ block_absmax_val_maybe = fmaxf(
227
+ block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.x * rms) * w.x));
228
+ block_absmax_val_maybe = fmaxf(
229
+ block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.y * rms) * w.y));
230
+ block_absmax_val_maybe = fmaxf(
231
+ block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.z * rms) * w.z));
232
+ block_absmax_val_maybe = fmaxf(
233
+ block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.w * rms) * w.w));
234
+ }
235
+
236
+ using BlockReduce = cub::BlockReduce<float, 1024>;
237
+ __shared__ typename BlockReduce::TempStorage reduceStore;
238
+ block_absmax_val_maybe =
239
+ BlockReduce(reduceStore)
240
+ .Reduce(block_absmax_val_maybe, cub::Max{}, blockDim.x);
241
+
242
+ __shared__ float s_token_scale;
243
+ if (threadIdx.x == 0) {
244
+ float scale = 0.0f;
245
+ if (scale_ub) {
246
+ scale = min(block_absmax_val_maybe, *scale_ub);
247
+ } else {
248
+ scale = block_absmax_val_maybe;
249
+ }
250
+ // token scale computation
251
+ scale = max(scale / qmax, min_scaling_factor);
252
+ s_token_scale = scale; // shared memory store
253
+ all_token_scales[blockIdx.x] = scale; // global output store
254
+ }
255
+ __syncthreads();
256
+
257
+ *token_scale = s_token_scale;
258
+}
259
+
260
+// hidden_size must be a multiple of 4
261
+template <typename scalar_t, typename scalar_out_t, bool is_scale_inverted,
262
+ bool has_residual = false>
263
+__device__ void norm_and_quant(scalar_out_t* __restrict__ output,
264
+ scalar_t const* __restrict__ input,
265
+ scalar_t const* __restrict__ weight,
266
+ float const rms, float const scale,
267
+ int32_t const hidden_size,
268
+ scalar_t* __restrict__ residual = nullptr) {
269
+ int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
270
+ ;
271
+
272
+ // Vectorized input/output/weight/residual to better utilize memory bandwidth.
273
+ vec4_t<scalar_t> const* vec_input =
274
+ reinterpret_cast<vec4_t<scalar_t> const*>(&input[token_offset]);
275
+ vec4_t<scalar_t> const* vec_weight =
276
+ reinterpret_cast<vec4_t<scalar_t> const*>(weight);
277
+ q8x4_t<scalar_out_t>* vec_output =
278
+ reinterpret_cast<q8x4_t<scalar_out_t>*>(&output[token_offset]);
279
+ vec4_t<scalar_t>* vec_residual = nullptr;
280
+ if constexpr (has_residual) {
281
+ vec_residual = reinterpret_cast<vec4_t<scalar_t>*>(&residual[token_offset]);
282
+ }
283
+
284
+ int32_t const num_vec_elems = hidden_size >> 2;
285
+
286
+// TODO(luka/varun) extract into type-agnostic vectorized quant function to
287
+// replace scaled_fp8_conversion_vec
288
+#pragma unroll 4
289
+ for (int32_t i = threadIdx.x; i < num_vec_elems; i += blockDim.x) {
290
+ vec4_t<scalar_t> const in = vec_input[i];
291
+ vec4_t<scalar_t> const w = vec_weight[i];
292
+
293
+ vec4_t<float> x;
294
+ x.x = static_cast<float>(in.x);
295
+ x.y = static_cast<float>(in.y);
296
+ x.z = static_cast<float>(in.z);
297
+ x.w = static_cast<float>(in.w);
298
+ if constexpr (has_residual) {
299
+ vec4_t<scalar_t> r = vec_residual[i];
300
+ x.x += static_cast<float>(r.x);
301
+ x.y += static_cast<float>(r.y);
302
+ x.z += static_cast<float>(r.z);
303
+ x.w += static_cast<float>(r.w);
304
+ // Update residual
305
+ r.x = static_cast<scalar_t>(x.x);
306
+ r.y = static_cast<scalar_t>(x.y);
307
+ r.z = static_cast<scalar_t>(x.z);
308
+ r.w = static_cast<scalar_t>(x.w);
309
+ vec_residual[i] = r;
310
+ }
311
+
312
+ q8x4_t<scalar_out_t> out;
313
+ out.x = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
314
+ static_cast<scalar_t>(x.x * rms) * w.x, scale);
315
+ out.y = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
316
+ static_cast<scalar_t>(x.y * rms) * w.y, scale);
317
+ out.z = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
318
+ static_cast<scalar_t>(x.z * rms) * w.z, scale);
319
+ out.w = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
320
+ static_cast<scalar_t>(x.w * rms) * w.w, scale);
321
+ vec_output[i] = out;
322
+ }
323
+}
324
+
325
+} // namespace vectorized
326
+
327
+} // namespace vllm
csrc/quantization/fused_kernels/quant_conversions.cuh
new
+81
@@ -0,0 +1,81 @@
1
+#pragma once
2
+
3
+/**
4
+ * __device__ helper functions to deal with float -> quant datatype conversion
5
+ */
6
+
7
+#include "quantization/vectorization.cuh"
8
+// TODO(luka/varun):refactor common.cuh to use this file instead
9
+#include "quantization/fp8/common.cuh"
10
+
11
+namespace vllm {
12
+
13
+// TODO(luka/varun): combine into common utilities for int8
14
+// (with int8_quant_kernels.cu)
15
+static __device__ __forceinline__ int8_t float_to_int8_rn(float const x) {
16
+#ifdef USE_ROCM
17
+ static const float i8_min =
18
+ static_cast<float>(std::numeric_limits<int8_t>::min());
19
+ static const float i8_max =
20
+ static_cast<float>(std::numeric_limits<int8_t>::max());
21
+ // round
22
+ float dst = std::nearbyint(x);
23
+ // saturate
24
+ dst = std::clamp(dst, i8_min, i8_max);
25
+ return static_cast<int8_t>(dst);
26
+#else
27
+ // CUDA path
28
+ uint32_t dst;
29
+ asm volatile("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(dst) : "f"(x));
30
+ return reinterpret_cast<const int8_t&>(dst);
31
+#endif
32
+}
33
+
34
+static __device__ __forceinline__ FP8_TYPE float_to_fp8(float const x) {
35
+ float const r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
36
+ return static_cast<FP8_TYPE>(r);
37
+}
38
+
39
+template <typename quant_type_t, bool is_scale_inverted, typename enable = void>
40
+struct ScaledQuant;
41
+
42
+template <typename quant_type_t, bool is_scale_inverted>
43
+struct ScaledQuant<
44
+ quant_type_t, is_scale_inverted,
45
+ typename std::enable_if_t<std::is_same_v<quant_type_t, int8_t>>> {
46
+ static __device__ __forceinline__ quant_type_t quant_fn(float const x,
47
+ float const scale) {
48
+ if constexpr (is_scale_inverted) {
49
+ return float_to_int8_rn(x * scale);
50
+ } else {
51
+ return float_to_int8_rn(x / scale);
52
+ }
53
+ }
54
+};
55
+
56
+template <typename quant_type_t, bool is_scale_inverted>
57
+struct ScaledQuant<
58
+ quant_type_t, is_scale_inverted,
59
+ typename std::enable_if_t<std::is_same_v<quant_type_t, FP8_TYPE>>> {
60
+ static __device__ __forceinline__ quant_type_t quant_fn(float const x,
61
+ float const scale) {
62
+ if constexpr (is_scale_inverted) {
63
+ return float_to_fp8(x * scale);
64
+ } else {
65
+ return float_to_fp8(x / scale);
66
+ }
67
+ }
68
+};
69
+
70
+template <typename scalar_t, typename quant_type_t, bool is_scale_inverted>
71
+__device__ void scaled_quant_conversion(quant_type_t* __restrict__ output,
72
+ scalar_t const* __restrict__ input,
73
+ float const scale, int const tid,
74
+ int const num_elements,
75
+ int const step) {
76
+ for (int i = tid; i < num_elements; i += step) {
77
+ output[i] = ScaledQuant<quant_type_t, is_scale_inverted>(input[i], scale);
78
+ }
79
+}
80
+
81
+} // namespace vllm
csrc/quantization/vectorization.cuh
new
+33
@@ -0,0 +1,33 @@
1
+#pragma once
2
+/**
3
+ * __device__ datatypes vectorized by 4
4
+ */
5
+
6
+// Include both AMD and NVIDIA fp8 types to avoid circular import
7
+// TODO(luka/varun) use FP8_TYPE instead after refactoring
8
+#include <c10/util/Float8_e4m3fnuz.h>
9
+#include <c10/util/Float8_e4m3fn.h>
10
+
11
+namespace vllm {
12
+
13
+// Vectorization containers
14
+template <typename scalar_t>
15
+struct __align__(8) vec4_t {
16
+ scalar_t x;
17
+ scalar_t y;
18
+ scalar_t z;
19
+ scalar_t w;
20
+};
21
+
22
+template <typename quant_type_t>
23
+struct __align__(4) q8x4_t {
24
+ static_assert(std::is_same_v<quant_type_t, int8_t> ||
25
+ std::is_same_v<quant_type_t, c10::Float8_e4m3fn> ||
26
+ std::is_same_v<quant_type_t, c10::Float8_e4m3fnuz>);
27
+ quant_type_t x;
28
+ quant_type_t y;
29
+ quant_type_t z;
30
+ quant_type_t w;
31
+};
32
+
33
+} // namespace vllm
csrc/torch_bindings.cpp
+8
@@ -128,6 +128,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
128
ops.impl("fused_add_rms_norm_static_fp8_quant", torch::kCUDA,
129
&fused_add_rms_norm_static_fp8_quant);
130
131
+ // Fused Layernorm + Quant kernels
132
+ ops.def(
133
+ "rms_norm_dynamic_per_token_quant(Tensor! result, Tensor input, "
134
+ "Tensor weight, Tensor! scale, float epsilon, "
135
+ "Tensor? scale_ub, Tensor!? residual) -> ()");
136
+ ops.impl("rms_norm_dynamic_per_token_quant", torch::kCUDA,
137
+ &rms_norm_dynamic_per_token_quant);
138
+
139
// Rotary embedding
140
// Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
141
ops.def(
tests/compile/test_functionalization.py
+13
-8
@@ -4,10 +4,10 @@ import torch
4
import vllm.envs as envs
5
from vllm import LLM, SamplingParams
6
from vllm.compilation.fix_functionalization import FixFunctionalizationPass
7
-from vllm.compilation.fusion import (FusionPass, find_auto_fn,
8
- find_auto_fn_maybe)
7
+from vllm.compilation.fusion import (FUSED_OPS, FusionPass, QuantKey,
8
+ kFp8DynamicTokenSym, kFp8StaticTensorSym)
9
+from vllm.compilation.fx_utils import find_auto_fn, find_auto_fn_maybe, is_func
10
from vllm.compilation.reshapes import RedundantReshapesPass
10
-from vllm.compilation.vllm_inductor_pass import is_func
11
from vllm.config import CompilationConfig
12
13
from .backend import TestBackend
@@ -35,12 +35,16 @@ prompts = [
35
]
36
37
38
-@pytest.mark.parametrize("model",
39
- ["nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8-e2e"])
38
+@pytest.mark.parametrize(
39
+ "model, quant_key",
40
+ [("nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8-e2e", kFp8StaticTensorSym),
41
+ ("nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8_DYNAMIC-e2e",
42
+ kFp8DynamicTokenSym)])
43
@pytest.mark.parametrize("do_fusion", [True, False])
44
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
45
reason="Only test on CUDA")
43
-def test_fix_functionalization(model: str, do_fusion: bool):
46
+def test_fix_functionalization(model: str, quant_key: QuantKey,
47
+ do_fusion: bool):
48
torch.set_default_device("cuda")
49
50
config = CompilationConfig.PassConfig(enable_fusion=do_fusion,
@@ -78,8 +82,9 @@ def test_fix_functionalization(model: str, do_fusion: bool):
82
83
# OPS_IN_MODEL always appear. RMS_OP is fused away if we run fusion,
84
# and replaced by fused quantized ops in RMS_QUANT_OPS.
81
- ops = OPS_IN_MODEL + (RMS_QUANT_OPS["static_fp8"]
82
- if do_fusion else [RMS_OP])
85
+ rms_ops = [FUSED_OPS[(quant_key, True)], FUSED_OPS[(quant_key, False)]
86
+ ] if do_fusion else [RMS_OP]
87
+ ops = OPS_IN_MODEL + rms_ops
88
89
for op in ops:
90
find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
tests/compile/test_fusion.py
+43
-18
@@ -3,8 +3,9 @@ import torch
3
from compressed_tensors.quantization import FP8_DTYPE
4
5
import vllm.envs as envs
6
-from vllm.compilation.fusion import (FusionPass, find_auto_fn,
7
- find_auto_fn_maybe)
6
+from vllm.compilation.fusion import (FUSED_OPS, QUANT_OPS, FusedRMSQuantKey,
7
+ FusionPass, QuantKey)
8
+from vllm.compilation.fx_utils import find_auto_fn, find_auto_fn_maybe
9
from vllm.compilation.reshapes import RedundantReshapesPass
10
from vllm.config import CompilationConfig
11
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -16,24 +17,37 @@ from .backend import TestBackend
17
18
class TestModel(torch.nn.Module):
19
19
- def __init__(self, hidden_size: int, eps: float, *args, **kwargs):
20
+ def __init__(self, hidden_size: int, eps: float, static: bool, *args,
21
+ **kwargs):
22
super().__init__(*args, **kwargs)
23
self.norm = [RMSNorm(hidden_size, eps) for _ in range(3)]
22
- self.scale = [torch.rand(1, dtype=torch.float32) for _ in range(4)]
24
+ self.wscale = [torch.rand(1, dtype=torch.float32) for _ in range(2)]
25
+ if static:
26
+ self.scale = [torch.rand(1, dtype=torch.float32) for _ in range(2)]
27
+ else:
28
+ self.scale = [None for _ in range(2)]
29
self.w = [
30
torch.rand(hidden_size, hidden_size).to(dtype=FP8_DTYPE).t()
31
for _ in range(2)
32
]
33
34
def forward(self, x):
29
- resid = torch.relu(x)
35
+ resid = torch.sqrt(x)
36
y = self.norm[0](x)
37
32
- x2 = apply_fp8_linear(y, self.w[0], self.scale[0], self.scale[1])
38
+ x2 = apply_fp8_linear(y,
39
+ self.w[0],
40
+ self.wscale[0],
41
+ self.scale[0],
42
+ use_per_token_if_dynamic=True)
43
# make sure resid is used for replacement to work
44
y2, resid = self.norm[1](x2, resid)
45
36
- x3 = apply_fp8_linear(y2, self.w[1], self.scale[2], self.scale[3])
46
+ x3 = apply_fp8_linear(y2,
47
+ self.w[1],
48
+ self.wscale[1],
49
+ self.scale[1],
50
+ use_per_token_if_dynamic=True)
51
y3, resid = self.norm[2](x3, resid) # use resid here
52
return y3
53
@@ -42,14 +56,13 @@ class TestModel(torch.nn.Module):
56
@pytest.mark.parametrize("hidden_size", [64, 3392, 4096])
57
@pytest.mark.parametrize("num_tokens", [7, 256, 533, 2048, 2049])
58
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
59
+@pytest.mark.parametrize("static", [True, False])
60
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
61
reason="Only test on CUDA")
47
-def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
62
+def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps, static):
63
torch.set_default_device("cuda")
49
- torch.set_default_dtype(torch.float16)
50
-
51
- if eps != 1e-5:
52
- pytest.skip("Only test eps=1e-5 for now")
64
+ torch.set_default_dtype(dtype)
65
+ torch.manual_seed(1)
66
67
# Reshape pass is needed for the fusion pass to work
68
config = CompilationConfig.PassConfig(enable_fusion=True,
@@ -58,7 +71,7 @@ def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
71
fusion_pass = FusionPass.instance(config)
72
73
backend = TestBackend(reshape_pass, fusion_pass)
61
- model = TestModel(hidden_size, eps)
74
+ model = TestModel(hidden_size, eps, static)
75
76
# First dimension dynamic
77
x = torch.rand(num_tokens, hidden_size)
@@ -69,16 +82,28 @@ def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
82
model2 = torch.compile(model, backend=backend)
83
result2 = model2(x)
84
72
- # Check that it gives the same answer
73
- torch.testing.assert_close(result, result2, atol=1e-3, rtol=1e-3)
85
+ # Higher tol for dynamic, even higher for bfloat16
86
+ if static:
87
+ ATOL, RTOL = (1e-3, 1e-3)
88
+ elif dtype == torch.float16:
89
+ ATOL, RTOL = (2e-3, 2e-3)
90
+ else:
91
+ ATOL, RTOL = (1e-2, 1e-2)
92
+
93
+ torch.testing.assert_close(result, result2, atol=ATOL, rtol=RTOL)
94
95
# Check substitution worked
96
pre_nodes = backend.graph_pre_pass.nodes
97
post_nodes = backend.graph_post_pass.nodes
98
79
- rms_quant = torch.ops._C.rms_norm_static_fp8_quant.default
80
- add_rms_quant = torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
81
- fp8_quant = torch.ops._C.static_scaled_fp8_quant.default
99
+ # static is per-tensor, dynamic is per-token
100
+ key = QuantKey(dtype=FP8_DTYPE,
101
+ static=static,
102
+ per_tensor=static,
103
+ symmetric=True)
104
+ rms_quant = FUSED_OPS[FusedRMSQuantKey(key, False)]
105
+ add_rms_quant = FUSED_OPS[FusedRMSQuantKey(key, True)]
106
+ fp8_quant = QUANT_OPS[key]
107
108
# In pre-nodes, fp8 quant should be present and fused kernels should not
109
assert find_auto_fn_maybe(pre_nodes, rms_quant) is None
tests/kernels/test_fused_quant_layernorm.py
new
+171
@@ -0,0 +1,171 @@
1
+from typing import Optional, Tuple, Union
2
+
3
+import pytest
4
+import torch
5
+
6
+import vllm._custom_ops as ops
7
+from tests.kernels.utils import opcheck
8
+from vllm.model_executor.layers.layernorm import RMSNorm
9
+
10
+DTYPES = [torch.bfloat16, torch.float]
11
+QUANT_DTYPES = [torch.int8, torch.float8_e4m3fn]
12
+VEC_HIDDEN_SIZES = range(1024, 1030)
13
+# Avoid combinatorial explosion with full Cartesian product
14
+NUM_TOKENS_HIDDEN_SIZES = [
15
+ *[(1, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5120, 5137]],
16
+ *[(83, i) for i in [1, 1033, 2048, 5120]],
17
+ *[(2048, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5137]],
18
+ *[(4096, i) for i in [1, 64, 5137]],
19
+]
20
+
21
+ADD_RESIDUAL = [False, True]
22
+SCALE_UBS = [True, False]
23
+SEEDS = [0]
24
+CUDA_DEVICES = [
25
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
26
+]
27
+
28
+EPS = 1e-6
29
+
30
+## Helpers
31
+
32
+
33
+def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
34
+ return torch.as_tensor(x, dtype=torch.float32, device='cuda')
35
+
36
+
37
+def ref_rms_norm(rms_norm_layer: RMSNorm,
38
+ x: torch.Tensor,
39
+ residual: Optional[torch.Tensor]) \
40
+ -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
41
+ if residual is not None:
42
+ residual = residual.clone()
43
+ out, residual = rms_norm_layer.forward_native(x, residual)
44
+ else:
45
+ out = rms_norm_layer.forward_native(x)
46
+
47
+ return out, residual
48
+
49
+
50
+def ref_dynamic_per_token_quant(rms_norm_layer: RMSNorm,
51
+ x: torch.Tensor,
52
+ quant_dtype: torch.dtype,
53
+ residual: Optional[torch.Tensor],
54
+ scale_ub: Optional[torch.Tensor]) \
55
+ -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
56
+ if scale_ub is not None:
57
+ assert quant_dtype == torch.float8_e4m3fn
58
+
59
+ # Norm
60
+ torch_out, residual = ref_rms_norm(rms_norm_layer, x, residual)
61
+
62
+ # Quant
63
+ if quant_dtype == torch.float8_e4m3fn:
64
+ torch_out, scales = ops.scaled_fp8_quant(torch_out,
65
+ scale_ub=scale_ub,
66
+ use_per_token_if_dynamic=True)
67
+ else:
68
+ assert quant_dtype == torch.int8
69
+ torch_out, scales = ops.scaled_int8_quant(torch_out)
70
+
71
+ return torch_out, scales, residual
72
+
73
+
74
+def ref_impl(rms_norm_layer: RMSNorm,
75
+ x: torch.Tensor,
76
+ quant_dtype: torch.dtype,
77
+ residual: Optional[torch.Tensor],
78
+ scale_ub: Optional[torch.Tensor]) \
79
+ -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
80
+ return ref_dynamic_per_token_quant(rms_norm_layer, x, quant_dtype,
81
+ residual, scale_ub)
82
+
83
+
84
+def ops_dynamic_per_token_quant(weight: torch.Tensor,
85
+ x: torch.Tensor,
86
+ quant_dtype: torch.dtype,
87
+ residual: Optional[torch.Tensor],
88
+ scale_ub: Optional[torch.Tensor]) \
89
+ -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
90
+ if residual is not None:
91
+ residual = residual.clone()
92
+ out, scales = ops.rms_norm_dynamic_per_token_quant(x, weight, EPS,
93
+ quant_dtype, scale_ub,
94
+ residual)
95
+ return out, scales, residual
96
+
97
+
98
+def ops_impl(weight: torch.Tensor,
99
+ x: torch.Tensor,
100
+ quant_dtype: torch.dtype,
101
+ residual: Optional[torch.Tensor],
102
+ scale_ub: Optional[torch.Tensor]) \
103
+ -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
104
+ return ops_dynamic_per_token_quant(weight, x, quant_dtype, residual,
105
+ scale_ub)
106
+
107
+
108
+@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
109
+@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
110
+@pytest.mark.parametrize("scale_ub", SCALE_UBS)
111
+@pytest.mark.parametrize("dtype", DTYPES)
112
+@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
113
+@pytest.mark.parametrize("seed", SEEDS)
114
+@pytest.mark.parametrize("device", CUDA_DEVICES)
115
+@torch.inference_mode()
116
+def test_rms_norm(
117
+ num_tokens: int,
118
+ hidden_size: int,
119
+ add_residual: bool,
120
+ scale_ub: bool,
121
+ dtype: torch.dtype,
122
+ quant_dtype: torch.dtype,
123
+ seed: int,
124
+ device: str,
125
+) -> None:
126
+ torch.random.manual_seed(seed)
127
+ if torch.cuda.is_available():
128
+ torch.cuda.manual_seed(seed)
129
+ torch.set_default_device(device)
130
+
131
+ if scale_ub is not None and quant_dtype != torch.float8_e4m3fn:
132
+ # skip
133
+ return
134
+
135
+ layer = RMSNorm(hidden_size, EPS).to(dtype=dtype)
136
+
137
+ # Make weights
138
+ layer.weight.data.normal_(mean=1.0, std=0.1)
139
+
140
+ # Make inputs
141
+ scale = 1 / (hidden_size)
142
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale
143
+ residual = torch.randn_like(x) * scale if add_residual else None
144
+ if scale_ub is not None:
145
+ rms_x, _ = ref_rms_norm(layer, x, residual)
146
+ scale_ub = torch.mean(rms_x).to(dtype=torch.float32, device='cuda')
147
+
148
+ ref_out, ref_scales, ref_residual = \
149
+ ref_impl(layer, x, quant_dtype, residual, scale_ub)
150
+ ops_out, ops_scales, ops_residual = \
151
+ ops_impl(layer.weight, x, quant_dtype, residual, scale_ub)
152
+
153
+ assert ref_out.dtype == quant_dtype
154
+ assert ops_out.dtype == quant_dtype
155
+ assert torch.allclose(ref_scales, ops_scales)
156
+ if quant_dtype == torch.int8:
157
+ # big atol to account for round-off errors.
158
+ assert torch.allclose(ref_out, ops_out, atol=1)
159
+ else:
160
+ assert torch.allclose(ref_out.to(dtype=torch.float32),
161
+ ops_out.to(dtype=torch.float32))
162
+ if add_residual:
163
+ assert torch.allclose(ref_residual, ops_residual)
164
+
165
+ output = torch.empty_like(x, dtype=quant_dtype)
166
+ scales = torch.empty((x.numel() // x.shape[-1], 1),
167
+ device=x.device,
168
+ dtype=torch.float32)
169
+
170
+ opcheck(torch.ops._C.rms_norm_dynamic_per_token_quant,
171
+ (output, x, layer.weight, scales, 1e-5, scale_ub, residual))
vllm/_custom_ops.py
+20
@@ -249,6 +249,26 @@ def advance_step_flashinfer(num_seqs: int, num_queries: int, block_size: int,
249
block_table_bound)
250
251
252
+# fused quant layer norm ops
253
+def rms_norm_dynamic_per_token_quant(
254
+ input: torch.Tensor,
255
+ weight: torch.Tensor,
256
+ epsilon: float,
257
+ quant_dtype: torch.dtype,
258
+ scale_ub: Optional[torch.Tensor] = None,
259
+ residual: Optional[torch.Tensor] = None
260
+) -> Tuple[torch.Tensor, torch.Tensor]:
261
+ output = torch.empty_like(input, dtype=quant_dtype)
262
+ scales = torch.empty((input.numel() // input.shape[-1], 1),
263
+ device=input.device,
264
+ dtype=torch.float32)
265
+
266
+ torch.ops._C.rms_norm_dynamic_per_token_quant(output, input, weight,
267
+ scales, epsilon, scale_ub,
268
+ residual)
269
+ return output, scales
270
+
271
+
272
# quantization ops
273
# awq
274
def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor,
vllm/compilation/fix_functionalization.py
+6
-3
@@ -6,7 +6,8 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
7
from vllm.logger import init_logger
8
9
-from .vllm_inductor_pass import VllmInductorPass, is_func
9
+from .fx_utils import is_func
10
+from .vllm_inductor_pass import VllmInductorPass
11
12
logger = init_logger(__name__)
13
@@ -53,14 +54,16 @@ class FixFunctionalizationPass(VllmInductorPass):
54
self.insert_defunctionalized(graph, node)
55
self._remove(node)
56
56
- # These 2 replacements avoid the most copies for LLaMa.
57
+ # rms_norm replacements avoid the most copies for LLaMa.
58
elif at_target == torch.ops._C.fused_add_rms_norm.default:
59
mutated_args = {1: 'input', 2: 'residual'}
60
self.defunctionalize(graph, node, mutated_args)
61
elif at_target == torch.ops._C.fused_add_rms_norm_static_fp8_quant.default: # noqa: E501
62
mutated_args = {1: 'result', 2: 'residual'}
63
self.defunctionalize(graph, node, mutated_args)
63
-
64
+ elif at_target == torch.ops._C.rms_norm_dynamic_per_token_quant.default: # noqa: E501
65
+ mutated_args = {1: 'result', 2: 'scale', 3: 'residual'}
66
+ self.defunctionalize(graph, node, mutated_args)
67
elif at_target in [
68
torch.ops._C.rms_norm.default,
69
torch.ops._C.rms_norm_static_fp8_quant.default
vllm/compilation/fusion.py
+521
-198
@@ -1,129 +1,517 @@
1
-import operator
2
-from typing import Iterable, List, Optional
1
+from typing import Callable, Dict, List, NamedTuple, Optional, Tuple
2
3
import torch
4
+import torch._inductor.pattern_matcher as pm
5
+# TODO(luka) use vllm.utils once #10836 landed
6
+from compressed_tensors.quantization import FP8_DTYPE
7
+from torch import fx
8
from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
-from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7
- fwd_only, register_replacement)
9
+from torch._inductor.pattern_matcher import PatternMatcherPass
10
+from torch._ops import OpOverload
11
12
from vllm.config import CompilationConfig
13
from vllm.logger import init_logger
14
12
-from .vllm_inductor_pass import VllmInductorPass, is_func
15
+from .fx_utils import find_getitem_maybe
16
+from .multi_output_match import MultiOutputMatch
17
+from .vllm_inductor_pass import VllmInductorPass
18
19
logger = init_logger(__name__)
20
21
17
-def rms_pattern_static(result: torch.Tensor, result_rms: torch.Tensor,
18
- input: torch.Tensor, weight: torch.Tensor,
19
- scale: torch.Tensor):
20
- at1 = auto_functionalized(torch.ops._C.rms_norm.default,
21
- result=result_rms,
22
- input=input,
23
- weight=weight,
24
- epsilon=1e-5)
25
- at2 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
26
- result=result,
27
- input=at1[1],
28
- scale=scale)
29
-
30
- # result
31
- return at2[1]
32
-
33
-
34
-def rms_replacement_static(result: torch.Tensor, result_rms: torch.Tensor,
35
- input: torch.Tensor, weight: torch.Tensor,
36
- scale: torch.Tensor):
37
- at = auto_functionalized(torch.ops._C.rms_norm_static_fp8_quant.default,
38
- result=result,
39
- input=input,
40
- weight=weight,
41
- scale=scale,
42
- epsilon=1e-5)
43
-
44
- # result
45
- return at[1]
46
-
47
-
48
-def rms_pattern_residual_static(result: torch.Tensor, input: torch.Tensor,
49
- residual: torch.Tensor, weight: torch.Tensor,
50
- scale: torch.Tensor):
51
- at = auto_functionalized(torch.ops._C.fused_add_rms_norm.default,
52
- input=input,
53
- residual=residual,
54
- weight=weight,
55
- epsilon=1e-5)
56
- at1 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
57
- result=result,
58
- input=at[1],
59
- scale=scale)
60
-
61
- # result, residual
62
- return at1[1], at[2]
63
-
64
-
65
-def rms_replacement_residual_static(result: torch.Tensor, input: torch.Tensor,
66
- residual: torch.Tensor,
67
- weight: torch.Tensor, scale: torch.Tensor):
68
- at = auto_functionalized(
69
- torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
70
- result=result,
71
- input=input,
72
- residual=residual,
73
- weight=weight,
74
- scale=scale,
75
- epsilon=1e-5)
76
- # result, residual
77
- return at[1], at[2]
78
-
79
-
22
def empty_bf16(*args, **kwargs):
23
return torch.empty(*args, **kwargs, dtype=torch.bfloat16, device="cuda")
24
25
84
-def empty_fp8(*args, **kwargs):
85
- fp8 = torch.float8_e4m3fn
86
- return torch.empty(*args, **kwargs, dtype=fp8, device="cuda")
87
-
88
-
26
def empty_fp32(*args, **kwargs):
27
return torch.empty(*args, **kwargs, dtype=torch.float32, device="cuda")
28
29
93
-# Utilities for post-processing multi-output matches
30
+RMS_OP = torch.ops._C.rms_norm.default
31
+RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
32
+
33
+
34
+class QuantKey(NamedTuple):
35
+ """
36
+ Named tuple for identifying the type of quantization.
37
+ dtype: quantized data type
38
+ static: static quantization if True, dynamic if False
39
+ per_tensor: per-tensor quantization if True, per-token if False
40
+ symmetric: symmetric if True, asymmetric if False
41
+ """
42
+ dtype: torch.dtype
43
+ static: bool
44
+ per_tensor: bool = True
45
+ symmetric: bool = True
46
+
47
+ def __str__(self):
48
+ return (f"QuantKey({'static' if self.static else 'dynamic'},"
49
+ f"{fx.graph.dtype_abbrs[self.dtype]},"
50
+ f"{'per_tensor' if self.per_tensor else 'per_token'},"
51
+ f"{'a' if not self.symmetric else ''}symmetric)")
52
+
53
+
54
+kFp8StaticTensorSym = QuantKey(FP8_DTYPE, True, True, True)
55
+kFp8DynamicTensorSym = QuantKey(FP8_DTYPE, False, True, True)
56
+kFp8DynamicTokenSym = QuantKey(FP8_DTYPE, False, False, True)
57
+
58
+QUANT_OPS: Dict[QuantKey, OpOverload] = {
59
+ kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa
60
+ kFp8DynamicTensorSym:
61
+ torch.ops._C.dynamic_scaled_fp8_quant.default, # noqa
62
+ kFp8DynamicTokenSym:
63
+ torch.ops._C.dynamic_per_token_scaled_fp8_quant.default, # noqa
64
+}
65
+
66
+
67
+class FusedRMSQuantKey(NamedTuple):
68
+ """
69
+ Named tuple for identifying the type of RMSNorm + quant fusion.
70
+ quant: type of quantization
71
+ fused_add: does the op also perform the residual add
72
+ """
73
+ quant: QuantKey
74
+ fused_add: bool
75
+
76
+ def __str__(self):
77
+ return (f"FusedQuantKey({self.quant}, with"
78
+ f"{'' if self.fused_add else 'out'} residual)")
79
+
80
+
81
+FUSED_OPS: Dict[FusedRMSQuantKey, OpOverload] = {
82
+ FusedRMSQuantKey(kFp8StaticTensorSym, False):
83
+ torch.ops._C.rms_norm_static_fp8_quant.default, # noqa
84
+ FusedRMSQuantKey(kFp8StaticTensorSym, True):
85
+ torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, # noqa
86
+ FusedRMSQuantKey(kFp8DynamicTokenSym, False):
87
+ torch.ops._C.rms_norm_dynamic_per_token_quant.default, # noqa
88
+ FusedRMSQuantKey(kFp8DynamicTokenSym, True):
89
+ torch.ops._C.rms_norm_dynamic_per_token_quant.default, # noqa
90
+}
91
+
92
+
93
+class QuantMultiOutputMatch(MultiOutputMatch):
94
+
95
+ def __init__(self, match: pm.Match, quant_op, fused_op):
96
+ super().__init__(match)
97
+ assert isinstance(quant_op, OpOverload)
98
+ assert isinstance(fused_op, OpOverload)
99
+ self.QUANT_OP = quant_op # in-place quant op
100
+ self.FUSED_OP = fused_op # in-place fused quant op
101
+
102
+ def insert_fused_node(self, fused_return_mapping: Dict[int, Tuple[fx.Node,
103
+ int]],
104
+ **kwargs):
105
+ """
106
+ This utility function inserts an auto-functionalized node for FUSED_OP.
107
+ It also correctly sets its meta value and rebinds the users of the
108
+ unfused nodes to use the fused node instead.
109
+
110
+ :param fused_return_mapping: A dictionary, mapping from getitem indices
111
+ of the fused node result to a tuple of the old node and a getitem index.
112
+ :param kwargs: kwargs that get directly forwarded to the auto_fn node
113
+
114
+ Example:
115
+ If we want to replace this graph:
116
+ _, x1, x2 = auto_fn(op1)
117
+ _, y1, y2 = auto_fn(op2)
118
+
119
+ with
120
+ _, x1, y2, x2 = auto_fn(FUSED_OP)
121
+
122
+ we would call:
123
+ insert_fused_node({1: (op1_node, 1), 2: (op2_node, 2), 3: (op1_node, 2)}
124
+
125
+ Note that the 0th element is None for auto-functionalized in-place ops.
126
+ Hence, others appear 1-indexed.
127
+ """
128
+ fused_node = self.insert_auto_fn(self.FUSED_OP, kwargs)
129
+ indices = fused_return_mapping.keys()
130
+ getitem_nodes = self.insert_getitems(fused_node, indices)
131
+
132
+ # Prepare the meta value, use a list so it's mutable
133
+ meta_val = [None] * (max(indices) + 1)
134
+
135
+ # Iterate through elements of the tuple produced by fused_node
136
+ for idx, getitem_node in zip(indices, getitem_nodes):
137
+ old_node, old_idx = fused_return_mapping[idx]
138
+
139
+ # If the old value was never used, the old_getitem might not exist
140
+ old_getitem = find_getitem_maybe(old_node, old_idx)
141
+ if old_getitem is not None:
142
+ # Rebind the users of match getitem nodes to use the new nodes.
143
+ # The old nodes will be removed by DCE at the end of the pass.
144
+ old_getitem.replace_all_uses_with(getitem_node)
145
+ getitem_node.meta["val"] = old_getitem.meta["val"]
146
+
147
+ # Extract the appropriate meta value
148
+ # It is present even if the getitem node does not exist
149
+ meta_val[idx] = old_node.meta["val"][old_idx]
150
+
151
+ # Fix the meta value on the new fused node
152
+ fused_node.meta["val"] = tuple(meta_val)
153
+
154
+
155
+class RMSNormQuantPattern:
156
+
157
+ def __init__(self, epsilon: float, key: FusedRMSQuantKey):
158
+ self.epsilon = epsilon
159
+ self.quant_dtype = key.quant.dtype
160
+
161
+ assert key.quant in QUANT_OPS, \
162
+ f"unsupported quantization scheme {key.quant}"
163
+ self.QUANT_OP = QUANT_OPS[key.quant]
164
+
165
+ assert key in FUSED_OPS, \
166
+ f"unsupported fused rmsnorm+quant op for {key}"
167
+ self.FUSED_OP = FUSED_OPS[key]
168
+
169
+
170
+class RMSNormStaticQuantPattern(RMSNormQuantPattern):
171
+
172
+ def __init__(self,
173
+ epsilon: float,
174
+ quant_dtype: torch.dtype,
175
+ symmetric=True):
176
+ fused_key = FusedRMSQuantKey(fused_add=False,
177
+ quant=QuantKey(dtype=quant_dtype,
178
+ static=True,
179
+ per_tensor=True,
180
+ symmetric=symmetric))
181
+ super().__init__(epsilon, fused_key)
182
+
183
+ def register(self, pm_pass: PatternMatcherPass):
184
+ # Cannot use methods, as the self argument affects tracing
185
+ def pattern(result: torch.Tensor, result_rms: torch.Tensor,
186
+ input: torch.Tensor, weight: torch.Tensor,
187
+ scale: torch.Tensor):
188
+ at1 = auto_functionalized(RMS_OP,
189
+ result=result_rms,
190
+ input=input,
191
+ weight=weight,
192
+ epsilon=self.epsilon)
193
+ at2 = auto_functionalized(self.QUANT_OP,
194
+ result=result,
195
+ input=at1[1],
196
+ scale=scale)
197
+
198
+ # result
199
+ return at2[1]
200
+
201
+ def replacement(result: torch.Tensor, result_rms: torch.Tensor,
202
+ input: torch.Tensor, weight: torch.Tensor,
203
+ scale: torch.Tensor):
204
+ at = auto_functionalized(self.FUSED_OP,
205
+ result=result,
206
+ input=input,
207
+ weight=weight,
208
+ scale=scale,
209
+ epsilon=self.epsilon)
210
+
211
+ # result
212
+ return at[1]
213
+
214
+ inputs = [
215
+ torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
216
+ empty_bf16(5, 4), # result_rms
217
+ empty_bf16(5, 4), # input
218
+ empty_bf16(1, 5), # weight
219
+ empty_fp32(1, 1) # scale
220
+ ]
221
+
222
+ pm.register_replacement(pattern, replacement, inputs, pm.fwd_only,
223
+ pm_pass)
224
+
225
+
226
+class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
227
+
228
+ def __init__(self,
229
+ epsilon: float,
230
+ quant_dtype: torch.dtype,
231
+ symmetric=True):
232
+ key = FusedRMSQuantKey(fused_add=True,
233
+ quant=QuantKey(dtype=quant_dtype,
234
+ static=True,
235
+ per_tensor=True,
236
+ symmetric=symmetric))
237
+ super().__init__(epsilon, key)
238
+
239
+ def register(self, pm_pass: PatternMatcherPass,
240
+ record_match: Callable[[MultiOutputMatch], bool]):
241
+
242
+ def pattern(result: torch.Tensor, input: torch.Tensor,
243
+ residual: torch.Tensor, weight: torch.Tensor,
244
+ scale: torch.Tensor):
245
+ at = auto_functionalized(RMS_ADD_OP,
246
+ input=input,
247
+ residual=residual,
248
+ weight=weight,
249
+ epsilon=self.epsilon)
250
+ at1 = auto_functionalized(self.QUANT_OP,
251
+ result=result,
252
+ input=at[1],
253
+ scale=scale)
254
+
255
+ # result, residual
256
+ return at1[1], at[2]
257
+
258
+ def replacement(result: torch.Tensor, input: torch.Tensor,
259
+ residual: torch.Tensor, weight: torch.Tensor,
260
+ scale: torch.Tensor):
261
+ at = auto_functionalized(self.FUSED_OP,
262
+ result=result,
263
+ input=input,
264
+ residual=residual,
265
+ weight=weight,
266
+ scale=scale,
267
+ epsilon=self.epsilon)
268
+
269
+ # result, residual
270
+ return at[1], at[2]
271
+
272
+ inputs = [
273
+ torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
274
+ empty_bf16(5, 4), # input
275
+ empty_bf16(5, 4), # residual
276
+ empty_bf16(1, 5), # weight
277
+ empty_fp32(1, 1) # scale
278
+ ]
279
+
280
+ pm.register_replacement(
281
+ pattern,
282
+ replacement,
283
+ inputs,
284
+ pm.fwd_only,
285
+ pm_pass,
286
+ extra_check=lambda m: record_match(
287
+ self.Match(m, self.QUANT_OP, self.FUSED_OP)))
288
+
289
+ class Match(QuantMultiOutputMatch):
290
+
291
+ def process(self):
292
+ # Find the nodes in the match that we need to rebind
293
+ rms_node = self.find_auto_fn(RMS_ADD_OP)
294
+ quant_node = self.find_auto_fn(self.QUANT_OP)
295
+
296
+ assert len(rms_node.users) == 2
297
+ assert len(quant_node.users) == 1
298
+
299
+ # First, insert a new auto_functionalized node for the fused op,
300
+ # as well as getitem nodes to extract the result and residual.
301
+ # The auto_fn node returns a tuple of (None, result, residual).
302
+ #
303
+ # The resulting graph looks like this:
304
+ # at = auto_functionalized(torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, ...) # noqa
305
+ # result_node_new = at[1]
306
+ # residual_node_new = at[2]
307
+ with self.inserting_after_match():
308
+ # Missing epsilon, scalars cannot be inputs to the pattern
309
+ kwargs = self.match.kwargs.copy()
310
+
311
+ # 0 is always None
312
+ fused_return_mapping = {1: (quant_node, 1), 2: (rms_node, 2)}
313
+ self.insert_fused_node(fused_return_mapping,
314
+ epsilon=rms_node.kwargs["epsilon"],
315
+ **kwargs)
316
+
317
+
318
+class RMSNormDynamicQuantPattern(RMSNormQuantPattern):
319
+
320
+ def __init__(self,
321
+ epsilon: float,
322
+ quant_dtype: torch.dtype,
323
+ per_tensor: bool,
324
+ symmetric=True):
325
+ key = FusedRMSQuantKey(fused_add=False,
326
+ quant=QuantKey(dtype=quant_dtype,
327
+ static=False,
328
+ per_tensor=per_tensor,
329
+ symmetric=symmetric))
330
+ super().__init__(epsilon, key)
331
+
332
+ def register(self, pm_pass: PatternMatcherPass,
333
+ record_match: Callable[[MultiOutputMatch], bool]):
334
+
335
+ def pattern(result: torch.Tensor, result_rms: torch.Tensor,
336
+ input: torch.Tensor, weight: torch.Tensor,
337
+ scale: torch.Tensor):
338
+ at1 = auto_functionalized(RMS_OP,
339
+ result=result_rms,
340
+ input=input,
341
+ weight=weight,
342
+ epsilon=self.epsilon)
343
+ at2 = auto_functionalized(self.QUANT_OP,
344
+ result=result,
345
+ input=at1[1],
346
+ scale=scale,
347
+ scale_ub=None)
348
+
349
+ # result, scale
350
+ return at2[1], at2[2]
351
+
352
+ def replacement(result: torch.Tensor, result_rms: torch.Tensor,
353
+ input: torch.Tensor, weight: torch.Tensor,
354
+ scale: torch.Tensor):
355
+ at = auto_functionalized(self.FUSED_OP,
356
+ result=result,
357
+ input=input,
358
+ weight=weight,
359
+ scale=scale,
360
+ epsilon=self.epsilon,
361
+ scale_ub=None,
362
+ residual=None)
363
+
364
+ # result, scale
365
+ return at[1], at[2]
366
+
367
+ inputs = [
368
+ torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
369
+ empty_bf16(5, 4), # result_rms
370
+ empty_bf16(5, 4), # input
371
+ empty_bf16(1, 5), # weight
372
+ empty_fp32(1, 1) # scale
373
+ ]
374
+
375
+ pm.register_replacement(
376
+ pattern,
377
+ replacement,
378
+ inputs,
379
+ pm.fwd_only,
380
+ pm_pass,
381
+ extra_check=lambda m: record_match(
382
+ self.Match(m, self.QUANT_OP, self.FUSED_OP)))
383
384
+ class Match(QuantMultiOutputMatch):
385
96
-# Returns the first auto_functionalized node with the given op (if it exists)
97
-def find_auto_fn_maybe(nodes: Iterable[torch.fx.Node],
98
- op) -> Optional[torch.fx.Node]:
99
- for node in nodes:
100
- if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
101
- return node
102
- return None
386
+ def process(self):
387
+ # Find the nodes in the match that we need to rebind
388
+ rms_node = self.find_auto_fn(RMS_OP)
389
+ quant_node = self.find_auto_fn(self.QUANT_OP)
390
391
+ assert len(rms_node.users) == 1
392
+ assert len(quant_node.users) == 2
393
105
-# Returns the first auto_functionalized node with the given op
106
-def find_auto_fn(nodes: Iterable[torch.fx.Node], op) -> torch.fx.Node:
107
- node = find_auto_fn_maybe(nodes, op)
108
- assert node is not None, f"Could not find {op} in nodes {nodes}"
109
- return node
394
+ # First, insert a new auto_functionalized node for the fused op,
395
+ # as well as getitem nodes to extract the result and scale.
396
+ # The auto_fn node returns a tuple of (None, result, scale).
397
+ #
398
+ # The resulting graph looks like this:
399
+ # at = auto_functionalized(torch.ops._C.rms_norm_dynamic_per_token_quant.default, ...) # noqa
400
+ # result_node_new = at[1]
401
+ # scale_node_new = at[2]
402
+ with self.inserting_after_match():
403
+ # Missing epsilon, scalars cannot be inputs to the pattern
404
+ kwargs = self.match.kwargs.copy()
405
+ del kwargs["result_rms"] # not used in the fused op
406
+
407
+ fused_return_mapping = {1: (quant_node, 1), 2: (quant_node, 2)}
408
+ self.insert_fused_node(
409
+ fused_return_mapping,
410
+ epsilon=rms_node.kwargs["epsilon"],
411
+ scale_ub=None, # not used but required
412
+ residual=None, # not used but required
413
+ **kwargs)
414
+
415
+
416
+class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
417
+
418
+ def __init__(self,
419
+ epsilon: float,
420
+ quant_dtype: torch.dtype,
421
+ per_tensor: bool = True,
422
+ symmetric=True):
423
+ key = FusedRMSQuantKey(fused_add=True,
424
+ quant=QuantKey(dtype=quant_dtype,
425
+ static=False,
426
+ per_tensor=per_tensor,
427
+ symmetric=symmetric))
428
+ super().__init__(epsilon, key)
429
+
430
+ def register(self, pm_pass: PatternMatcherPass,
431
+ record_match: Callable[[MultiOutputMatch], bool]):
432
+
433
+ def pattern(result: torch.Tensor, input: torch.Tensor,
434
+ residual: torch.Tensor, weight: torch.Tensor,
435
+ scale: torch.Tensor):
436
+ at = auto_functionalized(RMS_ADD_OP,
437
+ input=input,
438
+ residual=residual,
439
+ weight=weight,
440
+ epsilon=self.epsilon)
441
+ at1 = auto_functionalized(self.QUANT_OP,
442
+ result=result,
443
+ input=at[1],
444
+ scale=scale,
445
+ scale_ub=None)
446
+
447
+ # result, residual, scale
448
+ return at1[1], at[2], at1[2]
449
+
450
+ def replacement(result: torch.Tensor, input: torch.Tensor,
451
+ residual: torch.Tensor, weight: torch.Tensor,
452
+ scale: torch.Tensor):
453
+ at = auto_functionalized(self.FUSED_OP,
454
+ result=result,
455
+ input=input,
456
+ weight=weight,
457
+ scale=scale,
458
+ epsilon=self.epsilon,
459
+ scale_ub=None,
460
+ residual=residual)
461
+
462
+ # result, residual, scale
463
+ return at[1], at[3], at[2]
464
465
+ inputs = [
466
+ torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
467
+ empty_bf16(5, 4), # input
468
+ empty_bf16(5, 4), # residual
469
+ empty_bf16(1, 5), # weight
470
+ empty_fp32(1, 1) # scale
471
+ ]
472
112
-# Returns the getitem node that extracts the idx-th element from node
113
-# (if it exists)
114
-def find_getitem_maybe(node: torch.fx.Node,
115
- idx: int) -> Optional[torch.fx.Node]:
116
- for user in node.users:
117
- if is_func(user, operator.getitem) and user.args[1] == idx:
118
- return user
119
- return None
473
+ pm.register_replacement(
474
+ pattern,
475
+ replacement,
476
+ inputs,
477
+ pm.fwd_only,
478
+ pm_pass,
479
+ extra_check=lambda m: record_match(
480
+ self.Match(m, self.QUANT_OP, self.FUSED_OP)))
481
482
+ class Match(QuantMultiOutputMatch):
483
122
-# Returns the getitem node that extracts the idx-th element from node
123
-def find_getitem(node: torch.fx.Node, idx: int) -> torch.fx.Node:
124
- ret = find_getitem_maybe(node, idx)
125
- assert ret is not None, f"Could not find getitem {idx} in node {node}"
126
- return ret
484
+ def process(self):
485
+ # Find the nodes in the match that we need to rebind
486
+ rms_node = self.find_auto_fn(RMS_ADD_OP)
487
+ quant_node = self.find_auto_fn(self.QUANT_OP)
488
+
489
+ assert len(rms_node.users) == 2
490
+ assert len(quant_node.users) == 2
491
+
492
+ # First, insert a new auto_functionalized node for the fused op,
493
+ # as well as getitem nodes to extract result, scale, and residual.
494
+ # The auto_fn node returns a tuple (None, result, scale, residual).
495
+ #
496
+ # The resulting graph looks like this:
497
+ # at = auto_functionalized(torch.ops._C.rms_norm_dynamic_per_token_quant.default, ...) # noqa
498
+ # result_node_new = at[1]
499
+ # scale_node_new = at[2]
500
+ # residual_node_new = at[3]
501
+ with self.inserting_after_match():
502
+ # Missing epsilon, scalars cannot be inputs to the pattern
503
+ kwargs = self.match.kwargs.copy()
504
+
505
+ fused_return_mapping = {
506
+ 1: (quant_node, 1), # result
507
+ 2: (quant_node, 2), # scale
508
+ 3: (rms_node, 2), # residual
509
+ }
510
+ self.insert_fused_node(
511
+ fused_return_mapping,
512
+ epsilon=rms_node.kwargs["epsilon"],
513
+ scale_ub=None, # not used but required
514
+ **kwargs)
515
516
517
class FusionPass(VllmInductorPass):
@@ -158,41 +546,39 @@ class FusionPass(VllmInductorPass):
546
"FusionPass singleton instance already exists"
547
super().__init__(config)
548
161
- self.matches: List[Match] = []
549
+ self.matches: List[MultiOutputMatch] = []
550
self.patterns: PatternMatcherPass = PatternMatcherPass(
551
pass_name="fusion_pass")
552
165
- # Fuse rms_norm + static_scaled_fp8_quant into
166
- # rms_norm_static_fp8_quant
167
- inputs = [
168
- empty_fp8(5, 4),
169
- empty_bf16(5, 4),
170
- empty_bf16(5, 4),
171
- empty_bf16(1, 5),
172
- empty_fp32(1, 1)
173
- ]
174
- register_replacement(rms_pattern_static, rms_replacement_static,
175
- inputs, fwd_only, self.patterns)
553
+ for epsilon in [1e-5, 1e-6]:
554
+ # Fuse rms_norm + static fp8 quant
555
+ RMSNormStaticQuantPattern(epsilon,
556
+ FP8_DTYPE).register(self.patterns)
557
177
- # Fuse fused_add_rms_norm + static_scaled_fp8_quant into
178
- # fused_add_rms_norm_static_fp8_quant
179
- # Because pattern has 2 outputs, we need to manually process the match
180
- # (see process_matches)
181
- inputs = [
182
- empty_fp8(5, 4),
183
- empty_bf16(5, 4),
184
- empty_bf16(5, 4),
185
- empty_bf16(1, 5),
186
- empty_fp32(1, 1)
187
- ]
188
- register_replacement(rms_pattern_residual_static,
189
- rms_replacement_residual_static,
190
- inputs,
191
- fwd_only,
192
- self.patterns,
193
- extra_check=lambda m: self.record_match(m))
194
-
195
- def record_match(self, match: Match) -> bool:
558
+ # Matches for patterns below have 2 or more outputs,
559
+ # so we need to process them manually (see process_matches)
560
+
561
+ # Fuse rms_norm + static fp8 quant
562
+ FusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(
563
+ self.patterns, self.record_match)
564
+
565
+ # Fuse rms_norm + dynamic per-token fp8 quant
566
+ RMSNormDynamicQuantPattern(epsilon, FP8_DTYPE,
567
+ per_tensor=False).register(
568
+ self.patterns, self.record_match)
569
+
570
+ # Fuse fused_add_rms_norm + dynamic per-token fp8 quant
571
+ FusedAddRMSNormDynamicQuantPattern(epsilon,
572
+ FP8_DTYPE,
573
+ per_tensor=False).register(
574
+ self.patterns,
575
+ self.record_match)
576
+
577
+ # WARNING: This is a hack to clear the pattern matcher cache
578
+ # and allow multiple values of epsilon.
579
+ torch._inductor.pattern_matcher._seen_patterns.clear()
580
+
581
+ def record_match(self, match: MultiOutputMatch) -> bool:
582
# Hijack the extra_check to record the match and
583
# save it for post-processing.
584
self.matches.append(match)
@@ -200,83 +586,20 @@ class FusionPass(VllmInductorPass):
586
# Return False to prevent automatic replacement.
587
return False
588
203
- def process_matches(self, graph: torch.fx.Graph):
589
+ def process_matches(self, graph: fx.Graph):
590
"""
591
Manually process multi-output matches and replace them with fused nodes.
206
- This is necessary because the automatic replacement for multi-output
207
- matches is broken: https://github.com/pytorch/pytorch/issues/137280
592
+ See MultiOutputMatch for more details.
593
"""
594
for match in self.matches:
210
- # To avoid use-before-definition errors, insert replacement nodes
211
- # after the last node in the match.
212
- # match.nodes is not guaranteed to be sorted.
213
- # Find the last node in the match.
214
- for last_node_in_match in reversed(graph.nodes):
215
- if last_node_in_match in match.nodes:
216
- break
217
- else:
218
- raise ValueError("No nodes in graph")
219
-
220
- # Insert a new auto_functionalized node for the fused operation,
221
- # as well as getitem nodes to extract the result and residual.
222
- # The auto_functionalized node returns a tuple of
223
- # (None, result, residual) - None is the function return value.
224
- # The resulting graph looks like this:
225
- # at = auto_functionalized(torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, ...) # noqa
226
- # result_node_new = at[1]
227
- # residual_node_new = at[2]
228
- with graph.inserting_after(last_node_in_match):
229
- kwargs = match.kwargs
230
- kwargs["epsilon"] = 1e-5 # Currently hard-coded in RMSNorm
231
-
232
- fused_node = graph.call_function(
233
- auto_functionalized,
234
- (torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
235
- ),
236
- kwargs=kwargs)
237
-
238
- graph.inserting_after(fused_node)
239
- result_node_new = graph.call_function(operator.getitem,
240
- (fused_node, 1))
241
- residual_node_new = graph.call_function(
242
- operator.getitem, (fused_node, 2))
243
-
244
- # Last part of replacement is rebinding the users of nodes in the
245
- # match to use the new nodes.
246
-
247
- # Find the nodes in the match that we need to rebind
248
- rms_node = find_auto_fn(match.nodes,
249
- torch.ops._C.fused_add_rms_norm.default)
250
- quant_node = find_auto_fn(
251
- match.nodes, torch.ops._C.static_scaled_fp8_quant.default)
252
-
253
- assert len(rms_node.users) == 2
254
- assert len(quant_node.users) == 1
255
-
256
- # meta["val"] is used by de-functionalization and has to contain the
257
- # value of the node (tuple of tensors) that would be returned by the
258
- # functionalized node during tracing.
259
-
260
- rms_tup = rms_node.meta["val"]
261
- quant_tup = quant_node.meta["val"]
262
-
263
- # The result of fused_node must be a tuple with the first element
264
- # None (the function return value) and the remaining elements
265
- # representing the mutated inputs.
266
- fused_tup = (None, quant_tup[1], rms_tup[1], rms_tup[2])
267
- fused_node.meta["val"] = fused_tup
268
-
269
- # Find the getitem nodes and replace their uses with the new nodes.
270
- # The old nodes will be removed by DCE at the end of the pass.
271
- find_getitem(rms_node, 2).replace_all_uses_with(residual_node_new)
272
- find_getitem(quant_node, 1).replace_all_uses_with(result_node_new)
595
+ match.process()
596
597
# Finally, remove matched nodes
598
graph.eliminate_dead_code()
599
assert all(node not in graph.nodes for match in self.matches
277
- for node in match.nodes)
600
+ for node in match.match.nodes)
601
279
- def __call__(self, graph: torch.fx.Graph):
602
+ def __call__(self, graph: fx.Graph):
603
self.begin()
604
self.dump_graph(graph, "before_fusion")
605
vllm/compilation/fx_utils.py
new
+42
@@ -0,0 +1,42 @@
1
+import operator
2
+from typing import Iterable, Optional
3
+
4
+from torch import fx
5
+from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
+from torch._ops import OpOverload
7
+
8
+
9
+def is_func(node: fx.Node, target) -> bool:
10
+ return node.op == "call_function" and node.target == target
11
+
12
+
13
+# Returns the first auto_functionalized node with the given op (if it exists)
14
+def find_auto_fn_maybe(nodes: Iterable[fx.Node],
15
+ op: OpOverload) -> Optional[fx.Node]:
16
+ for node in nodes:
17
+ if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
18
+ return node
19
+ return None
20
+
21
+
22
+# Returns the first auto_functionalized node with the given op
23
+def find_auto_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node:
24
+ node = find_auto_fn_maybe(nodes, op)
25
+ assert node is not None, f"Could not find {op} in nodes {nodes}"
26
+ return node
27
+
28
+
29
+# Returns the getitem node that extracts the idx-th element from node
30
+# (if it exists)
31
+def find_getitem_maybe(node: fx.Node, idx: int) -> Optional[fx.Node]:
32
+ for user in node.users:
33
+ if is_func(user, operator.getitem) and user.args[1] == idx:
34
+ return user
35
+ return None
36
+
37
+
38
+# Returns the getitem node that extracts the idx-th element from node
39
+def find_getitem(node: fx.Node, idx: int) -> fx.Node:
40
+ ret = find_getitem_maybe(node, idx)
41
+ assert ret is not None, f"Could not find getitem {idx} in node {node}"
42
+ return ret
vllm/compilation/multi_output_match.py
new
+105
@@ -0,0 +1,105 @@
1
+import abc
2
+import operator
3
+from abc import abstractmethod
4
+from typing import Iterable, List, Tuple
5
+
6
+from torch import fx
7
+from torch._higher_order_ops.auto_functionalize import auto_functionalized
8
+from torch._inductor import pattern_matcher as pm
9
+from torch._ops import OpOverload
10
+
11
+from vllm.compilation.fx_utils import find_auto_fn
12
+
13
+
14
+class MultiOutputMatch(abc.ABC):
15
+ """
16
+ This class provides utilities to process multi-output matches and
17
+ manually insert replacements.
18
+
19
+ This is necessary because the automatic replacement for multi-output
20
+ matches is broken: https://github.com/pytorch/pytorch/issues/137280
21
+ """
22
+
23
+ def __init__(self, match: pm.Match):
24
+ self.match = match
25
+
26
+ @abstractmethod
27
+ def process(self):
28
+ """
29
+ Process a multi-output match and manually insert the replacement.
30
+
31
+ This method should:
32
+ 1. Insert the replacement nodes after the last node in the match.
33
+ 2. Rebind the users of nodes in the match to use the new nodes.
34
+ 3. Set meta["val"] for de-functionalization.
35
+
36
+ The result of an auto-functionalized node is a tuple of tensors.
37
+ The first element is the return value of the function, usually None.
38
+ The remaining elements are the mutated args of the function.
39
+
40
+ All auto-functionalized nodes must contain a proper meta["val"],
41
+ as it is used by de-functionalization. meta["val"] has to contain the
42
+ value of the node (tuple of tensors) that would be returned by the
43
+ functionalized node during tracing.
44
+
45
+ Existing nodes in the graph all have this property set, but we have
46
+ to set it manually for new nodes we insert.
47
+
48
+ Example:
49
+ # op schema: foo(a: Tensor!, b: Tensor, c: Tensor!) -> None
50
+ at = auto_functionalized(torch.ops._C.foo.default, a, b, c)
51
+ # at.meta["val"] = (None, a, c)
52
+ """
53
+ raise NotImplementedError
54
+
55
+ @property
56
+ def nodes(self) -> List[fx.Node]:
57
+ return self.match.nodes
58
+
59
+ @property
60
+ def graph(self) -> fx.Graph:
61
+ return self.match.graph
62
+
63
+ def find_auto_fn(self, op) -> fx.Node:
64
+ """
65
+ Find the first auto_functionalized node with the given op in the match.
66
+ """
67
+ return find_auto_fn(self.nodes, op)
68
+
69
+ def inserting_after_match(self):
70
+ """
71
+ Insert nodes after the last node in the match.
72
+ This is done to avoid use-before-definition errors after inserting
73
+ replacement nodes.
74
+ """
75
+
76
+ # match.nodes is not guaranteed to be sorted.
77
+ # Find the last node in the match.
78
+ for last_node_in_match in reversed(self.graph.nodes):
79
+ if last_node_in_match in self.match.nodes:
80
+ break
81
+ else:
82
+ raise ValueError("No nodes in graph")
83
+
84
+ return self.graph.inserting_after(last_node_in_match)
85
+
86
+ def insert_getitems(self, tuple_node: fx.Node,
87
+ indices: Iterable[int]) -> Tuple[fx.Node, ...]:
88
+ """
89
+ Insert operator.getitem nodes to extract elements from a tuple node.
90
+
91
+ :param tuple_node: The tuple node to extract elements from.
92
+ :param indices: The indices of the elements to extract.
93
+ :return: Tuple of the new getitem nodes, corresponding to the indices.
94
+ """
95
+ with self.graph.inserting_after(tuple_node):
96
+ return tuple(
97
+ self.graph.call_function(operator.getitem, (tuple_node, idx))
98
+ for idx in indices)
99
+
100
+ def insert_auto_fn(self, op: OpOverload, kwargs):
101
+ """
102
+ Insert an auto_functionalized node with the given op and kwargs.
103
+ """
104
+ return self.graph.call_function(auto_functionalized, (op, ),
105
+ kwargs=kwargs)
vllm/compilation/reshapes.py
+2
-1
@@ -5,7 +5,8 @@ from torch import SymInt
5
6
from vllm.logger import init_logger
7
8
-from .vllm_inductor_pass import VllmInductorPass, is_func
8
+from .fx_utils import is_func
9
+from .vllm_inductor_pass import VllmInductorPass
10
11
logger = init_logger(__name__)
12
vllm/compilation/vllm_inductor_pass.py
-4
@@ -16,10 +16,6 @@ from .inductor_pass import InductorPass
16
logger = init_logger(__name__)
17
18
19
-def is_func(node: torch.fx.Node, target) -> bool:
20
- return node.op == "call_function" and node.target == target
21
-
22
-
19
class VllmInductorPass(InductorPass):
20
"""
21
An inductor pass with access to vLLM PassConfig.