A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[torch.compile] Fuse RMSNorm with quant (#9138)
Signed-off-by: luka <luka@neuralmagic.com> Co-authored-by: youkaichao <youkaichao@126.com>
Luka Govedič committed
Nov 8, 2024 at 16:20 UTC
4f93dfe952522c3f784be6542d69be2a172b8496
17 files changed
+1335
-368
CMakeLists.txt
+1
@@ -191,6 +191,7 @@ set(VLLM_EXT_SRC
191
"csrc/pos_encoding_kernels.cu"
192
"csrc/activation_kernels.cu"
193
"csrc/layernorm_kernels.cu"
194
+ "csrc/layernorm_quant_kernels.cu"
195
"csrc/quantization/gptq/q_gemm.cu"
196
"csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
197
"csrc/quantization/fp8/common.cu"
csrc/layernorm_kernels.cu
+4
-161
@@ -1,21 +1,13 @@
1
-#include <torch/all.h>
2
-#include <ATen/cuda/CUDAContext.h>
1
+#include "type_convert.cuh"
2
+#include "dispatch_utils.h"
3
+
4
+#include <torch/cuda.h>
5
#include <c10/cuda/CUDAGuard.h>
6
5
-#include "dispatch_utils.h"
7
#ifndef USE_ROCM
7
- #include <cuda_bf16.h>
8
- #include <cuda_fp16.h>
9
- #include <cub/util_type.cuh>
8
#include <cub/cub.cuh>
9
#else
12
- #include <hip/hip_bf16.h>
13
- #include <hip/hip_fp16.h>
14
- #include <hipcub/util_type.hpp>
10
#include <hipcub/hipcub.hpp>
16
-
17
-using __nv_bfloat16 = __hip_bfloat16;
18
-using __nv_bfloat162 = __hip_bfloat162;
11
#endif
12
13
namespace vllm {
@@ -51,155 +43,6 @@ __global__ void rms_norm_kernel(
43
}
44
}
45
54
-/* Converter structs for the conversion from torch types to HIP/CUDA types,
55
- and the associated type conversions within HIP/CUDA. These helpers need
56
- to be implemented for now because the relevant type conversion
57
- operators/constructors are not consistently implemented by HIP/CUDA, so
58
- a generic conversion via type casts cannot be implemented.
59
-
60
- Each struct should have the member static constexpr bool `exists`:
61
- If false, the optimized kernel is not used for the corresponding torch type.
62
- If true, the struct should be fully defined as shown in the examples below.
63
- */
64
-template <typename torch_type>
65
-struct _typeConvert {
66
- static constexpr bool exists = false;
67
-};
68
-
69
-#if defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
70
-// CUDA < 12.0 runs into issues with packed type conversion
71
-template <>
72
-struct _typeConvert<c10::Half> {
73
- static constexpr bool exists = true;
74
- using hip_type = __half;
75
- using packed_hip_type = __half2;
76
-
77
- __device__ static inline float convert(hip_type x) { return __half2float(x); }
78
- __device__ static inline float2 convert(packed_hip_type x) {
79
- return __half22float2(x);
80
- }
81
- __device__ static inline hip_type convert(float x) {
82
- return __float2half_rn(x);
83
- }
84
- __device__ static inline packed_hip_type convert(float2 x) {
85
- return __float22half2_rn(x);
86
- }
87
-};
88
-
89
- #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
90
-// CUDA_ARCH < 800 does not have BF16 support
91
-// TODO: Add in ROCm support once public headers handle bf16 maturely
92
-template <>
93
-struct _typeConvert<c10::BFloat16> {
94
- static constexpr bool exists = true;
95
- using hip_type = __nv_bfloat16;
96
- using packed_hip_type = __nv_bfloat162;
97
-
98
- __device__ static inline float convert(hip_type x) {
99
- return __bfloat162float(x);
100
- }
101
- __device__ static inline float2 convert(packed_hip_type x) {
102
- return __bfloat1622float2(x);
103
- }
104
- __device__ static inline hip_type convert(float x) {
105
- return __float2bfloat16(x);
106
- }
107
- __device__ static inline packed_hip_type convert(float2 x) {
108
- return __float22bfloat162_rn(x);
109
- }
110
-};
111
- #endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
112
-#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >=
113
- // 12000))
114
-
115
-/* Vector POD struct to generate vectorized and packed FP16/BF16 ops
116
- for appropriate specializations of fused_add_rms_norm_kernel.
117
- Only functions that are necessary in that kernel are implemented.
118
- Alignment to 16 bytes is required to use 128-bit global memory ops.
119
- */
120
-template <typename scalar_t, int width>
121
-struct alignas(16) _f16Vec {
122
- /* Not theoretically necessary that width is a power of 2 but should
123
- almost always be the case for optimization purposes */
124
- static_assert(width > 0 && (width & (width - 1)) == 0,
125
- "Width is not a positive power of 2!");
126
- using Converter = _typeConvert<scalar_t>;
127
- using T1 = typename Converter::hip_type;
128
- using T2 = typename Converter::packed_hip_type;
129
- T1 data[width];
130
-
131
- __device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
132
- if constexpr (width % 2 == 0) {
133
-#pragma unroll
134
- for (int i = 0; i < width; i += 2) {
135
- T2 temp{data[i], data[i + 1]};
136
- temp += T2{other.data[i], other.data[i + 1]};
137
- data[i] = temp.x;
138
- data[i + 1] = temp.y;
139
- }
140
- } else {
141
-#pragma unroll
142
- for (int i = 0; i < width; ++i) data[i] += other.data[i];
143
- }
144
- return *this;
145
- }
146
-
147
- __device__ _f16Vec& operator*=(const _f16Vec<scalar_t, width>& other) {
148
- if constexpr (width % 2 == 0) {
149
-#pragma unroll
150
- for (int i = 0; i < width; i += 2) {
151
- T2 temp{data[i], data[i + 1]};
152
- temp *= T2{other.data[i], other.data[i + 1]};
153
- data[i] = temp.x;
154
- data[i + 1] = temp.y;
155
- }
156
- } else {
157
-#pragma unroll
158
- for (int i = 0; i < width; ++i) data[i] *= other.data[i];
159
- }
160
- return *this;
161
- }
162
-
163
- __device__ _f16Vec& operator*=(const float scale) {
164
- if constexpr (width % 2 == 0) {
165
-#pragma unroll
166
- for (int i = 0; i < width; i += 2) {
167
- float2 temp_f = Converter::convert(T2{data[i], data[i + 1]});
168
- temp_f.x *= scale;
169
- temp_f.y *= scale;
170
- T2 temp = Converter::convert(temp_f);
171
- data[i] = temp.x;
172
- data[i + 1] = temp.y;
173
- }
174
- } else {
175
-#pragma unroll
176
- for (int i = 0; i < width; ++i) {
177
- float temp = Converter::convert(data[i]) * scale;
178
- data[i] = Converter::convert(temp);
179
- }
180
- }
181
- return *this;
182
- }
183
-
184
- __device__ float sum_squares() const {
185
- float result = 0.0f;
186
- if constexpr (width % 2 == 0) {
187
-#pragma unroll
188
- for (int i = 0; i < width; i += 2) {
189
- float2 z = Converter::convert(T2{data[i], data[i + 1]});
190
- result += z.x * z.x + z.y * z.y;
191
- }
192
- } else {
193
-#pragma unroll
194
- for (int i = 0; i < width; ++i) {
195
- float x = Converter::convert(data[i]);
196
- result += x * x;
197
- }
198
- }
199
- return result;
200
- }
201
-};
202
-
46
/* Function specialization in the case of FP16/BF16 tensors.
47
Additional optimizations we can make in this case are
48
packed and vectorized operations, which help with the
csrc/layernorm_quant_kernels.cu
new
+234
@@ -0,0 +1,234 @@
1
+/*
2
+ * This file contains the CUDA kernels for the fused quantized layernorm.
3
+ * The kernels correspond to the kernels in layernorm_kernels.cu, except they
4
+ * also produce quantized output directly.
5
+ * Currently, only static fp8 quantization is supported.
6
+ */
7
+
8
+#include "type_convert.cuh"
9
+#include "quantization/fp8/common.cuh"
10
+#include "dispatch_utils.h"
11
+
12
+#include <torch/cuda.h>
13
+#include <c10/cuda/CUDAGuard.h>
14
+
15
+#ifndef USE_ROCM
16
+ #include <cub/cub.cuh>
17
+#else
18
+ #include <hipcub/hipcub.hpp>
19
+#endif
20
+
21
+namespace vllm {
22
+
23
+// TODO(woosuk): Further optimize this kernel.
24
+template <typename scalar_t>
25
+__global__ void rms_norm_static_fp8_quant_kernel(
26
+ FP8_TYPE* __restrict__ out, // [..., hidden_size]
27
+ const scalar_t* __restrict__ input, // [..., hidden_size]
28
+ const scalar_t* __restrict__ weight, // [hidden_size]
29
+ const float* __restrict__ scale, // [1]
30
+ const float epsilon, const int num_tokens, const int hidden_size) {
31
+ __shared__ float s_variance;
32
+ float variance = 0.0f;
33
+
34
+ for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
35
+ const float x = (float)input[blockIdx.x * hidden_size + idx];
36
+ variance += x * x;
37
+ }
38
+
39
+ using BlockReduce = cub::BlockReduce<float, 1024>;
40
+ __shared__ typename BlockReduce::TempStorage reduceStore;
41
+ variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
42
+
43
+ if (threadIdx.x == 0) {
44
+ s_variance = rsqrtf(variance / hidden_size + epsilon);
45
+ }
46
+ __syncthreads();
47
+
48
+ // invert scale to avoid division
49
+ float const scale_inv = 1.0f / *scale;
50
+
51
+ for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
52
+ float x = (float)input[blockIdx.x * hidden_size + idx];
53
+ float const out_norm = ((scalar_t)(x * s_variance)) * weight[idx];
54
+ out[blockIdx.x * hidden_size + idx] =
55
+ scaled_fp8_conversion<true>(out_norm, scale_inv);
56
+ }
57
+}
58
+
59
+/* Function specialization in the case of FP16/BF16 tensors.
60
+ Additional optimizations we can make in this case are
61
+ packed and vectorized operations, which help with the
62
+ memory latency bottleneck. */
63
+template <typename scalar_t, int width>
64
+__global__ std::enable_if_t<(width > 0) && _typeConvert<scalar_t>::exists>
65
+fused_add_rms_norm_static_fp8_quant_kernel(
66
+ FP8_TYPE* __restrict__ out, // [..., hidden_size]
67
+ scalar_t* __restrict__ input, // [..., hidden_size]
68
+ scalar_t* __restrict__ residual, // [..., hidden_size]
69
+ const scalar_t* __restrict__ weight, // [hidden_size]
70
+ const float* __restrict__ scale, // [1]
71
+ const float epsilon, const int num_tokens, const int hidden_size) {
72
+ // Sanity checks on our vector struct and type-punned pointer arithmetic
73
+ static_assert(std::is_pod_v<_f16Vec<scalar_t, width>>);
74
+ static_assert(sizeof(_f16Vec<scalar_t, width>) == sizeof(scalar_t) * width);
75
+
76
+ const int vec_hidden_size = hidden_size / width;
77
+ __shared__ float s_variance;
78
+ float variance = 0.0f;
79
+ /* These and the argument pointers are all declared `restrict` as they are
80
+ not aliased in practice. Argument pointers should not be dereferenced
81
+ in this kernel as that would be undefined behavior */
82
+ auto* __restrict__ input_v =
83
+ reinterpret_cast<_f16Vec<scalar_t, width>*>(input);
84
+ auto* __restrict__ residual_v =
85
+ reinterpret_cast<_f16Vec<scalar_t, width>*>(residual);
86
+ auto* __restrict__ weight_v =
87
+ reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
88
+
89
+ for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
90
+ int id = blockIdx.x * vec_hidden_size + idx;
91
+ _f16Vec<scalar_t, width> temp = input_v[id];
92
+ temp += residual_v[id];
93
+ variance += temp.sum_squares();
94
+ residual_v[id] = temp;
95
+ }
96
+
97
+ using BlockReduce = cub::BlockReduce<float, 1024>;
98
+ __shared__ typename BlockReduce::TempStorage reduceStore;
99
+ variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
100
+
101
+ if (threadIdx.x == 0) {
102
+ s_variance = rsqrtf(variance / hidden_size + epsilon);
103
+ }
104
+ __syncthreads();
105
+
106
+ // invert scale to avoid division
107
+ float const scale_inv = 1.0f / *scale;
108
+
109
+ for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
110
+ int id = blockIdx.x * vec_hidden_size + idx;
111
+ _f16Vec<scalar_t, width> temp = residual_v[id];
112
+ temp *= s_variance;
113
+ temp *= weight_v[idx];
114
+#pragma unroll
115
+ for (int i = 0; i < width; ++i) {
116
+ out[id * width + i] =
117
+ scaled_fp8_conversion<true>(float(temp.data[i]), scale_inv);
118
+ }
119
+ }
120
+}
121
+
122
+/* Generic fused_add_rms_norm_kernel
123
+ The width field is not used here but necessary for other specializations.
124
+ */
125
+template <typename scalar_t, int width>
126
+__global__ std::enable_if_t<(width == 0) || !_typeConvert<scalar_t>::exists>
127
+fused_add_rms_norm_static_fp8_quant_kernel(
128
+ FP8_TYPE* __restrict__ out, // [..., hidden_size]
129
+ scalar_t* __restrict__ input, // [..., hidden_size]
130
+ scalar_t* __restrict__ residual, // [..., hidden_size]
131
+ const scalar_t* __restrict__ weight, // [hidden_size]
132
+ const float* __restrict__ scale, // [1]
133
+ const float epsilon, const int num_tokens, const int hidden_size) {
134
+ __shared__ float s_variance;
135
+ float variance = 0.0f;
136
+
137
+ for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
138
+ scalar_t z = input[blockIdx.x * hidden_size + idx];
139
+ z += residual[blockIdx.x * hidden_size + idx];
140
+ float x = (float)z;
141
+ variance += x * x;
142
+ residual[blockIdx.x * hidden_size + idx] = z;
143
+ }
144
+
145
+ using BlockReduce = cub::BlockReduce<float, 1024>;
146
+ __shared__ typename BlockReduce::TempStorage reduceStore;
147
+ variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
148
+
149
+ if (threadIdx.x == 0) {
150
+ s_variance = rsqrtf(variance / hidden_size + epsilon);
151
+ }
152
+ __syncthreads();
153
+
154
+ // invert scale to avoid division
155
+ float const scale_inv = 1.0f / *scale;
156
+
157
+ for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
158
+ float x = (float)residual[blockIdx.x * hidden_size + idx];
159
+ float const out_norm = ((scalar_t)(x * s_variance)) * weight[idx];
160
+ out[blockIdx.x * hidden_size + idx] =
161
+ scaled_fp8_conversion<true>(out_norm, scale_inv);
162
+ }
163
+}
164
+
165
+} // namespace vllm
166
+
167
+void rms_norm_static_fp8_quant(torch::Tensor& out, // [..., hidden_size]
168
+ torch::Tensor& input, // [..., hidden_size]
169
+ torch::Tensor& weight, // [hidden_size]
170
+ torch::Tensor& scale, // [1]
171
+ double epsilon) {
172
+ int hidden_size = input.size(-1);
173
+ int num_tokens = input.numel() / hidden_size;
174
+
175
+ dim3 grid(num_tokens);
176
+ dim3 block(std::min(hidden_size, 1024));
177
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
178
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
179
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_kernel", [&] {
180
+ vllm::rms_norm_static_fp8_quant_kernel<scalar_t>
181
+ <<<grid, block, 0, stream>>>(
182
+ out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(),
183
+ weight.data_ptr<scalar_t>(), scale.data_ptr<float>(), epsilon,
184
+ num_tokens, hidden_size);
185
+ });
186
+}
187
+
188
+#define LAUNCH_FUSED_ADD_RMS_NORM(width) \
189
+ VLLM_DISPATCH_FLOATING_TYPES( \
190
+ input.scalar_type(), "fused_add_rms_norm_kernel", [&] { \
191
+ vllm::fused_add_rms_norm_static_fp8_quant_kernel<scalar_t, width> \
192
+ <<<grid, block, 0, stream>>>( \
193
+ out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(), \
194
+ residual.data_ptr<scalar_t>(), weight.data_ptr<scalar_t>(), \
195
+ scale.data_ptr<float>(), epsilon, num_tokens, hidden_size); \
196
+ });
197
+
198
+void fused_add_rms_norm_static_fp8_quant(
199
+ torch::Tensor& out, // [..., hidden_size],
200
+ torch::Tensor& input, // [..., hidden_size]
201
+ torch::Tensor& residual, // [..., hidden_size]
202
+ torch::Tensor& weight, // [hidden_size]
203
+ torch::Tensor& scale, // [1]
204
+ double epsilon) {
205
+ int hidden_size = input.size(-1);
206
+ int num_tokens = input.numel() / hidden_size;
207
+
208
+ dim3 grid(num_tokens);
209
+ /* This kernel is memory-latency bound in many scenarios.
210
+ When num_tokens is large, a smaller block size allows
211
+ for increased block occupancy on CUs and better latency
212
+ hiding on global mem ops. */
213
+ const int max_block_size = (num_tokens < 256) ? 1024 : 256;
214
+ dim3 block(std::min(hidden_size, max_block_size));
215
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
216
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
217
+ /*If the tensor types are FP16/BF16, try to use the optimized kernel
218
+ with packed + vectorized ops.
219
+ Max optimization is achieved with a width-8 vector of FP16/BF16s
220
+ since we can load at most 128 bits at once in a global memory op.
221
+ However, this requires each tensor's data to be aligned to 16
222
+ bytes.
223
+ */
224
+ auto inp_ptr = reinterpret_cast<std::uintptr_t>(input.data_ptr());
225
+ auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
226
+ auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight.data_ptr());
227
+ bool ptrs_are_aligned =
228
+ inp_ptr % 16 == 0 && res_ptr % 16 == 0 && wt_ptr % 16 == 0;
229
+ if (ptrs_are_aligned && hidden_size % 8 == 0) {
230
+ LAUNCH_FUSED_ADD_RMS_NORM(8);
231
+ } else {
232
+ LAUNCH_FUSED_ADD_RMS_NORM(0);
233
+ }
234
+}
csrc/ops.h
+10
@@ -56,6 +56,16 @@ void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
56
void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
57
torch::Tensor& weight, double epsilon);
58
59
+void rms_norm_static_fp8_quant(torch::Tensor& out, torch::Tensor& input,
60
+ torch::Tensor& weight, torch::Tensor& scale,
61
+ double epsilon);
62
+
63
+void fused_add_rms_norm_static_fp8_quant(torch::Tensor& out,
64
+ torch::Tensor& input,
65
+ torch::Tensor& residual,
66
+ torch::Tensor& weight,
67
+ torch::Tensor& scale, double epsilon);
68
+
69
void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
70
torch::Tensor& key, int64_t head_size,
71
torch::Tensor& cos_sin_cache, bool is_neox);
csrc/quantization/fp8/common.cu
+3
-172
@@ -1,185 +1,16 @@
1
-#include <ATen/cuda/CUDAContext.h>
2
-#include <torch/all.h>
3
-#include <c10/cuda/CUDAGuard.h>
4
-
5
-#include <cmath>
6
-
7
-#include "cuda_compat.h"
1
+#include "common.cuh"
2
#include "dispatch_utils.h"
3
4
+#include <c10/cuda/CUDAGuard.h>
5
+
6
#ifndef USE_ROCM
11
- #include <cub/util_type.cuh>
7
#include <cub/cub.cuh>
8
#else
14
- #include <hipcub/util_type.hpp>
9
#include <hipcub/hipcub.hpp>
10
#endif
11
18
-#ifndef USE_ROCM
19
-using FP8_TYPE = c10::Float8_e4m3fn;
20
-C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX =
21
- std::numeric_limits<FP8_TYPE>::max();
22
-#else
23
- #include "amd/hip_float8.h"
24
-using FP8_TYPE = c10::Float8_e4m3fnuz;
25
-// Using the default max value from pytorch (240.0) will cause accuracy
26
-// issue when running dynamic quantization. Here use 224.0f for rocm.
27
-constexpr auto FP8_E4M3_MAX = 224.0f;
28
-#endif
29
-
12
namespace vllm {
13
32
-__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
33
- float old;
34
- old = (value >= 0)
35
- ? __int_as_float(atomicMax((int*)addr, __float_as_int(value)))
36
- : __uint_as_float(
37
- atomicMin((unsigned int*)addr, __float_as_uint(value)));
38
-
39
- return old;
40
-}
41
-
42
-template <bool is_scale_inverted>
43
-__device__ __forceinline__ FP8_TYPE scaled_fp8_conversion(float const val,
44
- float const scale) {
45
- float x = 0.0f;
46
- if constexpr (is_scale_inverted) {
47
- x = val * scale;
48
- } else {
49
- x = val / scale;
50
- }
51
-
52
- float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
53
-#ifndef USE_ROCM
54
- return static_cast<c10::Float8_e4m3fn>(r);
55
-#else
56
- // Use hardware cvt instruction for fp8 on rocm
57
- return c10::Float8_e4m3fnuz(hip_fp8(r).data,
58
- c10::Float8_e4m3fnuz::from_bits());
59
-#endif
60
-}
61
-
62
-// Compute the absolute maximum m of the input tensor and store
63
-// m / float8_e4m3::max() in *scale. Each thread block performs a
64
-// reduction tree and the memory in scale is atomically updated.
65
-// So to get the right answer, *scale needs to be initialized to
66
-// a value <= 0.0 and we need to wait for all thread blocks to
67
-// finish before consuming *scale.
68
-template <typename scalar_t>
69
-__global__ void segmented_max_reduction(float* __restrict__ scale,
70
- const scalar_t* __restrict__ input,
71
- int64_t num_elems) {
72
- __shared__ float cache[1024];
73
- int64_t i = blockDim.x * blockIdx.x + threadIdx.x;
74
-
75
- // First store maximum for all values processes by
76
- // the current thread in cache[threadIdx.x]
77
- scalar_t tmp = 0.0;
78
- while (i < num_elems) {
79
- float x = static_cast<float>(input[i]);
80
- tmp = max(tmp, fabs(x));
81
- i += blockDim.x * gridDim.x;
82
- }
83
- cache[threadIdx.x] = tmp;
84
-
85
- __syncthreads();
86
-
87
- // Now perform parallel reduction within the thread block
88
- int ib = blockDim.x / 2;
89
- while (ib != 0) {
90
- if (threadIdx.x < ib && cache[threadIdx.x + ib] > cache[threadIdx.x]) {
91
- cache[threadIdx.x] = cache[threadIdx.x + ib];
92
- }
93
- __syncthreads();
94
- ib /= 2;
95
- }
96
- // Finally, since cache[0] contains the maximum for this thread block,
97
- // atomically write the max to the target location
98
- if (threadIdx.x == 0) {
99
- atomicMaxFloat(scale, cache[0] / FP8_E4M3_MAX);
100
- }
101
-}
102
-
103
-template <typename scalar_t>
104
-struct __align__(8) vec4_t {
105
- scalar_t x;
106
- scalar_t y;
107
- scalar_t z;
108
- scalar_t w;
109
-};
110
-
111
-typedef struct __align__(4) {
112
- FP8_TYPE x;
113
- FP8_TYPE y;
114
- FP8_TYPE z;
115
- FP8_TYPE w;
116
-}
117
-float8x4_t;
118
-
119
-template <typename scalar_t>
120
-__device__ float thread_max_vec(scalar_t const* __restrict__ input,
121
- int64_t const num_elems, int const tid,
122
- int const step) {
123
- // Vectorized input/output to better utilize memory bandwidth.
124
- vec4_t<scalar_t> const* vectorized_in =
125
- reinterpret_cast<vec4_t<scalar_t> const*>(input);
126
-
127
- int64_t const num_vec_elems = num_elems >> 2;
128
- float absmax_val = 0.0f;
129
-
130
-#pragma unroll 4
131
- for (int64_t i = tid; i < num_vec_elems; i += step) {
132
- vec4_t<scalar_t> in_vec = vectorized_in[i];
133
- absmax_val = max(absmax_val, fabs(in_vec.x));
134
- absmax_val = max(absmax_val, fabs(in_vec.y));
135
- absmax_val = max(absmax_val, fabs(in_vec.z));
136
- absmax_val = max(absmax_val, fabs(in_vec.w));
137
- }
138
-
139
- // Handle the remaining elements if num_elems is not divisible by 4
140
- for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
141
- absmax_val = max(absmax_val, fabs(input[i]));
142
- }
143
-
144
- return absmax_val;
145
-}
146
-
147
-template <typename scalar_t, bool is_scale_inverted>
148
-__device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
149
- scalar_t const* __restrict__ input,
150
- float const scale,
151
- int64_t const num_elems,
152
- int const tid, int const step) {
153
- // Vectorized input/output to better utilize memory bandwidth.
154
- vec4_t<scalar_t> const* vectorized_in =
155
- reinterpret_cast<vec4_t<scalar_t> const*>(input);
156
- float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
157
-
158
- int64_t const num_vec_elems = num_elems >> 2;
159
-
160
-#pragma unroll 4
161
- for (int64_t i = tid; i < num_vec_elems; i += step) {
162
- vec4_t<scalar_t> in_vec = vectorized_in[i];
163
- float8x4_t out_vec;
164
-
165
- out_vec.x = scaled_fp8_conversion<is_scale_inverted>(
166
- static_cast<float>(in_vec.x), scale);
167
- out_vec.y = scaled_fp8_conversion<is_scale_inverted>(
168
- static_cast<float>(in_vec.y), scale);
169
- out_vec.z = scaled_fp8_conversion<is_scale_inverted>(
170
- static_cast<float>(in_vec.z), scale);
171
- out_vec.w = scaled_fp8_conversion<is_scale_inverted>(
172
- static_cast<float>(in_vec.w), scale);
173
- vectorized_out[i] = out_vec;
174
- }
175
-
176
- // Handle the remaining elements if num_elems is not divisible by 4
177
- for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
178
- out[i] = scaled_fp8_conversion<is_scale_inverted>(
179
- static_cast<float>(input[i]), scale);
180
- }
181
-}
182
-
14
template <typename scalar_t>
15
__global__ void scaled_fp8_quant_kernel(FP8_TYPE* __restrict__ out,
16
const scalar_t* __restrict__ input,
csrc/quantization/fp8/common.cuh
new
+172
@@ -0,0 +1,172 @@
1
+#pragma once
2
+
3
+#include <cmath>
4
+
5
+#ifndef USE_ROCM
6
+ #include <c10/util/Float8_e4m3fn.h>
7
+using FP8_TYPE = c10::Float8_e4m3fn;
8
+C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX =
9
+ std::numeric_limits<FP8_TYPE>::max();
10
+#else
11
+ #include <c10/util/Float8_e4m3fnuz.h>
12
+ #include "amd/hip_float8.h"
13
+using FP8_TYPE = c10::Float8_e4m3fnuz;
14
+// Using the default max value from pytorch (240.0) will cause accuracy
15
+// issue when running dynamic quantization. Here use 224.0f for rocm.
16
+constexpr auto FP8_E4M3_MAX = 224.0f;
17
+#endif
18
+
19
+namespace vllm {
20
+
21
+__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
22
+ float old;
23
+ old = (value >= 0)
24
+ ? __int_as_float(atomicMax((int*)addr, __float_as_int(value)))
25
+ : __uint_as_float(
26
+ atomicMin((unsigned int*)addr, __float_as_uint(value)));
27
+
28
+ return old;
29
+}
30
+
31
+template <bool is_scale_inverted>
32
+__device__ __forceinline__ FP8_TYPE scaled_fp8_conversion(float const val,
33
+ float const scale) {
34
+ float x = 0.0f;
35
+ if constexpr (is_scale_inverted) {
36
+ x = val * scale;
37
+ } else {
38
+ x = val / scale;
39
+ }
40
+
41
+ float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
42
+#ifndef USE_ROCM
43
+ return static_cast<c10::Float8_e4m3fn>(r);
44
+#else
45
+ // Use hardware cvt instruction for fp8 on rocm
46
+ return c10::Float8_e4m3fnuz(hip_fp8(r).data,
47
+ c10::Float8_e4m3fnuz::from_bits());
48
+#endif
49
+}
50
+
51
+// Compute the absolute maximum m of the input tensor and store
52
+// m / float8_e4m3::max() in *scale. Each thread block performs a
53
+// reduction tree and the memory in scale is atomically updated.
54
+// So to get the right answer, *scale needs to be initialized to
55
+// a value <= 0.0 and we need to wait for all thread blocks to
56
+// finish before consuming *scale.
57
+template <typename scalar_t>
58
+__global__ void segmented_max_reduction(float* __restrict__ scale,
59
+ const scalar_t* __restrict__ input,
60
+ int64_t num_elems) {
61
+ __shared__ float cache[1024];
62
+ int64_t i = blockDim.x * blockIdx.x + threadIdx.x;
63
+
64
+ // First store maximum for all values processes by
65
+ // the current thread in cache[threadIdx.x]
66
+ scalar_t tmp = 0.0;
67
+ while (i < num_elems) {
68
+ float x = static_cast<float>(input[i]);
69
+ tmp = max(tmp, fabs(x));
70
+ i += blockDim.x * gridDim.x;
71
+ }
72
+ cache[threadIdx.x] = tmp;
73
+
74
+ __syncthreads();
75
+
76
+ // Now perform parallel reduction within the thread block
77
+ int ib = blockDim.x / 2;
78
+ while (ib != 0) {
79
+ if (threadIdx.x < ib && cache[threadIdx.x + ib] > cache[threadIdx.x]) {
80
+ cache[threadIdx.x] = cache[threadIdx.x + ib];
81
+ }
82
+ __syncthreads();
83
+ ib /= 2;
84
+ }
85
+ // Finally, since cache[0] contains the maximum for this thread block,
86
+ // atomically write the max to the target location
87
+ if (threadIdx.x == 0) {
88
+ atomicMaxFloat(scale, cache[0] / FP8_E4M3_MAX);
89
+ }
90
+}
91
+
92
+template <typename scalar_t>
93
+struct __align__(8) vec4_t {
94
+ scalar_t x;
95
+ scalar_t y;
96
+ scalar_t z;
97
+ scalar_t w;
98
+};
99
+
100
+typedef struct __align__(4) {
101
+ FP8_TYPE x;
102
+ FP8_TYPE y;
103
+ FP8_TYPE z;
104
+ FP8_TYPE w;
105
+}
106
+float8x4_t;
107
+
108
+template <typename scalar_t>
109
+__device__ float thread_max_vec(scalar_t const* __restrict__ input,
110
+ int64_t const num_elems, int const tid,
111
+ int const step) {
112
+ // Vectorized input/output to better utilize memory bandwidth.
113
+ vec4_t<scalar_t> const* vectorized_in =
114
+ reinterpret_cast<vec4_t<scalar_t> const*>(input);
115
+
116
+ int64_t const num_vec_elems = num_elems >> 2;
117
+ float absmax_val = 0.0f;
118
+
119
+#pragma unroll 4
120
+ for (int64_t i = tid; i < num_vec_elems; i += step) {
121
+ vec4_t<scalar_t> in_vec = vectorized_in[i];
122
+ absmax_val = max(absmax_val, fabs(in_vec.x));
123
+ absmax_val = max(absmax_val, fabs(in_vec.y));
124
+ absmax_val = max(absmax_val, fabs(in_vec.z));
125
+ absmax_val = max(absmax_val, fabs(in_vec.w));
126
+ }
127
+
128
+ // Handle the remaining elements if num_elems is not divisible by 4
129
+ for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
130
+ absmax_val = max(absmax_val, fabs(input[i]));
131
+ }
132
+
133
+ return absmax_val;
134
+}
135
+
136
+template <typename scalar_t, bool is_scale_inverted>
137
+__device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
138
+ scalar_t const* __restrict__ input,
139
+ float const scale,
140
+ int64_t const num_elems,
141
+ int const tid, int const step) {
142
+ // Vectorized input/output to better utilize memory bandwidth.
143
+ vec4_t<scalar_t> const* vectorized_in =
144
+ reinterpret_cast<vec4_t<scalar_t> const*>(input);
145
+ float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
146
+
147
+ int64_t const num_vec_elems = num_elems >> 2;
148
+
149
+#pragma unroll 4
150
+ for (int64_t i = tid; i < num_vec_elems; i += step) {
151
+ vec4_t<scalar_t> in_vec = vectorized_in[i];
152
+ float8x4_t out_vec;
153
+
154
+ out_vec.x = scaled_fp8_conversion<is_scale_inverted>(
155
+ static_cast<float>(in_vec.x), scale);
156
+ out_vec.y = scaled_fp8_conversion<is_scale_inverted>(
157
+ static_cast<float>(in_vec.y), scale);
158
+ out_vec.z = scaled_fp8_conversion<is_scale_inverted>(
159
+ static_cast<float>(in_vec.z), scale);
160
+ out_vec.w = scaled_fp8_conversion<is_scale_inverted>(
161
+ static_cast<float>(in_vec.w), scale);
162
+ vectorized_out[i] = out_vec;
163
+ }
164
+
165
+ // Handle the remaining elements if num_elems is not divisible by 4
166
+ for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
167
+ out[i] = scaled_fp8_conversion<is_scale_inverted>(
168
+ static_cast<float>(input[i]), scale);
169
+ }
170
+}
171
+
172
+} // namespace vllm
\ No newline at end of file
csrc/torch_bindings.cpp
+25
-6
@@ -101,7 +101,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
101
// Layernorm
102
// Apply Root Mean Square (RMS) Normalization to the input tensor.
103
ops.def(
104
- "rms_norm(Tensor! out, Tensor input, Tensor weight, float epsilon) -> "
104
+ "rms_norm(Tensor! result, Tensor input, Tensor weight, float epsilon) -> "
105
"()");
106
ops.impl("rms_norm", torch::kCUDA, &rms_norm);
107
@@ -111,6 +111,23 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
111
"float epsilon) -> ()");
112
ops.impl("fused_add_rms_norm", torch::kCUDA, &fused_add_rms_norm);
113
114
+ // Layernorm-quant
115
+ // Apply Root Mean Square (RMS) Normalization to the input tensor.
116
+ ops.def(
117
+ "rms_norm_static_fp8_quant(Tensor! result, Tensor input, Tensor weight, "
118
+ "Tensor scale, float epsilon) -> "
119
+ "()");
120
+ ops.impl("rms_norm_static_fp8_quant", torch::kCUDA,
121
+ &rms_norm_static_fp8_quant);
122
+
123
+ // In-place fused Add and RMS Normalization.
124
+ ops.def(
125
+ "fused_add_rms_norm_static_fp8_quant(Tensor! result, Tensor input, "
126
+ "Tensor! residual, Tensor weight, "
127
+ "Tensor scale, float epsilon) -> ()");
128
+ ops.impl("fused_add_rms_norm_static_fp8_quant", torch::kCUDA,
129
+ &fused_add_rms_norm_static_fp8_quant);
130
+
131
// Rotary embedding
132
// Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
133
ops.def(
@@ -322,18 +339,20 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
339
340
// Compute FP8 quantized tensor for given scaling factor.
341
ops.def(
325
- "static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
342
+ "static_scaled_fp8_quant(Tensor! result, Tensor input, Tensor scale) -> "
343
+ "()");
344
ops.impl("static_scaled_fp8_quant", torch::kCUDA, &static_scaled_fp8_quant);
345
346
// Compute dynamic-per-tensor FP8 quantized tensor and scaling factor.
347
ops.def(
330
- "dynamic_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
348
+ "dynamic_scaled_fp8_quant(Tensor! result, Tensor input, Tensor! scale) "
349
+ "-> "
350
"()");
351
ops.impl("dynamic_scaled_fp8_quant", torch::kCUDA, &dynamic_scaled_fp8_quant);
352
353
// Compute dynamic-per-token FP8 quantized tensor and scaling factor.
354
ops.def(
336
- "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, "
355
+ "dynamic_per_token_scaled_fp8_quant(Tensor! result, Tensor input, "
356
"Tensor! scale, Tensor? scale_ub) -> "
357
"()");
358
ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
@@ -341,13 +360,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
360
361
// Compute int8 quantized tensor for given scaling factor.
362
ops.def(
344
- "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
363
+ "static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
364
"Tensor? azp) -> ()");
365
ops.impl("static_scaled_int8_quant", torch::kCUDA, &static_scaled_int8_quant);
366
367
// Compute int8 quantized tensor and scaling factor
368
ops.def(
350
- "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale, "
369
+ "dynamic_scaled_int8_quant(Tensor! result, Tensor input, Tensor! scale, "
370
"Tensor!? azp) -> ()");
371
ops.impl("dynamic_scaled_int8_quant", torch::kCUDA,
372
&dynamic_scaled_int8_quant);
csrc/type_convert.cuh
new
+165
@@ -0,0 +1,165 @@
1
+#pragma once
2
+
3
+#include <torch/all.h>
4
+
5
+#ifndef USE_ROCM
6
+ #include <cuda_bf16.h>
7
+ #include <cuda_fp16.h>
8
+#else
9
+ #include <hip/hip_bf16.h>
10
+ #include <hip/hip_fp16.h>
11
+
12
+using __nv_bfloat16 = __hip_bfloat16;
13
+using __nv_bfloat162 = __hip_bfloat162;
14
+#endif
15
+
16
+namespace vllm {
17
+/* Converter structs for the conversion from torch types to HIP/CUDA types,
18
+ and the associated type conversions within HIP/CUDA. These helpers need
19
+ to be implemented for now because the relevant type conversion
20
+ operators/constructors are not consistently implemented by HIP/CUDA, so
21
+ a generic conversion via type casts cannot be implemented.
22
+
23
+ Each struct should have the member static constexpr bool `exists`:
24
+ If false, the optimized kernel is not used for the corresponding torch type.
25
+ If true, the struct should be fully defined as shown in the examples below.
26
+ */
27
+template <typename torch_type>
28
+struct _typeConvert {
29
+ static constexpr bool exists = false;
30
+};
31
+
32
+#if defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
33
+// CUDA < 12.0 runs into issues with packed type conversion
34
+template <>
35
+struct _typeConvert<c10::Half> {
36
+ static constexpr bool exists = true;
37
+ using hip_type = __half;
38
+ using packed_hip_type = __half2;
39
+
40
+ __device__ static inline float convert(hip_type x) { return __half2float(x); }
41
+ __device__ static inline float2 convert(packed_hip_type x) {
42
+ return __half22float2(x);
43
+ }
44
+ __device__ static inline hip_type convert(float x) {
45
+ return __float2half_rn(x);
46
+ }
47
+ __device__ static inline packed_hip_type convert(float2 x) {
48
+ return __float22half2_rn(x);
49
+ }
50
+};
51
+
52
+ #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
53
+// CUDA_ARCH < 800 does not have BF16 support
54
+// TODO: Add in ROCm support once public headers handle bf16 maturely
55
+template <>
56
+struct _typeConvert<c10::BFloat16> {
57
+ static constexpr bool exists = true;
58
+ using hip_type = __nv_bfloat16;
59
+ using packed_hip_type = __nv_bfloat162;
60
+
61
+ __device__ static inline float convert(hip_type x) {
62
+ return __bfloat162float(x);
63
+ }
64
+ __device__ static inline float2 convert(packed_hip_type x) {
65
+ return __bfloat1622float2(x);
66
+ }
67
+ __device__ static inline hip_type convert(float x) {
68
+ return __float2bfloat16(x);
69
+ }
70
+ __device__ static inline packed_hip_type convert(float2 x) {
71
+ return __float22bfloat162_rn(x);
72
+ }
73
+};
74
+ #endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
75
+#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >=
76
+ // 12000))
77
+
78
+/* Vector POD struct to generate vectorized and packed FP16/BF16 ops
79
+ for appropriate specializations of fused_add_rms_norm_kernel.
80
+ Only functions that are necessary in that kernel are implemented.
81
+ Alignment to 16 bytes is required to use 128-bit global memory ops.
82
+ */
83
+template <typename scalar_t, int width>
84
+struct alignas(16) _f16Vec {
85
+ /* Not theoretically necessary that width is a power of 2 but should
86
+ almost always be the case for optimization purposes */
87
+ static_assert(width > 0 && (width & (width - 1)) == 0,
88
+ "Width is not a positive power of 2!");
89
+ using Converter = _typeConvert<scalar_t>;
90
+ using T1 = typename Converter::hip_type;
91
+ using T2 = typename Converter::packed_hip_type;
92
+ T1 data[width];
93
+
94
+ __device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
95
+ if constexpr (width % 2 == 0) {
96
+#pragma unroll
97
+ for (int i = 0; i < width; i += 2) {
98
+ T2 temp{data[i], data[i + 1]};
99
+ temp += T2{other.data[i], other.data[i + 1]};
100
+ data[i] = temp.x;
101
+ data[i + 1] = temp.y;
102
+ }
103
+ } else {
104
+#pragma unroll
105
+ for (int i = 0; i < width; ++i) data[i] += other.data[i];
106
+ }
107
+ return *this;
108
+ }
109
+
110
+ __device__ _f16Vec& operator*=(const _f16Vec<scalar_t, width>& other) {
111
+ if constexpr (width % 2 == 0) {
112
+#pragma unroll
113
+ for (int i = 0; i < width; i += 2) {
114
+ T2 temp{data[i], data[i + 1]};
115
+ temp *= T2{other.data[i], other.data[i + 1]};
116
+ data[i] = temp.x;
117
+ data[i + 1] = temp.y;
118
+ }
119
+ } else {
120
+#pragma unroll
121
+ for (int i = 0; i < width; ++i) data[i] *= other.data[i];
122
+ }
123
+ return *this;
124
+ }
125
+
126
+ __device__ _f16Vec& operator*=(const float scale) {
127
+ if constexpr (width % 2 == 0) {
128
+#pragma unroll
129
+ for (int i = 0; i < width; i += 2) {
130
+ float2 temp_f = Converter::convert(T2{data[i], data[i + 1]});
131
+ temp_f.x *= scale;
132
+ temp_f.y *= scale;
133
+ T2 temp = Converter::convert(temp_f);
134
+ data[i] = temp.x;
135
+ data[i + 1] = temp.y;
136
+ }
137
+ } else {
138
+#pragma unroll
139
+ for (int i = 0; i < width; ++i) {
140
+ float temp = Converter::convert(data[i]) * scale;
141
+ data[i] = Converter::convert(temp);
142
+ }
143
+ }
144
+ return *this;
145
+ }
146
+
147
+ __device__ float sum_squares() const {
148
+ float result = 0.0f;
149
+ if constexpr (width % 2 == 0) {
150
+#pragma unroll
151
+ for (int i = 0; i < width; i += 2) {
152
+ float2 z = Converter::convert(T2{data[i], data[i + 1]});
153
+ result += z.x * z.x + z.y * z.y;
154
+ }
155
+ } else {
156
+#pragma unroll
157
+ for (int i = 0; i < width; ++i) {
158
+ float x = Converter::convert(data[i]);
159
+ result += x * x;
160
+ }
161
+ }
162
+ return result;
163
+ }
164
+};
165
+} // namespace vllm
\ No newline at end of file
tests/compile/backend.py
new
+33
@@ -0,0 +1,33 @@
1
+from copy import deepcopy
2
+from typing import Callable
3
+
4
+import torch
5
+
6
+
7
+class TestBackend:
8
+ """
9
+ This class provides a simple Inductor backend that can be used for testing.
10
+ It takes a list of custom passes and runs them after Inductor's passes.
11
+ It also saves the graph before and after the custom passes for inspection.
12
+ """
13
+
14
+ def __init__(self, *args: Callable[[torch.fx.Graph], None]):
15
+ self.custom_passes = args
16
+ from torch._inductor import config
17
+ self.current_config = config.shallow_copy_dict()
18
+ self.current_config['post_grad_custom_post_pass'] = self.post_pass
19
+
20
+ def __call__(self, graph: torch.fx.GraphModule, example_inputs):
21
+ from torch._inductor.compile_fx import compile_fx
22
+ return compile_fx(graph,
23
+ example_inputs,
24
+ config_patches=self.current_config)
25
+
26
+ def post_pass(self, graph: torch.fx.Graph):
27
+ self.graph_pre_pass = deepcopy(graph)
28
+ for pass_ in self.custom_passes:
29
+ pass_(graph)
30
+
31
+ self.graph_post_pass = deepcopy(graph)
32
+ # assign by reference, will reflect the final state of the graph
33
+ self.final_graph = graph
tests/compile/test_fusion.py
new
+92
@@ -0,0 +1,92 @@
1
+import pytest
2
+import torch
3
+from compressed_tensors.quantization import FP8_DTYPE
4
+
5
+import vllm.envs as envs
6
+from vllm.compilation.config import CompilationConfig
7
+from vllm.compilation.fusion import (FusionPass, find_auto_fn,
8
+ find_auto_fn_maybe)
9
+from vllm.compilation.reshapes import RedundantReshapesPass
10
+from vllm.model_executor.layers.layernorm import RMSNorm
11
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12
+ apply_fp8_linear)
13
+
14
+from .backend import TestBackend
15
+
16
+
17
+class TestModel(torch.nn.Module):
18
+
19
+ def __init__(self, hidden_size: int, eps: float, *args, **kwargs):
20
+ super().__init__(*args, **kwargs)
21
+ self.norm = [RMSNorm(hidden_size, eps) for _ in range(3)]
22
+ self.scale = [torch.rand(1, dtype=torch.float32) for _ in range(4)]
23
+ self.w = [
24
+ torch.rand(hidden_size, hidden_size).to(dtype=FP8_DTYPE).t()
25
+ for _ in range(2)
26
+ ]
27
+
28
+ def forward(self, x):
29
+ resid = torch.relu(x)
30
+ y = self.norm[0](x)
31
+
32
+ x2 = apply_fp8_linear(y, self.w[0], self.scale[0], self.scale[1])
33
+ # make sure resid is used for replacement to work
34
+ y2, resid = self.norm[1](x2, resid)
35
+
36
+ x3 = apply_fp8_linear(y2, self.w[1], self.scale[2], self.scale[3])
37
+ y3, resid = self.norm[2](x3, resid) # use resid here
38
+ return y3
39
+
40
+
41
+# Init does pattern registration, which can only happen once
42
+config = CompilationConfig(enable_fusion=True)
43
+reshape_pass = RedundantReshapesPass(config)
44
+fusion_pass = FusionPass.instance(config)
45
+
46
+
47
+@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
48
+@pytest.mark.parametrize("hidden_size", [64, 3392, 4096])
49
+@pytest.mark.parametrize("num_tokens", [7, 256, 533, 2048, 2049])
50
+@pytest.mark.parametrize("eps", [1e-5, 1e-6])
51
+@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
52
+ reason="Only test on CUDA")
53
+def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
54
+ torch.set_default_device("cuda")
55
+ torch.set_default_dtype(torch.float16)
56
+
57
+ if eps != 1e-5:
58
+ pytest.skip("Only test eps=1e-5 for now")
59
+
60
+ # Reshape pass is needed for the fusion pass to work
61
+ backend = TestBackend(reshape_pass, fusion_pass)
62
+ model = TestModel(hidden_size, eps)
63
+
64
+ # First dimension dynamic
65
+ x = torch.rand(num_tokens, hidden_size)
66
+ torch._dynamo.mark_dynamic(x, 0)
67
+
68
+ result = model(x)
69
+
70
+ model2 = torch.compile(model, backend=backend)
71
+ result2 = model2(x)
72
+
73
+ # Check that it gives the same answer
74
+ torch.testing.assert_close(result, result2, atol=1e-3, rtol=1e-3)
75
+
76
+ # Check substitution worked
77
+ pre_nodes = backend.graph_pre_pass.nodes
78
+ post_nodes = backend.graph_post_pass.nodes
79
+
80
+ rms_quant = torch.ops._C.rms_norm_static_fp8_quant.default
81
+ add_rms_quant = torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
82
+ fp8_quant = torch.ops._C.static_scaled_fp8_quant.default
83
+
84
+ # In pre-nodes, fp8 quant should be present and fused kernels should not
85
+ assert find_auto_fn_maybe(pre_nodes, rms_quant) is None
86
+ assert find_auto_fn_maybe(pre_nodes, add_rms_quant) is None
87
+ find_auto_fn(pre_nodes, fp8_quant)
88
+
89
+ # In post-nodes, fused kernels should be present and fp8 quant should not
90
+ find_auto_fn(post_nodes, rms_quant)
91
+ find_auto_fn(post_nodes, add_rms_quant)
92
+ assert find_auto_fn_maybe(post_nodes, fp8_quant) is None
tests/kernels/test_layernorm.py
+74
-1
@@ -1,13 +1,14 @@
1
import pytest
2
import torch
3
4
+from tests.kernels.quant_utils import FP8_DTYPE
5
from tests.kernels.utils import opcheck
6
from vllm.model_executor.layers.layernorm import RMSNorm
7
from vllm.platforms import current_platform
8
9
DTYPES = [torch.half, torch.bfloat16, torch.float]
10
NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
10
-HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192,
11
+HIDDEN_SIZES = [8, 768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192,
12
8199] # Arbitrary values for testing
13
ADD_RESIDUAL = [False, True]
14
SEEDS = [0]
@@ -59,3 +60,75 @@ def test_rms_norm(
60
else:
61
opcheck(torch.ops._C.rms_norm,
62
(out, x, layer.weight.data, layer.variance_epsilon))
63
+
64
+
65
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
66
+@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
67
+@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
68
+@pytest.mark.parametrize("dtype", DTYPES)
69
+@pytest.mark.parametrize("quant_scale", [1.0, 0.01, 10.0])
70
+@pytest.mark.parametrize("seed", SEEDS)
71
+@pytest.mark.parametrize("device", CUDA_DEVICES)
72
+def test_fused_rms_norm_quant(
73
+ num_tokens: int,
74
+ hidden_size: int,
75
+ add_residual: bool,
76
+ dtype: torch.dtype,
77
+ quant_scale: float,
78
+ seed: int,
79
+ device: str,
80
+) -> None:
81
+ current_platform.seed_everything(seed)
82
+ torch.set_default_device(device)
83
+
84
+ weight = torch.empty(hidden_size, dtype=dtype).normal_(mean=1.0, std=0.1)
85
+ scale = 1 / (2 * hidden_size)
86
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype)
87
+ x *= scale
88
+ if add_residual:
89
+ residual = torch.randn_like(x) * scale
90
+ residual_fused = residual.clone()
91
+ else:
92
+ residual = residual_fused = None
93
+
94
+ out_norm = torch.empty_like(x)
95
+ out_quant = torch.empty_like(x, dtype=FP8_DTYPE)
96
+ out_quant_fused = torch.empty_like(out_quant)
97
+
98
+ quant_scale_t = torch.tensor(quant_scale, dtype=torch.float32)
99
+
100
+ if add_residual:
101
+ torch.ops._C.fused_add_rms_norm_static_fp8_quant(
102
+ out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6)
103
+
104
+ # Unfused kernel is in-place so it goes second
105
+ # Also use a separate clone of x to avoid modifying the input
106
+ x_unfused = x.clone()
107
+ torch.ops._C.fused_add_rms_norm(x_unfused, residual, weight, 1e-6)
108
+ torch.ops._C.static_scaled_fp8_quant(out_quant, x_unfused,
109
+ quant_scale_t)
110
+
111
+ torch.cuda.synchronize()
112
+ torch.testing.assert_close(residual_fused,
113
+ residual,
114
+ atol=1e-2,
115
+ rtol=1e-2)
116
+
117
+ opcheck(
118
+ torch.ops._C.fused_add_rms_norm_static_fp8_quant,
119
+ (out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6))
120
+ else:
121
+ torch.ops._C.rms_norm_static_fp8_quant(out_quant_fused, x, weight,
122
+ quant_scale_t, 1e-6)
123
+
124
+ torch.ops._C.rms_norm(out_norm, x, weight, 1e-6)
125
+ torch.ops._C.static_scaled_fp8_quant(out_quant, out_norm,
126
+ quant_scale_t)
127
+
128
+ opcheck(torch.ops._C.rms_norm_static_fp8_quant,
129
+ (out_quant_fused, x, weight, quant_scale_t, 1e-6))
130
+
131
+ torch.testing.assert_close(out_quant_fused.to(dtype=torch.float32),
132
+ out_quant.to(dtype=torch.float32),
133
+ atol=1e-3,
134
+ rtol=1e-3)
vllm/compilation/backends.py
+94
-15
@@ -2,7 +2,8 @@ import copy
2
import dataclasses
3
import operator
4
from contextlib import ExitStack
5
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
5
+from typing import (Any, Callable, Dict, List, Optional, Sequence, Set, Tuple,
6
+ Union)
7
from unittest.mock import patch
8
9
import torch
@@ -10,11 +11,13 @@ import torch.fx as fx
11
12
import vllm.envs as envs
13
from vllm.logger import init_logger
13
-from vllm.utils import weak_ref_tensors
14
+from vllm.utils import combine_fx_passes, weak_ref_tensors
15
16
from .config import CompilationConfig
17
from .counter import compilation_counter
18
+from .fusion import FusionPass
19
from .levels import CompilationLevel
20
+from .reshapes import RedundantReshapesPass
21
22
logger = init_logger(__name__)
23
@@ -99,28 +102,74 @@ def fix_functionalization(graph: fx.Graph):
102
user.replace_all_uses_with(replace_node)
103
nodes_to_remove.append(user)
104
nodes_to_remove.append(node)
105
+ elif (node.args[0] ==
106
+ torch.ops._C.fused_add_rms_norm_static_fp8_quant.default):
107
+ # manual replace for fused_add_rms_norm_static_fp8_quant
108
+ # this is the most effective optimization for llama
109
+ # failing to do this will result in many unnecessary copies
110
+
111
+ kwargs = node.kwargs
112
+
113
+ result = kwargs['result']
114
+ residual = kwargs['residual']
115
+
116
+ # Create a new call to
117
+ # torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
118
+ with graph.inserting_before(node):
119
+ # just insert the call to the custom op
120
+ # NOTE: don't run dead code elimination,
121
+ # otherwise this op will be removed
122
+ graph.call_function(
123
+ torch.ops._C.fused_add_rms_norm_static_fp8_quant.
124
+ default,
125
+ kwargs=kwargs)
126
+
127
+ for user in list(node.users):
128
+ if user.op == 'call_function' and user.target == operator.getitem: # noqa
129
+ # Remove the getitem node
130
+ if user.args[1] == 1:
131
+ replace_node = result
132
+ elif user.args[1] == 2:
133
+ replace_node = residual
134
+ user.replace_all_uses_with(replace_node)
135
+ nodes_to_remove.append(user)
136
+ nodes_to_remove.append(node)
137
138
elif node.args[0] == torch.ops._C.rms_norm.default:
139
# manual replace for rms_norm
140
141
kwargs = node.kwargs
142
108
- input = kwargs['input']
109
- out = kwargs['out']
110
- weight = kwargs['weight']
111
- epsilon = kwargs['epsilon']
112
- # Create a new call to torch.ops._C.rotary_embedding.default
113
- # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
143
+ replace_node = kwargs['result']
144
+ # Create a new call to torch.ops._C.rms_norm.default
145
with graph.inserting_before(node):
146
# just insert the call to the custom op
147
# NOTE: don't run dead code elimination,
148
# otherwise this op will be removed
118
- graph.call_function(
119
- torch.ops._C.rms_norm.default,
120
- args=(out, input, weight, epsilon),
121
- )
149
+ graph.call_function(torch.ops._C.rms_norm.default,
150
+ kwargs=kwargs)
151
123
- replace_node = out
152
+ for user in list(node.users):
153
+ if user.op == 'call_function' and user.target == operator.getitem: # noqa
154
+ user.replace_all_uses_with(replace_node)
155
+ nodes_to_remove.append(user)
156
+ nodes_to_remove.append(node)
157
+
158
+ elif node.args[
159
+ 0] == torch.ops._C.rms_norm_static_fp8_quant.default: # noqa
160
+ # manual replace for rms_norm_static_fp8_quant
161
+
162
+ kwargs = node.kwargs
163
+
164
+ replace_node = kwargs['result']
165
+ # Create a new call to torch.ops._C.rms_norm_static_fp8_quant.default # noqa
166
+ with graph.inserting_before(node):
167
+ # just insert the call to the custom op
168
+ # NOTE: don't run dead code elimination,
169
+ # otherwise this op will be removed
170
+ graph.call_function(
171
+ torch.ops._C.rms_norm_static_fp8_quant.default,
172
+ kwargs=kwargs)
173
174
for user in list(node.users):
175
if user.op == 'call_function' and user.target == operator.getitem: # noqa
@@ -136,7 +185,7 @@ def fix_functionalization(graph: fx.Graph):
185
input = kwargs['input']
186
out = kwargs['out']
187
139
- # Create a new call to torch.ops._C.rotary_embedding.default
188
+ # Create a new call to torch.ops._C.silu_and_mul.default
189
# cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
190
with graph.inserting_before(node):
191
# just insert the call to the custom op
@@ -319,6 +368,13 @@ class VllmBackend:
368
369
The major work of this backend is to split the graph into
370
piecewise graphs, and pass them to the piecewise backend.
371
+
372
+ This backend also handles custom passes and adds them to Inductor config.
373
+ The order of the post-grad post-passes is:
374
+ 1. post_grad_passes (constructor parameter)
375
+ 2. config["post_grad_custom_post_pass"]
376
+ 3. fix_functionalization
377
+ This way, all passes operate on a functionalized graph.
378
"""
379
380
compilation_configs: CompilationConfig
@@ -330,8 +386,10 @@ class VllmBackend:
386
split_gm: fx.GraphModule
387
piecewise_graphs: List[SplitItem]
388
returned_callable: Callable
389
+ # Inductor passes to run on the graph pre-defunctionalization
390
+ post_grad_passes: Sequence[Callable]
391
334
- def __init__(self, ):
392
+ def __init__(self, post_grad_passes: Sequence[Callable] = ()):
393
global global_graph_pool
394
if global_graph_pool is None:
395
global_graph_pool = torch.cuda.graph_pool_handle()
@@ -340,10 +398,30 @@ class VllmBackend:
398
# streams, it might not be safe to share a global pool.
399
# only investigate this when we use multiple streams
400
self.graph_pool = global_graph_pool
401
+ self.post_grad_passes = post_grad_passes
402
403
# `torch.compile` is JIT compiled, so we don't need to
404
# do anything here
405
406
+ def add_passes_to_config(self):
407
+ config = self.compilation_configs
408
+ passes = list(self.post_grad_passes)
409
+
410
+ passes = passes + [RedundantReshapesPass(config)]
411
+
412
+ if config.enable_fusion:
413
+ passes = passes + [FusionPass.instance(config)]
414
+
415
+ inductor_config = config.inductor_compile_config
416
+ if "post_grad_custom_post_pass" in inductor_config:
417
+ passes = passes + [inductor_config["post_grad_custom_post_pass"]]
418
+
419
+ # add the fix_functionalization pass last, so that all other
420
+ # passes operate on a functionalized graph
421
+ passes = passes + [fix_functionalization]
422
+ combined_pass = combine_fx_passes(passes)
423
+ inductor_config["post_grad_custom_post_pass"] = combined_pass
424
+
425
def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
426
427
compilation_counter.num_graphs_seen += 1
@@ -357,6 +435,7 @@ class VllmBackend:
435
# we get the sizes to capture for cudagraph
436
# from compilation context
437
self.compilation_configs = CompilationConfig.select_and_init_config()
438
+ self.add_passes_to_config()
439
440
self.split_gm, self.piecewise_graphs = split_graph(
441
graph, self.compilation_configs.non_cudagraph_ops)
vllm/compilation/config.py
+12
-13
@@ -1,4 +1,5 @@
1
import copy
2
+from pathlib import Path
3
from typing import Any, Dict, List, Optional
4
5
from pydantic import BaseModel, Field, PrivateAttr
@@ -50,6 +51,12 @@ class CompilationConfig(BaseModel):
51
name because the config uses json format. If we pass the config
52
from Python, functions can also be passed directly via Python object
53
constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
54
+ - Custom inductor passes:
55
+ - dump_graph_stages: list of stages for which we want to dump the graph.
56
+ Each pass defines its own stages (before, after, maybe in-between).
57
+ - dump_graph_dir: directory to dump the graph. Default is .
58
+ - enable_fusion: whether to enable the custom fusion pass.
59
+ TODO better pass enabling system.
60
61
Why we have different sizes for cudagraph and inductor:
62
- cudagraph: a cudagraph captured for a specific size can only be used
@@ -72,6 +79,10 @@ class CompilationConfig(BaseModel):
79
cudagraph_num_of_warmups: int = 0
80
cudagraph_capture_sizes: Optional[List[int]] = None
81
82
+ dump_graph_stages: List[str] = Field(default_factory=list)
83
+ dump_graph_dir: Path = Field(default=Path("."))
84
+ enable_fusion: bool = True
85
+
86
# not configurable, computed after init
87
compile_sizes: List[int] = PrivateAttr
88
capture_sizes: List[int] = PrivateAttr
@@ -81,7 +92,7 @@ class CompilationConfig(BaseModel):
92
if not isinstance(v, str):
93
assert callable(v), (
94
f"pass {k} should be a function or a qualified name")
84
- self.inductor_passes[k] = v
95
+ self.inductor_compile_config[k] = v
96
continue
97
98
# resolve function from qualified name
@@ -91,18 +102,6 @@ class CompilationConfig(BaseModel):
102
func = __import__(module).__dict__[func_name]
103
self.inductor_compile_config[k] = func
104
94
- from vllm.compilation.backends import fix_functionalization
95
- from vllm.utils import combine_fx_passes
96
- if "post_grad_custom_post_pass" in self.inductor_compile_config:
97
- self.inductor_compile_config[
98
- "post_grad_custom_post_pass"] = combine_fx_passes(
99
- fix_functionalization,
100
- self.inductor_compile_config["post_grad_custom_post_pass"],
101
- )
102
- else:
103
- self.inductor_compile_config[
104
- "post_grad_custom_post_pass"] = fix_functionalization
105
-
105
def init_during_runtime(self):
106
"""To complete the initialization of config,
107
we need to know the compile context, which is only available
vllm/compilation/fusion.py
new
+291
@@ -0,0 +1,291 @@
1
+import operator
2
+from typing import Iterable, List, Optional
3
+
4
+import torch
5
+from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
+from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7
+ fwd_only, register_replacement)
8
+
9
+from vllm.compilation.config import CompilationConfig
10
+from vllm.compilation.inductor_pass import InductorPass
11
+from vllm.logger import init_logger
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+def rms_pattern_static(result: torch.Tensor, result_rms: torch.Tensor,
17
+ input: torch.Tensor, weight: torch.Tensor,
18
+ scale: torch.Tensor):
19
+ at1 = auto_functionalized(torch.ops._C.rms_norm.default,
20
+ result=result_rms,
21
+ input=input,
22
+ weight=weight,
23
+ epsilon=1e-5)
24
+ at2 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
25
+ result=result,
26
+ input=at1[1],
27
+ scale=scale)
28
+
29
+ # result
30
+ return at2[1]
31
+
32
+
33
+def rms_replacement_static(result: torch.Tensor, result_rms: torch.Tensor,
34
+ input: torch.Tensor, weight: torch.Tensor,
35
+ scale: torch.Tensor):
36
+ at = auto_functionalized(torch.ops._C.rms_norm_static_fp8_quant.default,
37
+ result=result,
38
+ input=input,
39
+ weight=weight,
40
+ scale=scale,
41
+ epsilon=1e-5)
42
+
43
+ # result
44
+ return at[1]
45
+
46
+
47
+def rms_pattern_residual_static(result: torch.Tensor, input: torch.Tensor,
48
+ residual: torch.Tensor, weight: torch.Tensor,
49
+ scale: torch.Tensor):
50
+ at = auto_functionalized(torch.ops._C.fused_add_rms_norm.default,
51
+ input=input,
52
+ residual=residual,
53
+ weight=weight,
54
+ epsilon=1e-5)
55
+ at1 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
56
+ result=result,
57
+ input=at[1],
58
+ scale=scale)
59
+
60
+ # result, residual
61
+ return at1[1], at[2]
62
+
63
+
64
+def rms_replacement_residual_static(result: torch.Tensor, input: torch.Tensor,
65
+ residual: torch.Tensor,
66
+ weight: torch.Tensor, scale: torch.Tensor):
67
+ at = auto_functionalized(
68
+ torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
69
+ result=result,
70
+ input=input,
71
+ residual=residual,
72
+ weight=weight,
73
+ scale=scale,
74
+ epsilon=1e-5)
75
+ # result, residual
76
+ return at[1], at[2]
77
+
78
+
79
+def empty_bf16(*args, **kwargs):
80
+ return torch.empty(*args, **kwargs, dtype=torch.bfloat16, device="cuda")
81
+
82
+
83
+def empty_fp8(*args, **kwargs):
84
+ fp8 = torch.float8_e4m3fn
85
+ return torch.empty(*args, **kwargs, dtype=fp8, device="cuda")
86
+
87
+
88
+def empty_fp32(*args, **kwargs):
89
+ return torch.empty(*args, **kwargs, dtype=torch.float32, device="cuda")
90
+
91
+
92
+# Utilities for post-processing multi-output matches
93
+def is_func(node: torch.fx.Node, target) -> bool:
94
+ return node.op == "call_function" and node.target == target
95
+
96
+
97
+# Returns the first auto_functionalized node with the given op (if it exists)
98
+def find_auto_fn_maybe(nodes: Iterable[torch.fx.Node],
99
+ op) -> Optional[torch.fx.Node]:
100
+ for node in nodes:
101
+ if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
102
+ return node
103
+ return None
104
+
105
+
106
+# Returns the first auto_functionalized node with the given op
107
+def find_auto_fn(nodes: Iterable[torch.fx.Node], op) -> torch.fx.Node:
108
+ node = find_auto_fn_maybe(nodes, op)
109
+ assert node is not None, f"Could not find {op} in nodes {nodes}"
110
+ return node
111
+
112
+
113
+# Returns the getitem node that extracts the idx-th element from node
114
+# (if it exists)
115
+def find_getitem_maybe(node: torch.fx.Node,
116
+ idx: int) -> Optional[torch.fx.Node]:
117
+ for user in node.users:
118
+ if is_func(user, operator.getitem) and user.args[1] == idx:
119
+ return user
120
+ return None
121
+
122
+
123
+# Returns the getitem node that extracts the idx-th element from node
124
+def find_getitem(node: torch.fx.Node, idx: int) -> torch.fx.Node:
125
+ ret = find_getitem_maybe(node, idx)
126
+ assert ret is not None, f"Could not find getitem {idx} in node {node}"
127
+ return ret
128
+
129
+
130
+class FusionPass(InductorPass):
131
+ """
132
+ This pass fuses a pre-defined set of custom ops into fused ops.
133
+ It uses the torch pattern matcher to find the patterns and replace them.
134
+ It also manually processes multi-output matches, as those are broken in
135
+ the torch pattern matcher.
136
+
137
+ Because patterns can only be registered once, the pass is a singleton.
138
+ This will be addressed in a future version of PyTorch:
139
+ https://github.com/pytorch/pytorch/pull/139321#issuecomment-2452354980
140
+ """
141
+
142
+ _instance: 'Optional[FusionPass]' = None
143
+
144
+ @classmethod
145
+ def instance(cls, config: CompilationConfig):
146
+ """
147
+ Get the singleton instance of the FusionPass.
148
+ If the instance exists, the config is updated but
149
+ initialization is not repeated.
150
+ """
151
+ if cls._instance is None:
152
+ cls._instance = FusionPass(config)
153
+ else:
154
+ cls._instance.config = config
155
+ return cls._instance
156
+
157
+ def __init__(self, config: CompilationConfig):
158
+ assert self.__class__._instance is None, \
159
+ "FusionPass singleton instance already exists"
160
+ super().__init__(config)
161
+
162
+ self.matches: List[Match] = []
163
+ self.patterns: PatternMatcherPass = PatternMatcherPass(
164
+ pass_name="fusion_pass")
165
+
166
+ # Fuse rms_norm + static_scaled_fp8_quant into
167
+ # rms_norm_static_fp8_quant
168
+ inputs = [
169
+ empty_fp8(5, 4),
170
+ empty_bf16(5, 4),
171
+ empty_bf16(5, 4),
172
+ empty_bf16(1, 5),
173
+ empty_fp32(1, 1)
174
+ ]
175
+ register_replacement(rms_pattern_static, rms_replacement_static,
176
+ inputs, fwd_only, self.patterns)
177
+
178
+ # Fuse fused_add_rms_norm + static_scaled_fp8_quant into
179
+ # fused_add_rms_norm_static_fp8_quant
180
+ # Because pattern has 2 outputs, we need to manually process the match
181
+ # (see process_matches)
182
+ inputs = [
183
+ empty_fp8(5, 4),
184
+ empty_bf16(5, 4),
185
+ empty_bf16(5, 4),
186
+ empty_bf16(1, 5),
187
+ empty_fp32(1, 1)
188
+ ]
189
+ register_replacement(rms_pattern_residual_static,
190
+ rms_replacement_residual_static,
191
+ inputs,
192
+ fwd_only,
193
+ self.patterns,
194
+ extra_check=lambda m: self.record_match(m))
195
+
196
+ def record_match(self, match: Match) -> bool:
197
+ # Hijack the extra_check to record the match and
198
+ # save it for post-processing.
199
+ self.matches.append(match)
200
+
201
+ # Return False to prevent automatic replacement.
202
+ return False
203
+
204
+ def process_matches(self, graph: torch.fx.Graph):
205
+ """
206
+ Manually process multi-output matches and replace them with fused nodes.
207
+ This is necessary because the automatic replacement for multi-output
208
+ matches is broken: https://github.com/pytorch/pytorch/issues/137280
209
+ """
210
+ for match in self.matches:
211
+ # To avoid use-before-definition errors, insert replacement nodes
212
+ # after the last node in the match.
213
+ # match.nodes is not guaranteed to be sorted.
214
+ # Find the last node in the match.
215
+ for last_node_in_match in reversed(graph.nodes):
216
+ if last_node_in_match in match.nodes:
217
+ break
218
+ else:
219
+ raise ValueError("No nodes in graph")
220
+
221
+ # Insert a new auto_functionalized node for the fused operation,
222
+ # as well as getitem nodes to extract the result and residual.
223
+ # The auto_functionalized node returns a tuple of
224
+ # (None, result, residual) - None is the function return value.
225
+ # The resulting graph looks like this:
226
+ # at = auto_functionalized(torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, ...) # noqa
227
+ # result_node_new = at[1]
228
+ # residual_node_new = at[2]
229
+ with graph.inserting_after(last_node_in_match):
230
+ kwargs = match.kwargs
231
+ kwargs["epsilon"] = 1e-5 # Currently hard-coded in RMSNorm
232
+
233
+ fused_node = graph.call_function(
234
+ auto_functionalized,
235
+ (torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
236
+ ),
237
+ kwargs=kwargs)
238
+
239
+ graph.inserting_after(fused_node)
240
+ result_node_new = graph.call_function(operator.getitem,
241
+ (fused_node, 1))
242
+ residual_node_new = graph.call_function(
243
+ operator.getitem, (fused_node, 2))
244
+
245
+ # Last part of replacement is rebinding the users of nodes in the
246
+ # match to use the new nodes.
247
+
248
+ # Find the nodes in the match that we need to rebind
249
+ rms_node = find_auto_fn(match.nodes,
250
+ torch.ops._C.fused_add_rms_norm.default)
251
+ quant_node = find_auto_fn(
252
+ match.nodes, torch.ops._C.static_scaled_fp8_quant.default)
253
+
254
+ assert len(rms_node.users) == 2
255
+ assert len(quant_node.users) == 1
256
+
257
+ # meta["val"] is used by de-functionalization and has to contain the
258
+ # value of the node (tuple of tensors) that would be returned by the
259
+ # functionalized node during tracing.
260
+
261
+ rms_tup = rms_node.meta["val"]
262
+ quant_tup = quant_node.meta["val"]
263
+
264
+ # The result of fused_node must be a tuple with the first element
265
+ # None (the function return value) and the remaining elements
266
+ # representing the mutated inputs.
267
+ fused_tup = (None, quant_tup[1], rms_tup[1], rms_tup[2])
268
+ fused_node.meta["val"] = fused_tup
269
+
270
+ # Find the getitem nodes and replace their uses with the new nodes.
271
+ # The old nodes will be removed by DCE at the end of the pass.
272
+ find_getitem(rms_node, 2).replace_all_uses_with(residual_node_new)
273
+ find_getitem(quant_node, 1).replace_all_uses_with(result_node_new)
274
+
275
+ # Finally, remove matched nodes
276
+ graph.eliminate_dead_code()
277
+ assert all(node not in graph.nodes for match in self.matches
278
+ for node in match.nodes)
279
+
280
+ def __call__(self, graph: torch.fx.Graph):
281
+ self.dump_graph(graph, "before_fusion")
282
+
283
+ count = self.patterns.apply(graph)
284
+ logger.info("Replaced %s patterns", count)
285
+ self.dump_graph(graph, "after_pattern_match")
286
+
287
+ # Manually process multi-output matches (and run DCE)
288
+ self.process_matches(graph)
289
+ logger.info("Post-processed %s matches", len(self.matches))
290
+ self.dump_graph(graph, "after_fusion")
291
+ self.matches.clear()
vllm/compilation/inductor_pass.py
new
+38
@@ -0,0 +1,38 @@
1
+from abc import ABC, abstractmethod
2
+
3
+import torch
4
+
5
+from vllm.compilation.config import CompilationConfig
6
+# yapf: disable
7
+from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8
+from vllm.distributed import (
9
+ get_tensor_model_parallel_world_size as get_tp_world_size)
10
+from vllm.distributed import model_parallel_is_initialized as p_is_init
11
+# yapf: enable
12
+from vllm.logger import init_logger
13
+
14
+logger = init_logger(__name__)
15
+
16
+
17
+class InductorPass(ABC):
18
+
19
+ @abstractmethod
20
+ def __call__(self, graph: torch.fx.Graph):
21
+ raise NotImplementedError
22
+
23
+ def __init__(self, config: CompilationConfig):
24
+ self.config = config
25
+
26
+ def dump_graph(self, graph: torch.fx.Graph, stage: str):
27
+ if stage in self.config.dump_graph_stages:
28
+ # Make sure filename includes rank in the distributed setting
29
+ parallel = p_is_init() and get_tp_world_size() > 1
30
+ rank = f"-{get_tp_rank()}" if parallel else ""
31
+ filepath = self.config.dump_graph_dir / f"{stage}{rank}.py"
32
+
33
+ logger.info("Printing graph to %s", filepath)
34
+ with open(filepath, "w") as f:
35
+ src = graph.python_code(root_module="self", verbose=True).src
36
+ # Add imports so it's not full of errors
37
+ print("import torch; from torch import device", file=f)
38
+ print(src, file=f)
vllm/compilation/reshapes.py
new
+85
@@ -0,0 +1,85 @@
1
+from typing import Union
2
+
3
+import torch.fx
4
+from torch import SymInt
5
+
6
+from vllm.compilation.fusion import is_func
7
+from vllm.compilation.inductor_pass import InductorPass
8
+from vllm.logger import init_logger
9
+
10
+logger = init_logger(__name__)
11
+
12
+
13
+class RedundantReshapesPass(InductorPass):
14
+ """
15
+ This is an inductor pass that removes redundant reshape operations.
16
+ It is required for RMSNorm-quant fusion to work properly.
17
+ That's because apply_fp8_linear adds a reshape, which is redundant
18
+ in the 2D-case.
19
+
20
+ Example graph:
21
+
22
+ getitem_1: "f16[s0, 4096]" = ...
23
+ view_1: "f16[s0, 4096]" = torch.reshape(getitem_1, [-1, 4096])
24
+ at = auto_functionalized(static_scaled_fp8_quant, input = view_1, ...)
25
+ out: "f8e4m3fn[s0, 4096]" = at[1]
26
+
27
+ Can be replaced with:
28
+ getitem_1: "f16[s0, 4096]" = ...
29
+ at = auto_functionalized(static_scaled_fp8_quant, input = getitem_1, ...)
30
+ out: "f8e4m3fn[s0, 4096]" = at[1]
31
+ """
32
+
33
+ def __call__(self, graph: torch.fx.Graph):
34
+ self.dump_graph(graph, "before_reshapes")
35
+ count = 0
36
+ # Remove no-op reshapes/views:
37
+ for node in graph.nodes:
38
+ if is_func(node, torch.ops.aten.reshape.default):
39
+ input, shape = node.args[:2]
40
+ input_shape = input.meta["val"].shape
41
+ if len(shape) != len(input_shape):
42
+ # Reshape changing rank, skip
43
+ continue
44
+
45
+ if shape.count(-1) > 1:
46
+ # Invalid reshape args, skip
47
+ continue
48
+
49
+ if all(
50
+ self.dims_equivalent(s, i_s)
51
+ for s, i_s in zip(shape, input_shape)):
52
+ node.replace_all_uses_with(input)
53
+ graph.erase_node(node)
54
+ count += 1
55
+
56
+ logger.info("Removed %s no-op reshapes", count)
57
+
58
+ self.dump_graph(graph, "after_reshapes")
59
+
60
+ def dims_equivalent(self, dim: Union[int, torch.fx.Node],
61
+ i_dim: Union[int, SymInt]) -> bool:
62
+ """
63
+ This function checks if two dimensions are equivalent.
64
+ :param dim: The dimension arg to reshape
65
+ :param i_dim: The corresponding dimension in the input tensor
66
+ :return: Are the dimensions equivalent?
67
+
68
+ There are three cases in which the dimensions are equivalent:
69
+ 1. The dimensions are equal (both integers)
70
+ 2. The reshape dimension is -1 (i.e. inferred)
71
+ 3. The dimensions both correspond to the same SymInt
72
+
73
+ While case 2 does not guarantee the dimensions are equal,
74
+ they are equal if all other dimensions are equal.
75
+
76
+ In case 3, the reshape dimension is a torch.fx.Node,
77
+ and its value is a SymInt. That value is equal to the
78
+ input dimension.
79
+
80
+ """
81
+ # Case 1 and 2
82
+ if dim == i_dim or dim == -1:
83
+ return True
84
+ # Case 3
85
+ return isinstance(dim, torch.fx.Node) and dim.meta["val"] == i_dim
vllm/envs.py
+2
@@ -68,6 +68,7 @@ if TYPE_CHECKING:
68
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
69
VLLM_SKIP_P2P_CHECK: bool = False
70
VLLM_TORCH_COMPILE_LEVEL: int = 0
71
+ VLLM_TORCH_COMPILE_CONFIG: Optional[str] = None
72
VLLM_CUSTOM_OPS: List[str] = []
73
VLLM_DISABLED_KERNELS: List[str] = []
74
VLLM_USE_V1: bool = False
@@ -226,6 +227,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
227
# and disabled when running with Inductor (compile_level >= Inductor).
228
"VLLM_CUSTOM_OPS":
229
lambda: os.environ.get("VLLM_CUSTOM_OPS", "").replace(" ", "").split(","),
230
+
231
# local rank of the process in the distributed setting, used to determine
232
# the GPU device id
233
"LOCAL_RANK":