@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4f93dfe952
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[torch.compile] Fuse RMSNorm with quant (#9138)

Signed-off-by: luka <luka@neuralmagic.com> Co-authored-by: youkaichao <youkaichao@126.com>

Luka Govedič committed Nov 8, 2024 at 16:20 UTC 4f93dfe952522c3f784be6542d69be2a172b8496
17 files changed +1335 -368
CMakeLists.txt
+1
@@ -191,6 +191,7 @@ set(VLLM_EXT_SRC
191 "csrc/pos_encoding_kernels.cu"
192 "csrc/activation_kernels.cu"
193 "csrc/layernorm_kernels.cu"
194 + "csrc/layernorm_quant_kernels.cu"
195 "csrc/quantization/gptq/q_gemm.cu"
196 "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
197 "csrc/quantization/fp8/common.cu"
csrc/layernorm_kernels.cu
+4 -161
@@ -1,21 +1,13 @@
1 -#include <torch/all.h>
2 -#include <ATen/cuda/CUDAContext.h>
1 +#include "type_convert.cuh"
2 +#include "dispatch_utils.h"
3 +
4 +#include <torch/cuda.h>
5 #include <c10/cuda/CUDAGuard.h>
6
5 -#include "dispatch_utils.h"
7 #ifndef USE_ROCM
7 - #include <cuda_bf16.h>
8 - #include <cuda_fp16.h>
9 - #include <cub/util_type.cuh>
8 #include <cub/cub.cuh>
9 #else
12 - #include <hip/hip_bf16.h>
13 - #include <hip/hip_fp16.h>
14 - #include <hipcub/util_type.hpp>
10 #include <hipcub/hipcub.hpp>
16 -
17 -using __nv_bfloat16 = __hip_bfloat16;
18 -using __nv_bfloat162 = __hip_bfloat162;
11 #endif
12
13 namespace vllm {
@@ -51,155 +43,6 @@ __global__ void rms_norm_kernel(
43 }
44 }
45
54 -/* Converter structs for the conversion from torch types to HIP/CUDA types,
55 - and the associated type conversions within HIP/CUDA. These helpers need
56 - to be implemented for now because the relevant type conversion
57 - operators/constructors are not consistently implemented by HIP/CUDA, so
58 - a generic conversion via type casts cannot be implemented.
59 -
60 - Each struct should have the member static constexpr bool `exists`:
61 - If false, the optimized kernel is not used for the corresponding torch type.
62 - If true, the struct should be fully defined as shown in the examples below.
63 - */
64 -template <typename torch_type>
65 -struct _typeConvert {
66 - static constexpr bool exists = false;
67 -};
68 -
69 -#if defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
70 -// CUDA < 12.0 runs into issues with packed type conversion
71 -template <>
72 -struct _typeConvert<c10::Half> {
73 - static constexpr bool exists = true;
74 - using hip_type = __half;
75 - using packed_hip_type = __half2;
76 -
77 - __device__ static inline float convert(hip_type x) { return __half2float(x); }
78 - __device__ static inline float2 convert(packed_hip_type x) {
79 - return __half22float2(x);
80 - }
81 - __device__ static inline hip_type convert(float x) {
82 - return __float2half_rn(x);
83 - }
84 - __device__ static inline packed_hip_type convert(float2 x) {
85 - return __float22half2_rn(x);
86 - }
87 -};
88 -
89 - #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
90 -// CUDA_ARCH < 800 does not have BF16 support
91 -// TODO: Add in ROCm support once public headers handle bf16 maturely
92 -template <>
93 -struct _typeConvert<c10::BFloat16> {
94 - static constexpr bool exists = true;
95 - using hip_type = __nv_bfloat16;
96 - using packed_hip_type = __nv_bfloat162;
97 -
98 - __device__ static inline float convert(hip_type x) {
99 - return __bfloat162float(x);
100 - }
101 - __device__ static inline float2 convert(packed_hip_type x) {
102 - return __bfloat1622float2(x);
103 - }
104 - __device__ static inline hip_type convert(float x) {
105 - return __float2bfloat16(x);
106 - }
107 - __device__ static inline packed_hip_type convert(float2 x) {
108 - return __float22bfloat162_rn(x);
109 - }
110 -};
111 - #endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
112 -#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >=
113 - // 12000))
114 -
115 -/* Vector POD struct to generate vectorized and packed FP16/BF16 ops
116 - for appropriate specializations of fused_add_rms_norm_kernel.
117 - Only functions that are necessary in that kernel are implemented.
118 - Alignment to 16 bytes is required to use 128-bit global memory ops.
119 - */
120 -template <typename scalar_t, int width>
121 -struct alignas(16) _f16Vec {
122 - /* Not theoretically necessary that width is a power of 2 but should
123 - almost always be the case for optimization purposes */
124 - static_assert(width > 0 && (width & (width - 1)) == 0,
125 - "Width is not a positive power of 2!");
126 - using Converter = _typeConvert<scalar_t>;
127 - using T1 = typename Converter::hip_type;
128 - using T2 = typename Converter::packed_hip_type;
129 - T1 data[width];
130 -
131 - __device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
132 - if constexpr (width % 2 == 0) {
133 -#pragma unroll
134 - for (int i = 0; i < width; i += 2) {
135 - T2 temp{data[i], data[i + 1]};
136 - temp += T2{other.data[i], other.data[i + 1]};
137 - data[i] = temp.x;
138 - data[i + 1] = temp.y;
139 - }
140 - } else {
141 -#pragma unroll
142 - for (int i = 0; i < width; ++i) data[i] += other.data[i];
143 - }
144 - return *this;
145 - }
146 -
147 - __device__ _f16Vec& operator*=(const _f16Vec<scalar_t, width>& other) {
148 - if constexpr (width % 2 == 0) {
149 -#pragma unroll
150 - for (int i = 0; i < width; i += 2) {
151 - T2 temp{data[i], data[i + 1]};
152 - temp *= T2{other.data[i], other.data[i + 1]};
153 - data[i] = temp.x;
154 - data[i + 1] = temp.y;
155 - }
156 - } else {
157 -#pragma unroll
158 - for (int i = 0; i < width; ++i) data[i] *= other.data[i];
159 - }
160 - return *this;
161 - }
162 -
163 - __device__ _f16Vec& operator*=(const float scale) {
164 - if constexpr (width % 2 == 0) {
165 -#pragma unroll
166 - for (int i = 0; i < width; i += 2) {
167 - float2 temp_f = Converter::convert(T2{data[i], data[i + 1]});
168 - temp_f.x *= scale;
169 - temp_f.y *= scale;
170 - T2 temp = Converter::convert(temp_f);
171 - data[i] = temp.x;
172 - data[i + 1] = temp.y;
173 - }
174 - } else {
175 -#pragma unroll
176 - for (int i = 0; i < width; ++i) {
177 - float temp = Converter::convert(data[i]) * scale;
178 - data[i] = Converter::convert(temp);
179 - }
180 - }
181 - return *this;
182 - }
183 -
184 - __device__ float sum_squares() const {
185 - float result = 0.0f;
186 - if constexpr (width % 2 == 0) {
187 -#pragma unroll
188 - for (int i = 0; i < width; i += 2) {
189 - float2 z = Converter::convert(T2{data[i], data[i + 1]});
190 - result += z.x * z.x + z.y * z.y;
191 - }
192 - } else {
193 -#pragma unroll
194 - for (int i = 0; i < width; ++i) {
195 - float x = Converter::convert(data[i]);
196 - result += x * x;
197 - }
198 - }
199 - return result;
200 - }
201 -};
202 -
46 /* Function specialization in the case of FP16/BF16 tensors.
47 Additional optimizations we can make in this case are
48 packed and vectorized operations, which help with the
csrc/layernorm_quant_kernels.cu new
+234
@@ -0,0 +1,234 @@
1 +/*
2 + * This file contains the CUDA kernels for the fused quantized layernorm.
3 + * The kernels correspond to the kernels in layernorm_kernels.cu, except they
4 + * also produce quantized output directly.
5 + * Currently, only static fp8 quantization is supported.
6 + */
7 +
8 +#include "type_convert.cuh"
9 +#include "quantization/fp8/common.cuh"
10 +#include "dispatch_utils.h"
11 +
12 +#include <torch/cuda.h>
13 +#include <c10/cuda/CUDAGuard.h>
14 +
15 +#ifndef USE_ROCM
16 + #include <cub/cub.cuh>
17 +#else
18 + #include <hipcub/hipcub.hpp>
19 +#endif
20 +
21 +namespace vllm {
22 +
23 +// TODO(woosuk): Further optimize this kernel.
24 +template <typename scalar_t>
25 +__global__ void rms_norm_static_fp8_quant_kernel(
26 + FP8_TYPE* __restrict__ out, // [..., hidden_size]
27 + const scalar_t* __restrict__ input, // [..., hidden_size]
28 + const scalar_t* __restrict__ weight, // [hidden_size]
29 + const float* __restrict__ scale, // [1]
30 + const float epsilon, const int num_tokens, const int hidden_size) {
31 + __shared__ float s_variance;
32 + float variance = 0.0f;
33 +
34 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
35 + const float x = (float)input[blockIdx.x * hidden_size + idx];
36 + variance += x * x;
37 + }
38 +
39 + using BlockReduce = cub::BlockReduce<float, 1024>;
40 + __shared__ typename BlockReduce::TempStorage reduceStore;
41 + variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
42 +
43 + if (threadIdx.x == 0) {
44 + s_variance = rsqrtf(variance / hidden_size + epsilon);
45 + }
46 + __syncthreads();
47 +
48 + // invert scale to avoid division
49 + float const scale_inv = 1.0f / *scale;
50 +
51 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
52 + float x = (float)input[blockIdx.x * hidden_size + idx];
53 + float const out_norm = ((scalar_t)(x * s_variance)) * weight[idx];
54 + out[blockIdx.x * hidden_size + idx] =
55 + scaled_fp8_conversion<true>(out_norm, scale_inv);
56 + }
57 +}
58 +
59 +/* Function specialization in the case of FP16/BF16 tensors.
60 + Additional optimizations we can make in this case are
61 + packed and vectorized operations, which help with the
62 + memory latency bottleneck. */
63 +template <typename scalar_t, int width>
64 +__global__ std::enable_if_t<(width > 0) && _typeConvert<scalar_t>::exists>
65 +fused_add_rms_norm_static_fp8_quant_kernel(
66 + FP8_TYPE* __restrict__ out, // [..., hidden_size]
67 + scalar_t* __restrict__ input, // [..., hidden_size]
68 + scalar_t* __restrict__ residual, // [..., hidden_size]
69 + const scalar_t* __restrict__ weight, // [hidden_size]
70 + const float* __restrict__ scale, // [1]
71 + const float epsilon, const int num_tokens, const int hidden_size) {
72 + // Sanity checks on our vector struct and type-punned pointer arithmetic
73 + static_assert(std::is_pod_v<_f16Vec<scalar_t, width>>);
74 + static_assert(sizeof(_f16Vec<scalar_t, width>) == sizeof(scalar_t) * width);
75 +
76 + const int vec_hidden_size = hidden_size / width;
77 + __shared__ float s_variance;
78 + float variance = 0.0f;
79 + /* These and the argument pointers are all declared `restrict` as they are
80 + not aliased in practice. Argument pointers should not be dereferenced
81 + in this kernel as that would be undefined behavior */
82 + auto* __restrict__ input_v =
83 + reinterpret_cast<_f16Vec<scalar_t, width>*>(input);
84 + auto* __restrict__ residual_v =
85 + reinterpret_cast<_f16Vec<scalar_t, width>*>(residual);
86 + auto* __restrict__ weight_v =
87 + reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
88 +
89 + for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
90 + int id = blockIdx.x * vec_hidden_size + idx;
91 + _f16Vec<scalar_t, width> temp = input_v[id];
92 + temp += residual_v[id];
93 + variance += temp.sum_squares();
94 + residual_v[id] = temp;
95 + }
96 +
97 + using BlockReduce = cub::BlockReduce<float, 1024>;
98 + __shared__ typename BlockReduce::TempStorage reduceStore;
99 + variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
100 +
101 + if (threadIdx.x == 0) {
102 + s_variance = rsqrtf(variance / hidden_size + epsilon);
103 + }
104 + __syncthreads();
105 +
106 + // invert scale to avoid division
107 + float const scale_inv = 1.0f / *scale;
108 +
109 + for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
110 + int id = blockIdx.x * vec_hidden_size + idx;
111 + _f16Vec<scalar_t, width> temp = residual_v[id];
112 + temp *= s_variance;
113 + temp *= weight_v[idx];
114 +#pragma unroll
115 + for (int i = 0; i < width; ++i) {
116 + out[id * width + i] =
117 + scaled_fp8_conversion<true>(float(temp.data[i]), scale_inv);
118 + }
119 + }
120 +}
121 +
122 +/* Generic fused_add_rms_norm_kernel
123 + The width field is not used here but necessary for other specializations.
124 + */
125 +template <typename scalar_t, int width>
126 +__global__ std::enable_if_t<(width == 0) || !_typeConvert<scalar_t>::exists>
127 +fused_add_rms_norm_static_fp8_quant_kernel(
128 + FP8_TYPE* __restrict__ out, // [..., hidden_size]
129 + scalar_t* __restrict__ input, // [..., hidden_size]
130 + scalar_t* __restrict__ residual, // [..., hidden_size]
131 + const scalar_t* __restrict__ weight, // [hidden_size]
132 + const float* __restrict__ scale, // [1]
133 + const float epsilon, const int num_tokens, const int hidden_size) {
134 + __shared__ float s_variance;
135 + float variance = 0.0f;
136 +
137 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
138 + scalar_t z = input[blockIdx.x * hidden_size + idx];
139 + z += residual[blockIdx.x * hidden_size + idx];
140 + float x = (float)z;
141 + variance += x * x;
142 + residual[blockIdx.x * hidden_size + idx] = z;
143 + }
144 +
145 + using BlockReduce = cub::BlockReduce<float, 1024>;
146 + __shared__ typename BlockReduce::TempStorage reduceStore;
147 + variance = BlockReduce(reduceStore).Reduce(variance, cub::Sum{}, blockDim.x);
148 +
149 + if (threadIdx.x == 0) {
150 + s_variance = rsqrtf(variance / hidden_size + epsilon);
151 + }
152 + __syncthreads();
153 +
154 + // invert scale to avoid division
155 + float const scale_inv = 1.0f / *scale;
156 +
157 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
158 + float x = (float)residual[blockIdx.x * hidden_size + idx];
159 + float const out_norm = ((scalar_t)(x * s_variance)) * weight[idx];
160 + out[blockIdx.x * hidden_size + idx] =
161 + scaled_fp8_conversion<true>(out_norm, scale_inv);
162 + }
163 +}
164 +
165 +} // namespace vllm
166 +
167 +void rms_norm_static_fp8_quant(torch::Tensor& out, // [..., hidden_size]
168 + torch::Tensor& input, // [..., hidden_size]
169 + torch::Tensor& weight, // [hidden_size]
170 + torch::Tensor& scale, // [1]
171 + double epsilon) {
172 + int hidden_size = input.size(-1);
173 + int num_tokens = input.numel() / hidden_size;
174 +
175 + dim3 grid(num_tokens);
176 + dim3 block(std::min(hidden_size, 1024));
177 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
178 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
179 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_kernel", [&] {
180 + vllm::rms_norm_static_fp8_quant_kernel<scalar_t>
181 + <<<grid, block, 0, stream>>>(
182 + out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(),
183 + weight.data_ptr<scalar_t>(), scale.data_ptr<float>(), epsilon,
184 + num_tokens, hidden_size);
185 + });
186 +}
187 +
188 +#define LAUNCH_FUSED_ADD_RMS_NORM(width) \
189 + VLLM_DISPATCH_FLOATING_TYPES( \
190 + input.scalar_type(), "fused_add_rms_norm_kernel", [&] { \
191 + vllm::fused_add_rms_norm_static_fp8_quant_kernel<scalar_t, width> \
192 + <<<grid, block, 0, stream>>>( \
193 + out.data_ptr<FP8_TYPE>(), input.data_ptr<scalar_t>(), \
194 + residual.data_ptr<scalar_t>(), weight.data_ptr<scalar_t>(), \
195 + scale.data_ptr<float>(), epsilon, num_tokens, hidden_size); \
196 + });
197 +
198 +void fused_add_rms_norm_static_fp8_quant(
199 + torch::Tensor& out, // [..., hidden_size],
200 + torch::Tensor& input, // [..., hidden_size]
201 + torch::Tensor& residual, // [..., hidden_size]
202 + torch::Tensor& weight, // [hidden_size]
203 + torch::Tensor& scale, // [1]
204 + double epsilon) {
205 + int hidden_size = input.size(-1);
206 + int num_tokens = input.numel() / hidden_size;
207 +
208 + dim3 grid(num_tokens);
209 + /* This kernel is memory-latency bound in many scenarios.
210 + When num_tokens is large, a smaller block size allows
211 + for increased block occupancy on CUs and better latency
212 + hiding on global mem ops. */
213 + const int max_block_size = (num_tokens < 256) ? 1024 : 256;
214 + dim3 block(std::min(hidden_size, max_block_size));
215 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
216 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
217 + /*If the tensor types are FP16/BF16, try to use the optimized kernel
218 + with packed + vectorized ops.
219 + Max optimization is achieved with a width-8 vector of FP16/BF16s
220 + since we can load at most 128 bits at once in a global memory op.
221 + However, this requires each tensor's data to be aligned to 16
222 + bytes.
223 + */
224 + auto inp_ptr = reinterpret_cast<std::uintptr_t>(input.data_ptr());
225 + auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
226 + auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight.data_ptr());
227 + bool ptrs_are_aligned =
228 + inp_ptr % 16 == 0 && res_ptr % 16 == 0 && wt_ptr % 16 == 0;
229 + if (ptrs_are_aligned && hidden_size % 8 == 0) {
230 + LAUNCH_FUSED_ADD_RMS_NORM(8);
231 + } else {
232 + LAUNCH_FUSED_ADD_RMS_NORM(0);
233 + }
234 +}
csrc/ops.h
+10
@@ -56,6 +56,16 @@ void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
56 void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
57 torch::Tensor& weight, double epsilon);
58
59 +void rms_norm_static_fp8_quant(torch::Tensor& out, torch::Tensor& input,
60 + torch::Tensor& weight, torch::Tensor& scale,
61 + double epsilon);
62 +
63 +void fused_add_rms_norm_static_fp8_quant(torch::Tensor& out,
64 + torch::Tensor& input,
65 + torch::Tensor& residual,
66 + torch::Tensor& weight,
67 + torch::Tensor& scale, double epsilon);
68 +
69 void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
70 torch::Tensor& key, int64_t head_size,
71 torch::Tensor& cos_sin_cache, bool is_neox);
csrc/quantization/fp8/common.cu
+3 -172
@@ -1,185 +1,16 @@
1 -#include <ATen/cuda/CUDAContext.h>
2 -#include <torch/all.h>
3 -#include <c10/cuda/CUDAGuard.h>
4 -
5 -#include <cmath>
6 -
7 -#include "cuda_compat.h"
1 +#include "common.cuh"
2 #include "dispatch_utils.h"
3
4 +#include <c10/cuda/CUDAGuard.h>
5 +
6 #ifndef USE_ROCM
11 - #include <cub/util_type.cuh>
7 #include <cub/cub.cuh>
8 #else
14 - #include <hipcub/util_type.hpp>
9 #include <hipcub/hipcub.hpp>
10 #endif
11
18 -#ifndef USE_ROCM
19 -using FP8_TYPE = c10::Float8_e4m3fn;
20 -C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX =
21 - std::numeric_limits<FP8_TYPE>::max();
22 -#else
23 - #include "amd/hip_float8.h"
24 -using FP8_TYPE = c10::Float8_e4m3fnuz;
25 -// Using the default max value from pytorch (240.0) will cause accuracy
26 -// issue when running dynamic quantization. Here use 224.0f for rocm.
27 -constexpr auto FP8_E4M3_MAX = 224.0f;
28 -#endif
29 -
12 namespace vllm {
13
32 -__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
33 - float old;
34 - old = (value >= 0)
35 - ? __int_as_float(atomicMax((int*)addr, __float_as_int(value)))
36 - : __uint_as_float(
37 - atomicMin((unsigned int*)addr, __float_as_uint(value)));
38 -
39 - return old;
40 -}
41 -
42 -template <bool is_scale_inverted>
43 -__device__ __forceinline__ FP8_TYPE scaled_fp8_conversion(float const val,
44 - float const scale) {
45 - float x = 0.0f;
46 - if constexpr (is_scale_inverted) {
47 - x = val * scale;
48 - } else {
49 - x = val / scale;
50 - }
51 -
52 - float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
53 -#ifndef USE_ROCM
54 - return static_cast<c10::Float8_e4m3fn>(r);
55 -#else
56 - // Use hardware cvt instruction for fp8 on rocm
57 - return c10::Float8_e4m3fnuz(hip_fp8(r).data,
58 - c10::Float8_e4m3fnuz::from_bits());
59 -#endif
60 -}
61 -
62 -// Compute the absolute maximum m of the input tensor and store
63 -// m / float8_e4m3::max() in *scale. Each thread block performs a
64 -// reduction tree and the memory in scale is atomically updated.
65 -// So to get the right answer, *scale needs to be initialized to
66 -// a value <= 0.0 and we need to wait for all thread blocks to
67 -// finish before consuming *scale.
68 -template <typename scalar_t>
69 -__global__ void segmented_max_reduction(float* __restrict__ scale,
70 - const scalar_t* __restrict__ input,
71 - int64_t num_elems) {
72 - __shared__ float cache[1024];
73 - int64_t i = blockDim.x * blockIdx.x + threadIdx.x;
74 -
75 - // First store maximum for all values processes by
76 - // the current thread in cache[threadIdx.x]
77 - scalar_t tmp = 0.0;
78 - while (i < num_elems) {
79 - float x = static_cast<float>(input[i]);
80 - tmp = max(tmp, fabs(x));
81 - i += blockDim.x * gridDim.x;
82 - }
83 - cache[threadIdx.x] = tmp;
84 -
85 - __syncthreads();
86 -
87 - // Now perform parallel reduction within the thread block
88 - int ib = blockDim.x / 2;
89 - while (ib != 0) {
90 - if (threadIdx.x < ib && cache[threadIdx.x + ib] > cache[threadIdx.x]) {
91 - cache[threadIdx.x] = cache[threadIdx.x + ib];
92 - }
93 - __syncthreads();
94 - ib /= 2;
95 - }
96 - // Finally, since cache[0] contains the maximum for this thread block,
97 - // atomically write the max to the target location
98 - if (threadIdx.x == 0) {
99 - atomicMaxFloat(scale, cache[0] / FP8_E4M3_MAX);
100 - }
101 -}
102 -
103 -template <typename scalar_t>
104 -struct __align__(8) vec4_t {
105 - scalar_t x;
106 - scalar_t y;
107 - scalar_t z;
108 - scalar_t w;
109 -};
110 -
111 -typedef struct __align__(4) {
112 - FP8_TYPE x;
113 - FP8_TYPE y;
114 - FP8_TYPE z;
115 - FP8_TYPE w;
116 -}
117 -float8x4_t;
118 -
119 -template <typename scalar_t>
120 -__device__ float thread_max_vec(scalar_t const* __restrict__ input,
121 - int64_t const num_elems, int const tid,
122 - int const step) {
123 - // Vectorized input/output to better utilize memory bandwidth.
124 - vec4_t<scalar_t> const* vectorized_in =
125 - reinterpret_cast<vec4_t<scalar_t> const*>(input);
126 -
127 - int64_t const num_vec_elems = num_elems >> 2;
128 - float absmax_val = 0.0f;
129 -
130 -#pragma unroll 4
131 - for (int64_t i = tid; i < num_vec_elems; i += step) {
132 - vec4_t<scalar_t> in_vec = vectorized_in[i];
133 - absmax_val = max(absmax_val, fabs(in_vec.x));
134 - absmax_val = max(absmax_val, fabs(in_vec.y));
135 - absmax_val = max(absmax_val, fabs(in_vec.z));
136 - absmax_val = max(absmax_val, fabs(in_vec.w));
137 - }
138 -
139 - // Handle the remaining elements if num_elems is not divisible by 4
140 - for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
141 - absmax_val = max(absmax_val, fabs(input[i]));
142 - }
143 -
144 - return absmax_val;
145 -}
146 -
147 -template <typename scalar_t, bool is_scale_inverted>
148 -__device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
149 - scalar_t const* __restrict__ input,
150 - float const scale,
151 - int64_t const num_elems,
152 - int const tid, int const step) {
153 - // Vectorized input/output to better utilize memory bandwidth.
154 - vec4_t<scalar_t> const* vectorized_in =
155 - reinterpret_cast<vec4_t<scalar_t> const*>(input);
156 - float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
157 -
158 - int64_t const num_vec_elems = num_elems >> 2;
159 -
160 -#pragma unroll 4
161 - for (int64_t i = tid; i < num_vec_elems; i += step) {
162 - vec4_t<scalar_t> in_vec = vectorized_in[i];
163 - float8x4_t out_vec;
164 -
165 - out_vec.x = scaled_fp8_conversion<is_scale_inverted>(
166 - static_cast<float>(in_vec.x), scale);
167 - out_vec.y = scaled_fp8_conversion<is_scale_inverted>(
168 - static_cast<float>(in_vec.y), scale);
169 - out_vec.z = scaled_fp8_conversion<is_scale_inverted>(
170 - static_cast<float>(in_vec.z), scale);
171 - out_vec.w = scaled_fp8_conversion<is_scale_inverted>(
172 - static_cast<float>(in_vec.w), scale);
173 - vectorized_out[i] = out_vec;
174 - }
175 -
176 - // Handle the remaining elements if num_elems is not divisible by 4
177 - for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
178 - out[i] = scaled_fp8_conversion<is_scale_inverted>(
179 - static_cast<float>(input[i]), scale);
180 - }
181 -}
182 -
14 template <typename scalar_t>
15 __global__ void scaled_fp8_quant_kernel(FP8_TYPE* __restrict__ out,
16 const scalar_t* __restrict__ input,
csrc/quantization/fp8/common.cuh new
+172
@@ -0,0 +1,172 @@
1 +#pragma once
2 +
3 +#include <cmath>
4 +
5 +#ifndef USE_ROCM
6 + #include <c10/util/Float8_e4m3fn.h>
7 +using FP8_TYPE = c10::Float8_e4m3fn;
8 +C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX =
9 + std::numeric_limits<FP8_TYPE>::max();
10 +#else
11 + #include <c10/util/Float8_e4m3fnuz.h>
12 + #include "amd/hip_float8.h"
13 +using FP8_TYPE = c10::Float8_e4m3fnuz;
14 +// Using the default max value from pytorch (240.0) will cause accuracy
15 +// issue when running dynamic quantization. Here use 224.0f for rocm.
16 +constexpr auto FP8_E4M3_MAX = 224.0f;
17 +#endif
18 +
19 +namespace vllm {
20 +
21 +__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) {
22 + float old;
23 + old = (value >= 0)
24 + ? __int_as_float(atomicMax((int*)addr, __float_as_int(value)))
25 + : __uint_as_float(
26 + atomicMin((unsigned int*)addr, __float_as_uint(value)));
27 +
28 + return old;
29 +}
30 +
31 +template <bool is_scale_inverted>
32 +__device__ __forceinline__ FP8_TYPE scaled_fp8_conversion(float const val,
33 + float const scale) {
34 + float x = 0.0f;
35 + if constexpr (is_scale_inverted) {
36 + x = val * scale;
37 + } else {
38 + x = val / scale;
39 + }
40 +
41 + float r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
42 +#ifndef USE_ROCM
43 + return static_cast<c10::Float8_e4m3fn>(r);
44 +#else
45 + // Use hardware cvt instruction for fp8 on rocm
46 + return c10::Float8_e4m3fnuz(hip_fp8(r).data,
47 + c10::Float8_e4m3fnuz::from_bits());
48 +#endif
49 +}
50 +
51 +// Compute the absolute maximum m of the input tensor and store
52 +// m / float8_e4m3::max() in *scale. Each thread block performs a
53 +// reduction tree and the memory in scale is atomically updated.
54 +// So to get the right answer, *scale needs to be initialized to
55 +// a value <= 0.0 and we need to wait for all thread blocks to
56 +// finish before consuming *scale.
57 +template <typename scalar_t>
58 +__global__ void segmented_max_reduction(float* __restrict__ scale,
59 + const scalar_t* __restrict__ input,
60 + int64_t num_elems) {
61 + __shared__ float cache[1024];
62 + int64_t i = blockDim.x * blockIdx.x + threadIdx.x;
63 +
64 + // First store maximum for all values processes by
65 + // the current thread in cache[threadIdx.x]
66 + scalar_t tmp = 0.0;
67 + while (i < num_elems) {
68 + float x = static_cast<float>(input[i]);
69 + tmp = max(tmp, fabs(x));
70 + i += blockDim.x * gridDim.x;
71 + }
72 + cache[threadIdx.x] = tmp;
73 +
74 + __syncthreads();
75 +
76 + // Now perform parallel reduction within the thread block
77 + int ib = blockDim.x / 2;
78 + while (ib != 0) {
79 + if (threadIdx.x < ib && cache[threadIdx.x + ib] > cache[threadIdx.x]) {
80 + cache[threadIdx.x] = cache[threadIdx.x + ib];
81 + }
82 + __syncthreads();
83 + ib /= 2;
84 + }
85 + // Finally, since cache[0] contains the maximum for this thread block,
86 + // atomically write the max to the target location
87 + if (threadIdx.x == 0) {
88 + atomicMaxFloat(scale, cache[0] / FP8_E4M3_MAX);
89 + }
90 +}
91 +
92 +template <typename scalar_t>
93 +struct __align__(8) vec4_t {
94 + scalar_t x;
95 + scalar_t y;
96 + scalar_t z;
97 + scalar_t w;
98 +};
99 +
100 +typedef struct __align__(4) {
101 + FP8_TYPE x;
102 + FP8_TYPE y;
103 + FP8_TYPE z;
104 + FP8_TYPE w;
105 +}
106 +float8x4_t;
107 +
108 +template <typename scalar_t>
109 +__device__ float thread_max_vec(scalar_t const* __restrict__ input,
110 + int64_t const num_elems, int const tid,
111 + int const step) {
112 + // Vectorized input/output to better utilize memory bandwidth.
113 + vec4_t<scalar_t> const* vectorized_in =
114 + reinterpret_cast<vec4_t<scalar_t> const*>(input);
115 +
116 + int64_t const num_vec_elems = num_elems >> 2;
117 + float absmax_val = 0.0f;
118 +
119 +#pragma unroll 4
120 + for (int64_t i = tid; i < num_vec_elems; i += step) {
121 + vec4_t<scalar_t> in_vec = vectorized_in[i];
122 + absmax_val = max(absmax_val, fabs(in_vec.x));
123 + absmax_val = max(absmax_val, fabs(in_vec.y));
124 + absmax_val = max(absmax_val, fabs(in_vec.z));
125 + absmax_val = max(absmax_val, fabs(in_vec.w));
126 + }
127 +
128 + // Handle the remaining elements if num_elems is not divisible by 4
129 + for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
130 + absmax_val = max(absmax_val, fabs(input[i]));
131 + }
132 +
133 + return absmax_val;
134 +}
135 +
136 +template <typename scalar_t, bool is_scale_inverted>
137 +__device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
138 + scalar_t const* __restrict__ input,
139 + float const scale,
140 + int64_t const num_elems,
141 + int const tid, int const step) {
142 + // Vectorized input/output to better utilize memory bandwidth.
143 + vec4_t<scalar_t> const* vectorized_in =
144 + reinterpret_cast<vec4_t<scalar_t> const*>(input);
145 + float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
146 +
147 + int64_t const num_vec_elems = num_elems >> 2;
148 +
149 +#pragma unroll 4
150 + for (int64_t i = tid; i < num_vec_elems; i += step) {
151 + vec4_t<scalar_t> in_vec = vectorized_in[i];
152 + float8x4_t out_vec;
153 +
154 + out_vec.x = scaled_fp8_conversion<is_scale_inverted>(
155 + static_cast<float>(in_vec.x), scale);
156 + out_vec.y = scaled_fp8_conversion<is_scale_inverted>(
157 + static_cast<float>(in_vec.y), scale);
158 + out_vec.z = scaled_fp8_conversion<is_scale_inverted>(
159 + static_cast<float>(in_vec.z), scale);
160 + out_vec.w = scaled_fp8_conversion<is_scale_inverted>(
161 + static_cast<float>(in_vec.w), scale);
162 + vectorized_out[i] = out_vec;
163 + }
164 +
165 + // Handle the remaining elements if num_elems is not divisible by 4
166 + for (int64_t i = num_vec_elems * 4 + tid; i < num_elems; i += step) {
167 + out[i] = scaled_fp8_conversion<is_scale_inverted>(
168 + static_cast<float>(input[i]), scale);
169 + }
170 +}
171 +
172 +} // namespace vllm
\ No newline at end of file
csrc/torch_bindings.cpp
+25 -6
@@ -101,7 +101,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
101 // Layernorm
102 // Apply Root Mean Square (RMS) Normalization to the input tensor.
103 ops.def(
104 - "rms_norm(Tensor! out, Tensor input, Tensor weight, float epsilon) -> "
104 + "rms_norm(Tensor! result, Tensor input, Tensor weight, float epsilon) -> "
105 "()");
106 ops.impl("rms_norm", torch::kCUDA, &rms_norm);
107
@@ -111,6 +111,23 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
111 "float epsilon) -> ()");
112 ops.impl("fused_add_rms_norm", torch::kCUDA, &fused_add_rms_norm);
113
114 + // Layernorm-quant
115 + // Apply Root Mean Square (RMS) Normalization to the input tensor.
116 + ops.def(
117 + "rms_norm_static_fp8_quant(Tensor! result, Tensor input, Tensor weight, "
118 + "Tensor scale, float epsilon) -> "
119 + "()");
120 + ops.impl("rms_norm_static_fp8_quant", torch::kCUDA,
121 + &rms_norm_static_fp8_quant);
122 +
123 + // In-place fused Add and RMS Normalization.
124 + ops.def(
125 + "fused_add_rms_norm_static_fp8_quant(Tensor! result, Tensor input, "
126 + "Tensor! residual, Tensor weight, "
127 + "Tensor scale, float epsilon) -> ()");
128 + ops.impl("fused_add_rms_norm_static_fp8_quant", torch::kCUDA,
129 + &fused_add_rms_norm_static_fp8_quant);
130 +
131 // Rotary embedding
132 // Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
133 ops.def(
@@ -322,18 +339,20 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
339
340 // Compute FP8 quantized tensor for given scaling factor.
341 ops.def(
325 - "static_scaled_fp8_quant(Tensor! out, Tensor input, Tensor scale) -> ()");
342 + "static_scaled_fp8_quant(Tensor! result, Tensor input, Tensor scale) -> "
343 + "()");
344 ops.impl("static_scaled_fp8_quant", torch::kCUDA, &static_scaled_fp8_quant);
345
346 // Compute dynamic-per-tensor FP8 quantized tensor and scaling factor.
347 ops.def(
330 - "dynamic_scaled_fp8_quant(Tensor! out, Tensor input, Tensor! scale) -> "
348 + "dynamic_scaled_fp8_quant(Tensor! result, Tensor input, Tensor! scale) "
349 + "-> "
350 "()");
351 ops.impl("dynamic_scaled_fp8_quant", torch::kCUDA, &dynamic_scaled_fp8_quant);
352
353 // Compute dynamic-per-token FP8 quantized tensor and scaling factor.
354 ops.def(
336 - "dynamic_per_token_scaled_fp8_quant(Tensor! out, Tensor input, "
355 + "dynamic_per_token_scaled_fp8_quant(Tensor! result, Tensor input, "
356 "Tensor! scale, Tensor? scale_ub) -> "
357 "()");
358 ops.impl("dynamic_per_token_scaled_fp8_quant", torch::kCUDA,
@@ -341,13 +360,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
360
361 // Compute int8 quantized tensor for given scaling factor.
362 ops.def(
344 - "static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
363 + "static_scaled_int8_quant(Tensor! result, Tensor input, Tensor scale,"
364 "Tensor? azp) -> ()");
365 ops.impl("static_scaled_int8_quant", torch::kCUDA, &static_scaled_int8_quant);
366
367 // Compute int8 quantized tensor and scaling factor
368 ops.def(
350 - "dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale, "
369 + "dynamic_scaled_int8_quant(Tensor! result, Tensor input, Tensor! scale, "
370 "Tensor!? azp) -> ()");
371 ops.impl("dynamic_scaled_int8_quant", torch::kCUDA,
372 &dynamic_scaled_int8_quant);
csrc/type_convert.cuh new
+165
@@ -0,0 +1,165 @@
1 +#pragma once
2 +
3 +#include <torch/all.h>
4 +
5 +#ifndef USE_ROCM
6 + #include <cuda_bf16.h>
7 + #include <cuda_fp16.h>
8 +#else
9 + #include <hip/hip_bf16.h>
10 + #include <hip/hip_fp16.h>
11 +
12 +using __nv_bfloat16 = __hip_bfloat16;
13 +using __nv_bfloat162 = __hip_bfloat162;
14 +#endif
15 +
16 +namespace vllm {
17 +/* Converter structs for the conversion from torch types to HIP/CUDA types,
18 + and the associated type conversions within HIP/CUDA. These helpers need
19 + to be implemented for now because the relevant type conversion
20 + operators/constructors are not consistently implemented by HIP/CUDA, so
21 + a generic conversion via type casts cannot be implemented.
22 +
23 + Each struct should have the member static constexpr bool `exists`:
24 + If false, the optimized kernel is not used for the corresponding torch type.
25 + If true, the struct should be fully defined as shown in the examples below.
26 + */
27 +template <typename torch_type>
28 +struct _typeConvert {
29 + static constexpr bool exists = false;
30 +};
31 +
32 +#if defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >= 12000))
33 +// CUDA < 12.0 runs into issues with packed type conversion
34 +template <>
35 +struct _typeConvert<c10::Half> {
36 + static constexpr bool exists = true;
37 + using hip_type = __half;
38 + using packed_hip_type = __half2;
39 +
40 + __device__ static inline float convert(hip_type x) { return __half2float(x); }
41 + __device__ static inline float2 convert(packed_hip_type x) {
42 + return __half22float2(x);
43 + }
44 + __device__ static inline hip_type convert(float x) {
45 + return __float2half_rn(x);
46 + }
47 + __device__ static inline packed_hip_type convert(float2 x) {
48 + return __float22half2_rn(x);
49 + }
50 +};
51 +
52 + #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
53 +// CUDA_ARCH < 800 does not have BF16 support
54 +// TODO: Add in ROCm support once public headers handle bf16 maturely
55 +template <>
56 +struct _typeConvert<c10::BFloat16> {
57 + static constexpr bool exists = true;
58 + using hip_type = __nv_bfloat16;
59 + using packed_hip_type = __nv_bfloat162;
60 +
61 + __device__ static inline float convert(hip_type x) {
62 + return __bfloat162float(x);
63 + }
64 + __device__ static inline float2 convert(packed_hip_type x) {
65 + return __bfloat1622float2(x);
66 + }
67 + __device__ static inline hip_type convert(float x) {
68 + return __float2bfloat16(x);
69 + }
70 + __device__ static inline packed_hip_type convert(float2 x) {
71 + return __float22bfloat162_rn(x);
72 + }
73 +};
74 + #endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
75 +#endif // defined(USE_ROCM) || (defined(CUDA_VERSION) && (CUDA_VERSION >=
76 + // 12000))
77 +
78 +/* Vector POD struct to generate vectorized and packed FP16/BF16 ops
79 + for appropriate specializations of fused_add_rms_norm_kernel.
80 + Only functions that are necessary in that kernel are implemented.
81 + Alignment to 16 bytes is required to use 128-bit global memory ops.
82 + */
83 +template <typename scalar_t, int width>
84 +struct alignas(16) _f16Vec {
85 + /* Not theoretically necessary that width is a power of 2 but should
86 + almost always be the case for optimization purposes */
87 + static_assert(width > 0 && (width & (width - 1)) == 0,
88 + "Width is not a positive power of 2!");
89 + using Converter = _typeConvert<scalar_t>;
90 + using T1 = typename Converter::hip_type;
91 + using T2 = typename Converter::packed_hip_type;
92 + T1 data[width];
93 +
94 + __device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
95 + if constexpr (width % 2 == 0) {
96 +#pragma unroll
97 + for (int i = 0; i < width; i += 2) {
98 + T2 temp{data[i], data[i + 1]};
99 + temp += T2{other.data[i], other.data[i + 1]};
100 + data[i] = temp.x;
101 + data[i + 1] = temp.y;
102 + }
103 + } else {
104 +#pragma unroll
105 + for (int i = 0; i < width; ++i) data[i] += other.data[i];
106 + }
107 + return *this;
108 + }
109 +
110 + __device__ _f16Vec& operator*=(const _f16Vec<scalar_t, width>& other) {
111 + if constexpr (width % 2 == 0) {
112 +#pragma unroll
113 + for (int i = 0; i < width; i += 2) {
114 + T2 temp{data[i], data[i + 1]};
115 + temp *= T2{other.data[i], other.data[i + 1]};
116 + data[i] = temp.x;
117 + data[i + 1] = temp.y;
118 + }
119 + } else {
120 +#pragma unroll
121 + for (int i = 0; i < width; ++i) data[i] *= other.data[i];
122 + }
123 + return *this;
124 + }
125 +
126 + __device__ _f16Vec& operator*=(const float scale) {
127 + if constexpr (width % 2 == 0) {
128 +#pragma unroll
129 + for (int i = 0; i < width; i += 2) {
130 + float2 temp_f = Converter::convert(T2{data[i], data[i + 1]});
131 + temp_f.x *= scale;
132 + temp_f.y *= scale;
133 + T2 temp = Converter::convert(temp_f);
134 + data[i] = temp.x;
135 + data[i + 1] = temp.y;
136 + }
137 + } else {
138 +#pragma unroll
139 + for (int i = 0; i < width; ++i) {
140 + float temp = Converter::convert(data[i]) * scale;
141 + data[i] = Converter::convert(temp);
142 + }
143 + }
144 + return *this;
145 + }
146 +
147 + __device__ float sum_squares() const {
148 + float result = 0.0f;
149 + if constexpr (width % 2 == 0) {
150 +#pragma unroll
151 + for (int i = 0; i < width; i += 2) {
152 + float2 z = Converter::convert(T2{data[i], data[i + 1]});
153 + result += z.x * z.x + z.y * z.y;
154 + }
155 + } else {
156 +#pragma unroll
157 + for (int i = 0; i < width; ++i) {
158 + float x = Converter::convert(data[i]);
159 + result += x * x;
160 + }
161 + }
162 + return result;
163 + }
164 +};
165 +} // namespace vllm
\ No newline at end of file
tests/compile/backend.py new
+33
@@ -0,0 +1,33 @@
1 +from copy import deepcopy
2 +from typing import Callable
3 +
4 +import torch
5 +
6 +
7 +class TestBackend:
8 + """
9 + This class provides a simple Inductor backend that can be used for testing.
10 + It takes a list of custom passes and runs them after Inductor's passes.
11 + It also saves the graph before and after the custom passes for inspection.
12 + """
13 +
14 + def __init__(self, *args: Callable[[torch.fx.Graph], None]):
15 + self.custom_passes = args
16 + from torch._inductor import config
17 + self.current_config = config.shallow_copy_dict()
18 + self.current_config['post_grad_custom_post_pass'] = self.post_pass
19 +
20 + def __call__(self, graph: torch.fx.GraphModule, example_inputs):
21 + from torch._inductor.compile_fx import compile_fx
22 + return compile_fx(graph,
23 + example_inputs,
24 + config_patches=self.current_config)
25 +
26 + def post_pass(self, graph: torch.fx.Graph):
27 + self.graph_pre_pass = deepcopy(graph)
28 + for pass_ in self.custom_passes:
29 + pass_(graph)
30 +
31 + self.graph_post_pass = deepcopy(graph)
32 + # assign by reference, will reflect the final state of the graph
33 + self.final_graph = graph
tests/compile/test_fusion.py new
+92
@@ -0,0 +1,92 @@
1 +import pytest
2 +import torch
3 +from compressed_tensors.quantization import FP8_DTYPE
4 +
5 +import vllm.envs as envs
6 +from vllm.compilation.config import CompilationConfig
7 +from vllm.compilation.fusion import (FusionPass, find_auto_fn,
8 + find_auto_fn_maybe)
9 +from vllm.compilation.reshapes import RedundantReshapesPass
10 +from vllm.model_executor.layers.layernorm import RMSNorm
11 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
12 + apply_fp8_linear)
13 +
14 +from .backend import TestBackend
15 +
16 +
17 +class TestModel(torch.nn.Module):
18 +
19 + def __init__(self, hidden_size: int, eps: float, *args, **kwargs):
20 + super().__init__(*args, **kwargs)
21 + self.norm = [RMSNorm(hidden_size, eps) for _ in range(3)]
22 + self.scale = [torch.rand(1, dtype=torch.float32) for _ in range(4)]
23 + self.w = [
24 + torch.rand(hidden_size, hidden_size).to(dtype=FP8_DTYPE).t()
25 + for _ in range(2)
26 + ]
27 +
28 + def forward(self, x):
29 + resid = torch.relu(x)
30 + y = self.norm[0](x)
31 +
32 + x2 = apply_fp8_linear(y, self.w[0], self.scale[0], self.scale[1])
33 + # make sure resid is used for replacement to work
34 + y2, resid = self.norm[1](x2, resid)
35 +
36 + x3 = apply_fp8_linear(y2, self.w[1], self.scale[2], self.scale[3])
37 + y3, resid = self.norm[2](x3, resid) # use resid here
38 + return y3
39 +
40 +
41 +# Init does pattern registration, which can only happen once
42 +config = CompilationConfig(enable_fusion=True)
43 +reshape_pass = RedundantReshapesPass(config)
44 +fusion_pass = FusionPass.instance(config)
45 +
46 +
47 +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
48 +@pytest.mark.parametrize("hidden_size", [64, 3392, 4096])
49 +@pytest.mark.parametrize("num_tokens", [7, 256, 533, 2048, 2049])
50 +@pytest.mark.parametrize("eps", [1e-5, 1e-6])
51 +@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
52 + reason="Only test on CUDA")
53 +def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
54 + torch.set_default_device("cuda")
55 + torch.set_default_dtype(torch.float16)
56 +
57 + if eps != 1e-5:
58 + pytest.skip("Only test eps=1e-5 for now")
59 +
60 + # Reshape pass is needed for the fusion pass to work
61 + backend = TestBackend(reshape_pass, fusion_pass)
62 + model = TestModel(hidden_size, eps)
63 +
64 + # First dimension dynamic
65 + x = torch.rand(num_tokens, hidden_size)
66 + torch._dynamo.mark_dynamic(x, 0)
67 +
68 + result = model(x)
69 +
70 + model2 = torch.compile(model, backend=backend)
71 + result2 = model2(x)
72 +
73 + # Check that it gives the same answer
74 + torch.testing.assert_close(result, result2, atol=1e-3, rtol=1e-3)
75 +
76 + # Check substitution worked
77 + pre_nodes = backend.graph_pre_pass.nodes
78 + post_nodes = backend.graph_post_pass.nodes
79 +
80 + rms_quant = torch.ops._C.rms_norm_static_fp8_quant.default
81 + add_rms_quant = torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
82 + fp8_quant = torch.ops._C.static_scaled_fp8_quant.default
83 +
84 + # In pre-nodes, fp8 quant should be present and fused kernels should not
85 + assert find_auto_fn_maybe(pre_nodes, rms_quant) is None
86 + assert find_auto_fn_maybe(pre_nodes, add_rms_quant) is None
87 + find_auto_fn(pre_nodes, fp8_quant)
88 +
89 + # In post-nodes, fused kernels should be present and fp8 quant should not
90 + find_auto_fn(post_nodes, rms_quant)
91 + find_auto_fn(post_nodes, add_rms_quant)
92 + assert find_auto_fn_maybe(post_nodes, fp8_quant) is None
tests/kernels/test_layernorm.py
+74 -1
@@ -1,13 +1,14 @@
1 import pytest
2 import torch
3
4 +from tests.kernels.quant_utils import FP8_DTYPE
5 from tests.kernels.utils import opcheck
6 from vllm.model_executor.layers.layernorm import RMSNorm
7 from vllm.platforms import current_platform
8
9 DTYPES = [torch.half, torch.bfloat16, torch.float]
10 NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
10 -HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192,
11 +HIDDEN_SIZES = [8, 768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192,
12 8199] # Arbitrary values for testing
13 ADD_RESIDUAL = [False, True]
14 SEEDS = [0]
@@ -59,3 +60,75 @@ def test_rms_norm(
60 else:
61 opcheck(torch.ops._C.rms_norm,
62 (out, x, layer.weight.data, layer.variance_epsilon))
63 +
64 +
65 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
66 +@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
67 +@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
68 +@pytest.mark.parametrize("dtype", DTYPES)
69 +@pytest.mark.parametrize("quant_scale", [1.0, 0.01, 10.0])
70 +@pytest.mark.parametrize("seed", SEEDS)
71 +@pytest.mark.parametrize("device", CUDA_DEVICES)
72 +def test_fused_rms_norm_quant(
73 + num_tokens: int,
74 + hidden_size: int,
75 + add_residual: bool,
76 + dtype: torch.dtype,
77 + quant_scale: float,
78 + seed: int,
79 + device: str,
80 +) -> None:
81 + current_platform.seed_everything(seed)
82 + torch.set_default_device(device)
83 +
84 + weight = torch.empty(hidden_size, dtype=dtype).normal_(mean=1.0, std=0.1)
85 + scale = 1 / (2 * hidden_size)
86 + x = torch.randn(num_tokens, hidden_size, dtype=dtype)
87 + x *= scale
88 + if add_residual:
89 + residual = torch.randn_like(x) * scale
90 + residual_fused = residual.clone()
91 + else:
92 + residual = residual_fused = None
93 +
94 + out_norm = torch.empty_like(x)
95 + out_quant = torch.empty_like(x, dtype=FP8_DTYPE)
96 + out_quant_fused = torch.empty_like(out_quant)
97 +
98 + quant_scale_t = torch.tensor(quant_scale, dtype=torch.float32)
99 +
100 + if add_residual:
101 + torch.ops._C.fused_add_rms_norm_static_fp8_quant(
102 + out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6)
103 +
104 + # Unfused kernel is in-place so it goes second
105 + # Also use a separate clone of x to avoid modifying the input
106 + x_unfused = x.clone()
107 + torch.ops._C.fused_add_rms_norm(x_unfused, residual, weight, 1e-6)
108 + torch.ops._C.static_scaled_fp8_quant(out_quant, x_unfused,
109 + quant_scale_t)
110 +
111 + torch.cuda.synchronize()
112 + torch.testing.assert_close(residual_fused,
113 + residual,
114 + atol=1e-2,
115 + rtol=1e-2)
116 +
117 + opcheck(
118 + torch.ops._C.fused_add_rms_norm_static_fp8_quant,
119 + (out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6))
120 + else:
121 + torch.ops._C.rms_norm_static_fp8_quant(out_quant_fused, x, weight,
122 + quant_scale_t, 1e-6)
123 +
124 + torch.ops._C.rms_norm(out_norm, x, weight, 1e-6)
125 + torch.ops._C.static_scaled_fp8_quant(out_quant, out_norm,
126 + quant_scale_t)
127 +
128 + opcheck(torch.ops._C.rms_norm_static_fp8_quant,
129 + (out_quant_fused, x, weight, quant_scale_t, 1e-6))
130 +
131 + torch.testing.assert_close(out_quant_fused.to(dtype=torch.float32),
132 + out_quant.to(dtype=torch.float32),
133 + atol=1e-3,
134 + rtol=1e-3)
vllm/compilation/backends.py
+94 -15
@@ -2,7 +2,8 @@ import copy
2 import dataclasses
3 import operator
4 from contextlib import ExitStack
5 -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
5 +from typing import (Any, Callable, Dict, List, Optional, Sequence, Set, Tuple,
6 + Union)
7 from unittest.mock import patch
8
9 import torch
@@ -10,11 +11,13 @@ import torch.fx as fx
11
12 import vllm.envs as envs
13 from vllm.logger import init_logger
13 -from vllm.utils import weak_ref_tensors
14 +from vllm.utils import combine_fx_passes, weak_ref_tensors
15
16 from .config import CompilationConfig
17 from .counter import compilation_counter
18 +from .fusion import FusionPass
19 from .levels import CompilationLevel
20 +from .reshapes import RedundantReshapesPass
21
22 logger = init_logger(__name__)
23
@@ -99,28 +102,74 @@ def fix_functionalization(graph: fx.Graph):
102 user.replace_all_uses_with(replace_node)
103 nodes_to_remove.append(user)
104 nodes_to_remove.append(node)
105 + elif (node.args[0] ==
106 + torch.ops._C.fused_add_rms_norm_static_fp8_quant.default):
107 + # manual replace for fused_add_rms_norm_static_fp8_quant
108 + # this is the most effective optimization for llama
109 + # failing to do this will result in many unnecessary copies
110 +
111 + kwargs = node.kwargs
112 +
113 + result = kwargs['result']
114 + residual = kwargs['residual']
115 +
116 + # Create a new call to
117 + # torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
118 + with graph.inserting_before(node):
119 + # just insert the call to the custom op
120 + # NOTE: don't run dead code elimination,
121 + # otherwise this op will be removed
122 + graph.call_function(
123 + torch.ops._C.fused_add_rms_norm_static_fp8_quant.
124 + default,
125 + kwargs=kwargs)
126 +
127 + for user in list(node.users):
128 + if user.op == 'call_function' and user.target == operator.getitem: # noqa
129 + # Remove the getitem node
130 + if user.args[1] == 1:
131 + replace_node = result
132 + elif user.args[1] == 2:
133 + replace_node = residual
134 + user.replace_all_uses_with(replace_node)
135 + nodes_to_remove.append(user)
136 + nodes_to_remove.append(node)
137
138 elif node.args[0] == torch.ops._C.rms_norm.default:
139 # manual replace for rms_norm
140
141 kwargs = node.kwargs
142
108 - input = kwargs['input']
109 - out = kwargs['out']
110 - weight = kwargs['weight']
111 - epsilon = kwargs['epsilon']
112 - # Create a new call to torch.ops._C.rotary_embedding.default
113 - # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
143 + replace_node = kwargs['result']
144 + # Create a new call to torch.ops._C.rms_norm.default
145 with graph.inserting_before(node):
146 # just insert the call to the custom op
147 # NOTE: don't run dead code elimination,
148 # otherwise this op will be removed
118 - graph.call_function(
119 - torch.ops._C.rms_norm.default,
120 - args=(out, input, weight, epsilon),
121 - )
149 + graph.call_function(torch.ops._C.rms_norm.default,
150 + kwargs=kwargs)
151
123 - replace_node = out
152 + for user in list(node.users):
153 + if user.op == 'call_function' and user.target == operator.getitem: # noqa
154 + user.replace_all_uses_with(replace_node)
155 + nodes_to_remove.append(user)
156 + nodes_to_remove.append(node)
157 +
158 + elif node.args[
159 + 0] == torch.ops._C.rms_norm_static_fp8_quant.default: # noqa
160 + # manual replace for rms_norm_static_fp8_quant
161 +
162 + kwargs = node.kwargs
163 +
164 + replace_node = kwargs['result']
165 + # Create a new call to torch.ops._C.rms_norm_static_fp8_quant.default # noqa
166 + with graph.inserting_before(node):
167 + # just insert the call to the custom op
168 + # NOTE: don't run dead code elimination,
169 + # otherwise this op will be removed
170 + graph.call_function(
171 + torch.ops._C.rms_norm_static_fp8_quant.default,
172 + kwargs=kwargs)
173
174 for user in list(node.users):
175 if user.op == 'call_function' and user.target == operator.getitem: # noqa
@@ -136,7 +185,7 @@ def fix_functionalization(graph: fx.Graph):
185 input = kwargs['input']
186 out = kwargs['out']
187
139 - # Create a new call to torch.ops._C.rotary_embedding.default
188 + # Create a new call to torch.ops._C.silu_and_mul.default
189 # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
190 with graph.inserting_before(node):
191 # just insert the call to the custom op
@@ -319,6 +368,13 @@ class VllmBackend:
368
369 The major work of this backend is to split the graph into
370 piecewise graphs, and pass them to the piecewise backend.
371 +
372 + This backend also handles custom passes and adds them to Inductor config.
373 + The order of the post-grad post-passes is:
374 + 1. post_grad_passes (constructor parameter)
375 + 2. config["post_grad_custom_post_pass"]
376 + 3. fix_functionalization
377 + This way, all passes operate on a functionalized graph.
378 """
379
380 compilation_configs: CompilationConfig
@@ -330,8 +386,10 @@ class VllmBackend:
386 split_gm: fx.GraphModule
387 piecewise_graphs: List[SplitItem]
388 returned_callable: Callable
389 + # Inductor passes to run on the graph pre-defunctionalization
390 + post_grad_passes: Sequence[Callable]
391
334 - def __init__(self, ):
392 + def __init__(self, post_grad_passes: Sequence[Callable] = ()):
393 global global_graph_pool
394 if global_graph_pool is None:
395 global_graph_pool = torch.cuda.graph_pool_handle()
@@ -340,10 +398,30 @@ class VllmBackend:
398 # streams, it might not be safe to share a global pool.
399 # only investigate this when we use multiple streams
400 self.graph_pool = global_graph_pool
401 + self.post_grad_passes = post_grad_passes
402
403 # `torch.compile` is JIT compiled, so we don't need to
404 # do anything here
405
406 + def add_passes_to_config(self):
407 + config = self.compilation_configs
408 + passes = list(self.post_grad_passes)
409 +
410 + passes = passes + [RedundantReshapesPass(config)]
411 +
412 + if config.enable_fusion:
413 + passes = passes + [FusionPass.instance(config)]
414 +
415 + inductor_config = config.inductor_compile_config
416 + if "post_grad_custom_post_pass" in inductor_config:
417 + passes = passes + [inductor_config["post_grad_custom_post_pass"]]
418 +
419 + # add the fix_functionalization pass last, so that all other
420 + # passes operate on a functionalized graph
421 + passes = passes + [fix_functionalization]
422 + combined_pass = combine_fx_passes(passes)
423 + inductor_config["post_grad_custom_post_pass"] = combined_pass
424 +
425 def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
426
427 compilation_counter.num_graphs_seen += 1
@@ -357,6 +435,7 @@ class VllmBackend:
435 # we get the sizes to capture for cudagraph
436 # from compilation context
437 self.compilation_configs = CompilationConfig.select_and_init_config()
438 + self.add_passes_to_config()
439
440 self.split_gm, self.piecewise_graphs = split_graph(
441 graph, self.compilation_configs.non_cudagraph_ops)
vllm/compilation/config.py
+12 -13
@@ -1,4 +1,5 @@
1 import copy
2 +from pathlib import Path
3 from typing import Any, Dict, List, Optional
4
5 from pydantic import BaseModel, Field, PrivateAttr
@@ -50,6 +51,12 @@ class CompilationConfig(BaseModel):
51 name because the config uses json format. If we pass the config
52 from Python, functions can also be passed directly via Python object
53 constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
54 + - Custom inductor passes:
55 + - dump_graph_stages: list of stages for which we want to dump the graph.
56 + Each pass defines its own stages (before, after, maybe in-between).
57 + - dump_graph_dir: directory to dump the graph. Default is .
58 + - enable_fusion: whether to enable the custom fusion pass.
59 + TODO better pass enabling system.
60
61 Why we have different sizes for cudagraph and inductor:
62 - cudagraph: a cudagraph captured for a specific size can only be used
@@ -72,6 +79,10 @@ class CompilationConfig(BaseModel):
79 cudagraph_num_of_warmups: int = 0
80 cudagraph_capture_sizes: Optional[List[int]] = None
81
82 + dump_graph_stages: List[str] = Field(default_factory=list)
83 + dump_graph_dir: Path = Field(default=Path("."))
84 + enable_fusion: bool = True
85 +
86 # not configurable, computed after init
87 compile_sizes: List[int] = PrivateAttr
88 capture_sizes: List[int] = PrivateAttr
@@ -81,7 +92,7 @@ class CompilationConfig(BaseModel):
92 if not isinstance(v, str):
93 assert callable(v), (
94 f"pass {k} should be a function or a qualified name")
84 - self.inductor_passes[k] = v
95 + self.inductor_compile_config[k] = v
96 continue
97
98 # resolve function from qualified name
@@ -91,18 +102,6 @@ class CompilationConfig(BaseModel):
102 func = __import__(module).__dict__[func_name]
103 self.inductor_compile_config[k] = func
104
94 - from vllm.compilation.backends import fix_functionalization
95 - from vllm.utils import combine_fx_passes
96 - if "post_grad_custom_post_pass" in self.inductor_compile_config:
97 - self.inductor_compile_config[
98 - "post_grad_custom_post_pass"] = combine_fx_passes(
99 - fix_functionalization,
100 - self.inductor_compile_config["post_grad_custom_post_pass"],
101 - )
102 - else:
103 - self.inductor_compile_config[
104 - "post_grad_custom_post_pass"] = fix_functionalization
105 -
105 def init_during_runtime(self):
106 """To complete the initialization of config,
107 we need to know the compile context, which is only available
vllm/compilation/fusion.py new
+291
@@ -0,0 +1,291 @@
1 +import operator
2 +from typing import Iterable, List, Optional
3 +
4 +import torch
5 +from torch._higher_order_ops.auto_functionalize import auto_functionalized
6 +from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7 + fwd_only, register_replacement)
8 +
9 +from vllm.compilation.config import CompilationConfig
10 +from vllm.compilation.inductor_pass import InductorPass
11 +from vllm.logger import init_logger
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +def rms_pattern_static(result: torch.Tensor, result_rms: torch.Tensor,
17 + input: torch.Tensor, weight: torch.Tensor,
18 + scale: torch.Tensor):
19 + at1 = auto_functionalized(torch.ops._C.rms_norm.default,
20 + result=result_rms,
21 + input=input,
22 + weight=weight,
23 + epsilon=1e-5)
24 + at2 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
25 + result=result,
26 + input=at1[1],
27 + scale=scale)
28 +
29 + # result
30 + return at2[1]
31 +
32 +
33 +def rms_replacement_static(result: torch.Tensor, result_rms: torch.Tensor,
34 + input: torch.Tensor, weight: torch.Tensor,
35 + scale: torch.Tensor):
36 + at = auto_functionalized(torch.ops._C.rms_norm_static_fp8_quant.default,
37 + result=result,
38 + input=input,
39 + weight=weight,
40 + scale=scale,
41 + epsilon=1e-5)
42 +
43 + # result
44 + return at[1]
45 +
46 +
47 +def rms_pattern_residual_static(result: torch.Tensor, input: torch.Tensor,
48 + residual: torch.Tensor, weight: torch.Tensor,
49 + scale: torch.Tensor):
50 + at = auto_functionalized(torch.ops._C.fused_add_rms_norm.default,
51 + input=input,
52 + residual=residual,
53 + weight=weight,
54 + epsilon=1e-5)
55 + at1 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
56 + result=result,
57 + input=at[1],
58 + scale=scale)
59 +
60 + # result, residual
61 + return at1[1], at[2]
62 +
63 +
64 +def rms_replacement_residual_static(result: torch.Tensor, input: torch.Tensor,
65 + residual: torch.Tensor,
66 + weight: torch.Tensor, scale: torch.Tensor):
67 + at = auto_functionalized(
68 + torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
69 + result=result,
70 + input=input,
71 + residual=residual,
72 + weight=weight,
73 + scale=scale,
74 + epsilon=1e-5)
75 + # result, residual
76 + return at[1], at[2]
77 +
78 +
79 +def empty_bf16(*args, **kwargs):
80 + return torch.empty(*args, **kwargs, dtype=torch.bfloat16, device="cuda")
81 +
82 +
83 +def empty_fp8(*args, **kwargs):
84 + fp8 = torch.float8_e4m3fn
85 + return torch.empty(*args, **kwargs, dtype=fp8, device="cuda")
86 +
87 +
88 +def empty_fp32(*args, **kwargs):
89 + return torch.empty(*args, **kwargs, dtype=torch.float32, device="cuda")
90 +
91 +
92 +# Utilities for post-processing multi-output matches
93 +def is_func(node: torch.fx.Node, target) -> bool:
94 + return node.op == "call_function" and node.target == target
95 +
96 +
97 +# Returns the first auto_functionalized node with the given op (if it exists)
98 +def find_auto_fn_maybe(nodes: Iterable[torch.fx.Node],
99 + op) -> Optional[torch.fx.Node]:
100 + for node in nodes:
101 + if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
102 + return node
103 + return None
104 +
105 +
106 +# Returns the first auto_functionalized node with the given op
107 +def find_auto_fn(nodes: Iterable[torch.fx.Node], op) -> torch.fx.Node:
108 + node = find_auto_fn_maybe(nodes, op)
109 + assert node is not None, f"Could not find {op} in nodes {nodes}"
110 + return node
111 +
112 +
113 +# Returns the getitem node that extracts the idx-th element from node
114 +# (if it exists)
115 +def find_getitem_maybe(node: torch.fx.Node,
116 + idx: int) -> Optional[torch.fx.Node]:
117 + for user in node.users:
118 + if is_func(user, operator.getitem) and user.args[1] == idx:
119 + return user
120 + return None
121 +
122 +
123 +# Returns the getitem node that extracts the idx-th element from node
124 +def find_getitem(node: torch.fx.Node, idx: int) -> torch.fx.Node:
125 + ret = find_getitem_maybe(node, idx)
126 + assert ret is not None, f"Could not find getitem {idx} in node {node}"
127 + return ret
128 +
129 +
130 +class FusionPass(InductorPass):
131 + """
132 + This pass fuses a pre-defined set of custom ops into fused ops.
133 + It uses the torch pattern matcher to find the patterns and replace them.
134 + It also manually processes multi-output matches, as those are broken in
135 + the torch pattern matcher.
136 +
137 + Because patterns can only be registered once, the pass is a singleton.
138 + This will be addressed in a future version of PyTorch:
139 + https://github.com/pytorch/pytorch/pull/139321#issuecomment-2452354980
140 + """
141 +
142 + _instance: 'Optional[FusionPass]' = None
143 +
144 + @classmethod
145 + def instance(cls, config: CompilationConfig):
146 + """
147 + Get the singleton instance of the FusionPass.
148 + If the instance exists, the config is updated but
149 + initialization is not repeated.
150 + """
151 + if cls._instance is None:
152 + cls._instance = FusionPass(config)
153 + else:
154 + cls._instance.config = config
155 + return cls._instance
156 +
157 + def __init__(self, config: CompilationConfig):
158 + assert self.__class__._instance is None, \
159 + "FusionPass singleton instance already exists"
160 + super().__init__(config)
161 +
162 + self.matches: List[Match] = []
163 + self.patterns: PatternMatcherPass = PatternMatcherPass(
164 + pass_name="fusion_pass")
165 +
166 + # Fuse rms_norm + static_scaled_fp8_quant into
167 + # rms_norm_static_fp8_quant
168 + inputs = [
169 + empty_fp8(5, 4),
170 + empty_bf16(5, 4),
171 + empty_bf16(5, 4),
172 + empty_bf16(1, 5),
173 + empty_fp32(1, 1)
174 + ]
175 + register_replacement(rms_pattern_static, rms_replacement_static,
176 + inputs, fwd_only, self.patterns)
177 +
178 + # Fuse fused_add_rms_norm + static_scaled_fp8_quant into
179 + # fused_add_rms_norm_static_fp8_quant
180 + # Because pattern has 2 outputs, we need to manually process the match
181 + # (see process_matches)
182 + inputs = [
183 + empty_fp8(5, 4),
184 + empty_bf16(5, 4),
185 + empty_bf16(5, 4),
186 + empty_bf16(1, 5),
187 + empty_fp32(1, 1)
188 + ]
189 + register_replacement(rms_pattern_residual_static,
190 + rms_replacement_residual_static,
191 + inputs,
192 + fwd_only,
193 + self.patterns,
194 + extra_check=lambda m: self.record_match(m))
195 +
196 + def record_match(self, match: Match) -> bool:
197 + # Hijack the extra_check to record the match and
198 + # save it for post-processing.
199 + self.matches.append(match)
200 +
201 + # Return False to prevent automatic replacement.
202 + return False
203 +
204 + def process_matches(self, graph: torch.fx.Graph):
205 + """
206 + Manually process multi-output matches and replace them with fused nodes.
207 + This is necessary because the automatic replacement for multi-output
208 + matches is broken: https://github.com/pytorch/pytorch/issues/137280
209 + """
210 + for match in self.matches:
211 + # To avoid use-before-definition errors, insert replacement nodes
212 + # after the last node in the match.
213 + # match.nodes is not guaranteed to be sorted.
214 + # Find the last node in the match.
215 + for last_node_in_match in reversed(graph.nodes):
216 + if last_node_in_match in match.nodes:
217 + break
218 + else:
219 + raise ValueError("No nodes in graph")
220 +
221 + # Insert a new auto_functionalized node for the fused operation,
222 + # as well as getitem nodes to extract the result and residual.
223 + # The auto_functionalized node returns a tuple of
224 + # (None, result, residual) - None is the function return value.
225 + # The resulting graph looks like this:
226 + # at = auto_functionalized(torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, ...) # noqa
227 + # result_node_new = at[1]
228 + # residual_node_new = at[2]
229 + with graph.inserting_after(last_node_in_match):
230 + kwargs = match.kwargs
231 + kwargs["epsilon"] = 1e-5 # Currently hard-coded in RMSNorm
232 +
233 + fused_node = graph.call_function(
234 + auto_functionalized,
235 + (torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
236 + ),
237 + kwargs=kwargs)
238 +
239 + graph.inserting_after(fused_node)
240 + result_node_new = graph.call_function(operator.getitem,
241 + (fused_node, 1))
242 + residual_node_new = graph.call_function(
243 + operator.getitem, (fused_node, 2))
244 +
245 + # Last part of replacement is rebinding the users of nodes in the
246 + # match to use the new nodes.
247 +
248 + # Find the nodes in the match that we need to rebind
249 + rms_node = find_auto_fn(match.nodes,
250 + torch.ops._C.fused_add_rms_norm.default)
251 + quant_node = find_auto_fn(
252 + match.nodes, torch.ops._C.static_scaled_fp8_quant.default)
253 +
254 + assert len(rms_node.users) == 2
255 + assert len(quant_node.users) == 1
256 +
257 + # meta["val"] is used by de-functionalization and has to contain the
258 + # value of the node (tuple of tensors) that would be returned by the
259 + # functionalized node during tracing.
260 +
261 + rms_tup = rms_node.meta["val"]
262 + quant_tup = quant_node.meta["val"]
263 +
264 + # The result of fused_node must be a tuple with the first element
265 + # None (the function return value) and the remaining elements
266 + # representing the mutated inputs.
267 + fused_tup = (None, quant_tup[1], rms_tup[1], rms_tup[2])
268 + fused_node.meta["val"] = fused_tup
269 +
270 + # Find the getitem nodes and replace their uses with the new nodes.
271 + # The old nodes will be removed by DCE at the end of the pass.
272 + find_getitem(rms_node, 2).replace_all_uses_with(residual_node_new)
273 + find_getitem(quant_node, 1).replace_all_uses_with(result_node_new)
274 +
275 + # Finally, remove matched nodes
276 + graph.eliminate_dead_code()
277 + assert all(node not in graph.nodes for match in self.matches
278 + for node in match.nodes)
279 +
280 + def __call__(self, graph: torch.fx.Graph):
281 + self.dump_graph(graph, "before_fusion")
282 +
283 + count = self.patterns.apply(graph)
284 + logger.info("Replaced %s patterns", count)
285 + self.dump_graph(graph, "after_pattern_match")
286 +
287 + # Manually process multi-output matches (and run DCE)
288 + self.process_matches(graph)
289 + logger.info("Post-processed %s matches", len(self.matches))
290 + self.dump_graph(graph, "after_fusion")
291 + self.matches.clear()
vllm/compilation/inductor_pass.py new
+38
@@ -0,0 +1,38 @@
1 +from abc import ABC, abstractmethod
2 +
3 +import torch
4 +
5 +from vllm.compilation.config import CompilationConfig
6 +# yapf: disable
7 +from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8 +from vllm.distributed import (
9 + get_tensor_model_parallel_world_size as get_tp_world_size)
10 +from vllm.distributed import model_parallel_is_initialized as p_is_init
11 +# yapf: enable
12 +from vllm.logger import init_logger
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +class InductorPass(ABC):
18 +
19 + @abstractmethod
20 + def __call__(self, graph: torch.fx.Graph):
21 + raise NotImplementedError
22 +
23 + def __init__(self, config: CompilationConfig):
24 + self.config = config
25 +
26 + def dump_graph(self, graph: torch.fx.Graph, stage: str):
27 + if stage in self.config.dump_graph_stages:
28 + # Make sure filename includes rank in the distributed setting
29 + parallel = p_is_init() and get_tp_world_size() > 1
30 + rank = f"-{get_tp_rank()}" if parallel else ""
31 + filepath = self.config.dump_graph_dir / f"{stage}{rank}.py"
32 +
33 + logger.info("Printing graph to %s", filepath)
34 + with open(filepath, "w") as f:
35 + src = graph.python_code(root_module="self", verbose=True).src
36 + # Add imports so it's not full of errors
37 + print("import torch; from torch import device", file=f)
38 + print(src, file=f)
vllm/compilation/reshapes.py new
+85
@@ -0,0 +1,85 @@
1 +from typing import Union
2 +
3 +import torch.fx
4 +from torch import SymInt
5 +
6 +from vllm.compilation.fusion import is_func
7 +from vllm.compilation.inductor_pass import InductorPass
8 +from vllm.logger import init_logger
9 +
10 +logger = init_logger(__name__)
11 +
12 +
13 +class RedundantReshapesPass(InductorPass):
14 + """
15 + This is an inductor pass that removes redundant reshape operations.
16 + It is required for RMSNorm-quant fusion to work properly.
17 + That's because apply_fp8_linear adds a reshape, which is redundant
18 + in the 2D-case.
19 +
20 + Example graph:
21 +
22 + getitem_1: "f16[s0, 4096]" = ...
23 + view_1: "f16[s0, 4096]" = torch.reshape(getitem_1, [-1, 4096])
24 + at = auto_functionalized(static_scaled_fp8_quant, input = view_1, ...)
25 + out: "f8e4m3fn[s0, 4096]" = at[1]
26 +
27 + Can be replaced with:
28 + getitem_1: "f16[s0, 4096]" = ...
29 + at = auto_functionalized(static_scaled_fp8_quant, input = getitem_1, ...)
30 + out: "f8e4m3fn[s0, 4096]" = at[1]
31 + """
32 +
33 + def __call__(self, graph: torch.fx.Graph):
34 + self.dump_graph(graph, "before_reshapes")
35 + count = 0
36 + # Remove no-op reshapes/views:
37 + for node in graph.nodes:
38 + if is_func(node, torch.ops.aten.reshape.default):
39 + input, shape = node.args[:2]
40 + input_shape = input.meta["val"].shape
41 + if len(shape) != len(input_shape):
42 + # Reshape changing rank, skip
43 + continue
44 +
45 + if shape.count(-1) > 1:
46 + # Invalid reshape args, skip
47 + continue
48 +
49 + if all(
50 + self.dims_equivalent(s, i_s)
51 + for s, i_s in zip(shape, input_shape)):
52 + node.replace_all_uses_with(input)
53 + graph.erase_node(node)
54 + count += 1
55 +
56 + logger.info("Removed %s no-op reshapes", count)
57 +
58 + self.dump_graph(graph, "after_reshapes")
59 +
60 + def dims_equivalent(self, dim: Union[int, torch.fx.Node],
61 + i_dim: Union[int, SymInt]) -> bool:
62 + """
63 + This function checks if two dimensions are equivalent.
64 + :param dim: The dimension arg to reshape
65 + :param i_dim: The corresponding dimension in the input tensor
66 + :return: Are the dimensions equivalent?
67 +
68 + There are three cases in which the dimensions are equivalent:
69 + 1. The dimensions are equal (both integers)
70 + 2. The reshape dimension is -1 (i.e. inferred)
71 + 3. The dimensions both correspond to the same SymInt
72 +
73 + While case 2 does not guarantee the dimensions are equal,
74 + they are equal if all other dimensions are equal.
75 +
76 + In case 3, the reshape dimension is a torch.fx.Node,
77 + and its value is a SymInt. That value is equal to the
78 + input dimension.
79 +
80 + """
81 + # Case 1 and 2
82 + if dim == i_dim or dim == -1:
83 + return True
84 + # Case 3
85 + return isinstance(dim, torch.fx.Node) and dim.meta["val"] == i_dim
vllm/envs.py
+2
@@ -68,6 +68,7 @@ if TYPE_CHECKING:
68 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
69 VLLM_SKIP_P2P_CHECK: bool = False
70 VLLM_TORCH_COMPILE_LEVEL: int = 0
71 + VLLM_TORCH_COMPILE_CONFIG: Optional[str] = None
72 VLLM_CUSTOM_OPS: List[str] = []
73 VLLM_DISABLED_KERNELS: List[str] = []
74 VLLM_USE_V1: bool = False
@@ -226,6 +227,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
227 # and disabled when running with Inductor (compile_level >= Inductor).
228 "VLLM_CUSTOM_OPS":
229 lambda: os.environ.get("VLLM_CUSTOM_OPS", "").replace(" ", "").split(","),
230 +
231 # local rank of the process in the distributed setting, used to determine
232 # the GPU device id
233 "LOCAL_RANK":