@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 30870b4f66
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[torch.compile] Dynamic fp8 + rms_norm fusion (#10906)

Signed-off-by: luka <luka@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Luka Govedič committed Dec 12, 2024 at 22:19 UTC 30870b4f66414020645608b81dced94d8a99111c
20 files changed +1736 -252
CMakeLists.txt
+2 -1
@@ -196,6 +196,7 @@ set(VLLM_EXT_SRC
196 "csrc/quantization/gptq/q_gemm.cu"
197 "csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
198 "csrc/quantization/fp8/common.cu"
199 + "csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu"
200 "csrc/quantization/gguf/gguf_kernel.cu"
201 "csrc/cuda_utils_kernels.cu"
202 "csrc/prepare_inputs/advance_step.cu"
@@ -300,7 +301,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
301 #
302 # For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
303 # kernels for the remaining archs that are not already built for 3x.
303 - cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
304 + cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
305 "7.5;8.0;8.6;8.7;8.9;9.0" "${CUDA_ARCHS}")
306 # subtract out the archs that are already built for 3x
307 list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
benchmarks/fused_kernels/layernorm_rms_benchmarks.py new
+173
@@ -0,0 +1,173 @@
1 +import pickle as pkl
2 +import time
3 +from dataclasses import dataclass
4 +from itertools import product
5 +from typing import Callable, Iterable, List, Optional
6 +
7 +import torch
8 +import torch.utils.benchmark as TBenchmark
9 +from torch.utils.benchmark import Measurement as TMeasurement
10 +from tqdm import tqdm
11 +
12 +import vllm._custom_ops as ops
13 +from vllm.model_executor.layers.layernorm import RMSNorm
14 +
15 +
16 +@dataclass
17 +class bench_params_t:
18 + num_tokens: int
19 + hidden_size: int
20 + add_residual: bool
21 + dtype: torch.dtype
22 +
23 + def description(self):
24 + return (f'N {self.num_tokens} '
25 + f'x D {self.hidden_size} '
26 + f'x R {self.add_residual} '
27 + f'x DT {self.dtype}')
28 +
29 +
30 +def get_bench_params() -> List[bench_params_t]:
31 + ## Test Fixtures
32 + NUM_TOKENS = [2**x for x in range(11)]
33 + HIDDEN_SIZES = list(range(1024, 8129, 1024))
34 + ADD_RESIDUAL = [True, False]
35 + DTYPES = [torch.bfloat16, torch.float]
36 +
37 + combinations = product(NUM_TOKENS, HIDDEN_SIZES, ADD_RESIDUAL, DTYPES)
38 + bench_params = list(map(lambda x: \
39 + bench_params_t(x[0], x[1], x[2], x[3]), combinations))
40 + return bench_params
41 +
42 +
43 +# Reference impls
44 +def unfused_int8_impl(rms_norm_layer: RMSNorm, x: torch.Tensor,
45 + residual: Optional[torch.Tensor],
46 + quant_dtype: torch.dtype):
47 + # Norm
48 + torch_out = None
49 + if residual is None:
50 + torch_out = rms_norm_layer.forward_cuda(x, residual)
51 + else:
52 + torch_out, _ = rms_norm_layer.forward_cuda(x, residual)
53 +
54 + # Quant
55 + torch_out, _, _ = ops.scaled_int8_quant(torch_out)
56 +
57 +
58 +def unfused_fp8_impl(rms_norm_layer: RMSNorm, x: torch.Tensor,
59 + residual: Optional[torch.Tensor],
60 + quant_dtype: torch.dtype):
61 + # Norm
62 + torch_out = None
63 + if residual is None:
64 + torch_out = rms_norm_layer.forward_cuda(x, residual)
65 + else:
66 + torch_out, _ = rms_norm_layer.forward_cuda(x, residual)
67 +
68 + # Quant
69 + torch_out, _ = ops.scaled_fp8_quant(torch_out)
70 +
71 +
72 +def fused_impl(
73 + rms_norm_layer: RMSNorm, # this stores the weights
74 + x: torch.Tensor,
75 + residual: Optional[torch.Tensor],
76 + quant_dtype: torch.dtype):
77 + out, _ = ops.rms_norm_dynamic_per_token_quant(x,
78 + rms_norm_layer.weight,
79 + 1e-6,
80 + quant_dtype,
81 + residual=residual)
82 +
83 +
84 +# Bench functions
85 +def bench_fn(rms_norm_layer: RMSNorm, x: torch.Tensor, residual: torch.Tensor,
86 + quant_dtype: torch.dtype, label: str, sub_label: str,
87 + fn: Callable, description: str) -> TMeasurement:
88 +
89 + min_run_time = 1
90 +
91 + globals = {
92 + "rms_norm_layer": rms_norm_layer,
93 + "x": x,
94 + "residual": residual,
95 + "quant_dtype": quant_dtype,
96 + "fn": fn,
97 + }
98 + return TBenchmark.Timer(
99 + stmt="fn(rms_norm_layer, x, residual, quant_dtype)",
100 + globals=globals,
101 + label=label,
102 + sub_label=sub_label,
103 + description=description,
104 + ).blocked_autorange(min_run_time=min_run_time)
105 +
106 +def bench(params: bench_params_t, label: str, sub_label: str) \
107 + -> Iterable[TMeasurement]:
108 +
109 + # Make inputs
110 + layer = RMSNorm(params.hidden_size, 1e-6).to(dtype=params.dtype)
111 + # Make weights
112 + layer.weight.data.normal_(mean=1.0, std=0.1)
113 + # Make inputs
114 + scale = 1 / params.hidden_size
115 + x = torch.randn(params.num_tokens,
116 + params.hidden_size,
117 + dtype=params.dtype,
118 + device='cuda') * scale
119 + residual = (torch.randn_like(x) * scale).to(device='cuda') \
120 + if params.add_residual else None
121 +
122 + timers = []
123 +
124 + # unfused int8 impl.
125 + timers.append(
126 + bench_fn(layer, x, residual, torch.int8, label, sub_label,
127 + unfused_int8_impl, "unfused_int8_impl"))
128 +
129 + # unfused fp8 impl.
130 + timers.append(
131 + bench_fn(layer, x, residual, torch.float8_e4m3fn, label, sub_label,
132 + unfused_fp8_impl, "unfused_fp8_impl"))
133 +
134 + # fused int8 impl.
135 + timers.append(
136 + bench_fn(layer, x, residual, torch.int8, label, sub_label, fused_impl,
137 + "fused_int8_impl"))
138 +
139 + # fused fp8 impl.
140 + timers.append(
141 + bench_fn(layer, x, residual, torch.float8_e4m3fn, label, sub_label,
142 + fused_impl, "fused_fp8_impl"))
143 +
144 + print_timers(timers)
145 +
146 + return timers
147 +
148 +
149 +# launch bench
150 +# runner
151 +def print_timers(timers: Iterable[TMeasurement]):
152 + compare = TBenchmark.Compare(timers)
153 + compare.print()
154 +
155 +
156 +def main():
157 + torch.set_default_device('cuda')
158 + bench_params = get_bench_params()
159 +
160 + timers = []
161 + for bp in tqdm(bench_params):
162 + timers.extend(
163 + bench(bp, "rms-norm-dynamic-per-token-quant", bp.description()))
164 + print_timers(timers)
165 +
166 + # pickle all the results
167 + timestamp = int(time.time())
168 + with open(f"rms_norm_dpt_quant-{timestamp}.pkl", "wb") as f:
169 + pkl.dump(timers, f)
170 +
171 +
172 +if __name__ == '__main__':
173 + main()
csrc/dispatch_utils.h
+14
@@ -14,6 +14,20 @@
14 #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15 AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
16
17 +// TODO(luka/varun): use FP8_TYPE macro after refactoring
18 +#ifndef USE_ROCM
19 + #define VLLM_DISPATCH_CASE_QUANT_TYPES(...) \
20 + AT_DISPATCH_CASE(at::ScalarType::Float8_e4m3fn, __VA_ARGS__) \
21 + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__)
22 +#else
23 + #define VLLM_DISPATCH_CASE_QUANT_TYPES(...) \
24 + AT_DISPATCH_CASE(at::ScalarType::Float8_e4m3fnuz, __VA_ARGS__) \
25 + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__)
26 +#endif
27 +
28 +#define VLLM_DISPATCH_QUANT_TYPES(TYPE, NAME, ...) \
29 + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_QUANT_TYPES(__VA_ARGS__))
30 +
31 #define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \
32 AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
33 AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
csrc/ops.h
+8
@@ -66,6 +66,14 @@ void fused_add_rms_norm_static_fp8_quant(torch::Tensor& out,
66 torch::Tensor& weight,
67 torch::Tensor& scale, double epsilon);
68
69 +void rms_norm_dynamic_per_token_quant(torch::Tensor& out,
70 + torch::Tensor const& input,
71 + torch::Tensor const& weight,
72 + torch::Tensor& scales,
73 + double const epsilon,
74 + std::optional<torch::Tensor> scale_ub,
75 + std::optional<torch::Tensor> residual);
76 +
77 void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
78 torch::Tensor& key, int64_t head_size,
79 torch::Tensor& cos_sin_cache, bool is_neox);
csrc/quantization/fp8/common.cuh
+7 -19
@@ -1,6 +1,9 @@
1 #pragma once
2
3 +#include "quantization/vectorization.cuh"
4 +
5 #include <cmath>
6 +#include <c10/core/ScalarType.h>
7
8 #ifndef USE_ROCM
9 #include <c10/util/Float8_e4m3fn.h>
@@ -15,6 +18,7 @@ using FP8_TYPE = c10::Float8_e4m3fnuz;
18 // issue when running dynamic quantization. Here use 224.0f for rocm.
19 constexpr auto FP8_E4M3_MAX = 224.0f;
20 #endif
21 +constexpr static auto kFp8Type = c10::CppTypeToScalarType<FP8_TYPE>::value;
22
23 namespace vllm {
24
@@ -89,22 +93,6 @@ __global__ void segmented_max_reduction(float* __restrict__ scale,
93 }
94 }
95
92 -template <typename scalar_t>
93 -struct __align__(8) vec4_t {
94 - scalar_t x;
95 - scalar_t y;
96 - scalar_t z;
97 - scalar_t w;
98 -};
99 -
100 -typedef struct __align__(4) {
101 - FP8_TYPE x;
102 - FP8_TYPE y;
103 - FP8_TYPE z;
104 - FP8_TYPE w;
105 -}
106 -float8x4_t;
107 -
96 template <typename scalar_t>
97 __device__ float thread_max_vec(scalar_t const* __restrict__ input,
98 int64_t const num_elems, int const tid,
@@ -139,10 +127,10 @@ __device__ void scaled_fp8_conversion_vec(FP8_TYPE* __restrict__ out,
127 float const scale,
128 int64_t const num_elems,
129 int const tid, int const step) {
130 + using float8x4_t = q8x4_t<FP8_TYPE>;
131 // Vectorized input/output to better utilize memory bandwidth.
143 - vec4_t<scalar_t> const* vectorized_in =
144 - reinterpret_cast<vec4_t<scalar_t> const*>(input);
145 - float8x4_t* vectorized_out = reinterpret_cast<float8x4_t*>(out);
132 + auto const* vectorized_in = reinterpret_cast<vec4_t<scalar_t> const*>(input);
133 + auto* vectorized_out = reinterpret_cast<float8x4_t*>(out);
134
135 int64_t const num_vec_elems = num_elems >> 2;
136
csrc/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu new
+160
@@ -0,0 +1,160 @@
1 +
2 +#include <ATen/cuda/CUDAContext.h>
3 +#include <c10/cuda/CUDAGuard.h>
4 +
5 +#include "../../dispatch_utils.h"
6 +#include "layernorm_utils.cuh"
7 +#include "quant_conversions.cuh"
8 +
9 +namespace vllm {
10 +
11 +template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
12 +__device__ void rms_norm_dynamic_per_token_quant_vec(
13 + scalar_out_t* __restrict__ out, // [..., hidden_size]
14 + float* __restrict__ scales, // [num_tokens]
15 + scalar_t const* __restrict__ input, // [..., hidden_size]
16 + scalar_t const* __restrict__ weight, // [hidden_size]
17 + float const* scale_ub, float const var_epsilon,
18 + float const min_scaling_factor, int32_t const hidden_size,
19 + scalar_t* __restrict__ residual = nullptr) {
20 + float rms = 0.0f;
21 + float token_scale = 0.0f;
22 +
23 + // Compute rms
24 + vllm::vectorized::compute_rms<scalar_t, has_residual>(
25 + &rms, input, hidden_size, var_epsilon, residual);
26 +
27 + // Compute scale
28 + vllm::vectorized::compute_dynamic_per_token_scales<scalar_t, scalar_out_t,
29 + has_residual>(
30 + &token_scale, scales, input, weight, rms, scale_ub, min_scaling_factor,
31 + hidden_size, residual);
32 +
33 + // RMS Norm + Quant
34 + if constexpr (std::is_same_v<scalar_out_t, int8_t>) {
35 + vllm::vectorized::norm_and_quant<scalar_t, scalar_out_t, true,
36 + has_residual>(
37 + out, input, weight, rms, 1.0f / token_scale, hidden_size, residual);
38 + } else {
39 + // FP8 - Do not invert token_scale for exact match with FBGemm
40 + vllm::vectorized::norm_and_quant<scalar_t, scalar_out_t, false,
41 + has_residual>(
42 + out, input, weight, rms, token_scale, hidden_size, residual);
43 + }
44 +}
45 +
46 +// RMS norm + quant kernel
47 +template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
48 +__global__ void rms_norm_dynamic_per_token_quant_kernel(
49 + scalar_out_t* __restrict__ out, // [..., hidden_size]
50 + float* __restrict__ scales, // [num_tokens]
51 + scalar_t const* __restrict__ input, // [..., hidden_size]
52 + scalar_t const* __restrict__ weight, // [hidden_size]
53 + float const* scale_ub, float const var_epsilon,
54 + float const min_scaling_factor, int32_t const hidden_size,
55 + scalar_t* __restrict__ residual = nullptr) {
56 + // For vectorization, token_input and token_output pointers need to be
57 + // aligned at 8-byte and 4-byte addresses respectively.
58 + bool const can_vectorize = hidden_size % 4 == 0;
59 +
60 + if (can_vectorize) {
61 + return rms_norm_dynamic_per_token_quant_vec<scalar_t, scalar_out_t,
62 + has_residual>(
63 + out, scales, input, weight, scale_ub, var_epsilon, min_scaling_factor,
64 + hidden_size, residual);
65 + }
66 +
67 + float rms = 0.0f;
68 + float token_scale = 0.0f;
69 +
70 + // Compute RMS
71 + vllm::compute_rms<scalar_t, has_residual>(&rms, input, hidden_size,
72 + var_epsilon, residual);
73 + // Compute Scale
74 + vllm::compute_dynamic_per_token_scales<scalar_t, scalar_out_t, has_residual>(
75 + &token_scale, scales, input, weight, rms, scale_ub, min_scaling_factor,
76 + hidden_size, residual);
77 +
78 + // RMS Norm + Quant
79 + if constexpr (std::is_same_v<scalar_out_t, int8_t>) {
80 + vllm::norm_and_quant<scalar_t, scalar_out_t, true, has_residual>(
81 + out, input, weight, rms, 1.0f / token_scale, hidden_size, residual);
82 + } else {
83 + // FP8 - Do not invert s_token_scale for exact match with FBGemm
84 + vllm::norm_and_quant<scalar_t, scalar_out_t, false, has_residual>(
85 + out, input, weight, rms, token_scale, hidden_size, residual);
86 + }
87 +}
88 +} // namespace vllm
89 +
90 +// Residual add + RMS norm + dynamic per token
91 +template <typename scalar_in_t>
92 +void rms_norm_dynamic_per_token_quant_dispatch(
93 + torch::Tensor& out, // [..., hidden_size]
94 + torch::Tensor const& input, // [..., hidden_size]
95 + torch::Tensor const& weight, // [hidden_size]
96 + torch::Tensor& scales, // [num_tokens]
97 + double const var_epsilon, // Variance epsilon used in norm calculation
98 + std::optional<at::Tensor> const& scale_ub,
99 + std::optional<at::Tensor>& residual) {
100 + int32_t hidden_size = input.size(-1);
101 + int32_t num_tokens = input.numel() / hidden_size;
102 +
103 + dim3 grid(num_tokens);
104 + dim3 block(std::min(hidden_size, 1024));
105 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
106 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
107 +
108 + const float min_scaling_factor =
109 + out.dtype() == torch::kInt8
110 + ? std::numeric_limits<float>::epsilon()
111 + : 1.0f / (std::numeric_limits<c10::Float8_e4m3fn>::max() * 512.f);
112 +
113 + if (residual.has_value()) {
114 + VLLM_DISPATCH_QUANT_TYPES(
115 + out.scalar_type(), "rms_norm_dynamic_per_token_quant_kernel", [&] {
116 + vllm::rms_norm_dynamic_per_token_quant_kernel<scalar_in_t, scalar_t,
117 + true>
118 + <<<grid, block, 0, stream>>>(
119 + out.data_ptr<scalar_t>(), scales.data_ptr<float>(),
120 + input.data_ptr<scalar_in_t>(), weight.data_ptr<scalar_in_t>(),
121 + scale_ub.has_value() ? scale_ub->data_ptr<float>() : nullptr,
122 + var_epsilon, min_scaling_factor, hidden_size,
123 + residual->data_ptr<scalar_in_t>());
124 + });
125 +
126 + } else {
127 + VLLM_DISPATCH_QUANT_TYPES(
128 + out.scalar_type(), "rms_norm_dynamic_per_token_quant_kernel", [&] {
129 + vllm::rms_norm_dynamic_per_token_quant_kernel<scalar_in_t, scalar_t,
130 + false>
131 + <<<grid, block, 0, stream>>>(
132 + out.data_ptr<scalar_t>(), scales.data_ptr<float>(),
133 + input.data_ptr<scalar_in_t>(), weight.data_ptr<scalar_in_t>(),
134 + scale_ub.has_value() ? scale_ub->data_ptr<float>() : nullptr,
135 + var_epsilon, min_scaling_factor, hidden_size, nullptr);
136 + });
137 + }
138 +}
139 +
140 +void rms_norm_dynamic_per_token_quant(
141 + torch::Tensor& out, // [..., hidden_size]
142 + torch::Tensor const& input, // [..., hidden_size]
143 + torch::Tensor const& weight, // [hidden_size]
144 + torch::Tensor& scales, // [num_tokens]
145 + double const var_epsilon, // Variance epsilon used in norm calculation
146 + std::optional<at::Tensor> scale_ub, std::optional<at::Tensor> residual) {
147 + TORCH_CHECK(out.dtype() == kFp8Type || out.dtype() == torch::kInt8);
148 + TORCH_CHECK(out.is_contiguous() && input.is_contiguous());
149 +
150 + if (scale_ub.has_value()) {
151 + TORCH_CHECK(out.dtype() == kFp8Type);
152 + }
153 + TORCH_CHECK(scales.dtype() == torch::kFloat32);
154 +
155 + VLLM_DISPATCH_FLOATING_TYPES(
156 + input.scalar_type(), "rms_norm_dynamic_per_token_quant_dispatch", [&] {
157 + rms_norm_dynamic_per_token_quant_dispatch<scalar_t>(
158 + out, input, weight, scales, var_epsilon, scale_ub, residual);
159 + });
160 +}
csrc/quantization/fused_kernels/layernorm_utils.cuh new
+327
@@ -0,0 +1,327 @@
1 +#pragma once
2 +
3 +/**
4 + * __device__ layernorm utilities.
5 + */
6 +
7 +#include "quantization/vectorization.cuh"
8 +#include "quant_conversions.cuh"
9 +
10 +#ifndef USE_ROCM
11 + #include <cub/cub.cuh>
12 +#else
13 + #include <hipcub/hipcub.hpp>
14 +#endif
15 +
16 +namespace vllm {
17 +
18 +// has_residual must be true, if residual is not a nullptr
19 +template <typename scalar_t, bool has_residual = false>
20 +__device__ void compute_rms(float* rms, scalar_t const* __restrict__ input,
21 + int32_t const hidden_size, float const epsilon,
22 + scalar_t const* __restrict__ residual = nullptr) {
23 + int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
24 + // sum of squares
25 + float ss = 0.0f;
26 +
27 + for (int32_t i = threadIdx.x; i < hidden_size; i += blockDim.x) {
28 + float x = static_cast<float>(input[token_offset + i]);
29 + if constexpr (has_residual) {
30 + x += static_cast<float>(residual[token_offset + i]);
31 + }
32 +
33 + ss += x * x;
34 + }
35 +
36 + using BlockReduce = cub::BlockReduce<float, 1024>;
37 + __shared__ typename BlockReduce::TempStorage reduceStore;
38 + ss = BlockReduce(reduceStore).Reduce(ss, cub::Sum{}, blockDim.x);
39 +
40 + __shared__ float s_rms;
41 + if (threadIdx.x == 0) {
42 + s_rms = rsqrtf(ss / hidden_size + epsilon);
43 + }
44 + __syncthreads();
45 +
46 + *rms = s_rms;
47 +}
48 +
49 +template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
50 +__device__ void compute_dynamic_per_token_scales(
51 + float* __restrict__ token_scale, float* __restrict__ all_token_scales,
52 + scalar_t const* __restrict__ input, scalar_t const* __restrict__ weight,
53 + float const rms, float const* __restrict__ scale_ub,
54 + float const min_scaling_factor, int32_t const hidden_size,
55 + scalar_t const* __restrict__ residual = nullptr) {
56 + int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
57 + ;
58 + constexpr scalar_out_t qmax{std::numeric_limits<scalar_out_t>::max()};
59 +
60 + float block_absmax_val_maybe = 0.0f;
61 + for (int32_t i = threadIdx.x; i < hidden_size; i += blockDim.x) {
62 + float x = static_cast<float>(input[token_offset + i]);
63 + if constexpr (has_residual) {
64 + x += static_cast<float>(residual[token_offset + i]);
65 + }
66 +
67 + x = static_cast<float>(static_cast<scalar_t>(x * rms) * weight[i]);
68 + block_absmax_val_maybe = fmaxf(block_absmax_val_maybe, fabsf(x));
69 + }
70 +
71 + using BlockReduce = cub::BlockReduce<float, 1024>;
72 + __shared__ typename BlockReduce::TempStorage reduceStore;
73 + block_absmax_val_maybe =
74 + BlockReduce(reduceStore)
75 + .Reduce(block_absmax_val_maybe, cub::Max{}, blockDim.x);
76 +
77 + __shared__ float s_token_scale;
78 + if (threadIdx.x == 0) {
79 + float scale = 0.0f;
80 + if (scale_ub) {
81 + scale = min(block_absmax_val_maybe, *scale_ub);
82 + } else {
83 + scale = block_absmax_val_maybe;
84 + }
85 + // token scale computation
86 + scale = max(scale / qmax, min_scaling_factor);
87 + s_token_scale = scale; // Shared memory store
88 + all_token_scales[blockIdx.x] = scale; // Global output store
89 + }
90 + __syncthreads();
91 +
92 + *token_scale = s_token_scale;
93 +}
94 +
95 +template <typename scalar_t, typename scalar_out_t, bool is_scale_inverted,
96 + bool has_residual = false>
97 +__device__ void norm_and_quant(scalar_out_t* __restrict__ output,
98 + scalar_t const* __restrict__ input,
99 + scalar_t const* __restrict__ weight,
100 + float const rms, float const scale,
101 + int32_t const hidden_size,
102 + scalar_t* __restrict__ residual = nullptr) {
103 + int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
104 + ;
105 +
106 + for (int32_t i = threadIdx.x; i < hidden_size; i += blockDim.x) {
107 + float x = static_cast<float>(input[token_offset + i]);
108 + if constexpr (has_residual) {
109 + x += static_cast<float>(residual[token_offset + i]);
110 + residual[token_offset + i] = static_cast<scalar_t>(x);
111 + }
112 + // Norm
113 + x = static_cast<float>(static_cast<scalar_t>(x * rms) * weight[i]);
114 + // Quant
115 + output[token_offset + i] =
116 + ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(x, scale);
117 + }
118 +}
119 +
120 +namespace vectorized {
121 +
122 +// Compute 1.0/rms(input)
123 +// hidden_size must be a multiple of 4
124 +template <typename scalar_t, bool has_residual = false>
125 +__device__ void compute_rms(float* rms, scalar_t const* __restrict__ input,
126 + int32_t const hidden_size, float const epsilon,
127 + scalar_t const* __restrict__ residual = nullptr) {
128 + int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
129 +
130 + // Vectorized input/output to better utilize memory bandwidth.
131 + vec4_t<scalar_t> const* vec_input =
132 + reinterpret_cast<vec4_t<scalar_t> const*>(&input[token_offset]);
133 + vec4_t<scalar_t> const* vec_residual = nullptr;
134 + if constexpr (has_residual) {
135 + vec_residual =
136 + reinterpret_cast<vec4_t<scalar_t> const*>(&residual[token_offset]);
137 + }
138 +
139 + // sum of squares
140 + float ss = 0.0f;
141 +
142 + int32_t const num_vec_elems = hidden_size >> 2;
143 +
144 +#pragma unroll 4
145 + for (int32_t i = threadIdx.x; i < num_vec_elems; i += blockDim.x) {
146 + vec4_t<scalar_t> in = vec_input[i];
147 +
148 + vec4_t<float> x;
149 + x.x = static_cast<float>(in.x);
150 + x.y = static_cast<float>(in.y);
151 + x.z = static_cast<float>(in.z);
152 + x.w = static_cast<float>(in.w);
153 + if constexpr (has_residual) {
154 + vec4_t<scalar_t> r = vec_residual[i];
155 + x.x += static_cast<float>(r.x);
156 + x.y += static_cast<float>(r.y);
157 + x.z += static_cast<float>(r.z);
158 + x.w += static_cast<float>(r.w);
159 + }
160 +
161 + ss += x.x * x.x;
162 + ss += x.y * x.y;
163 + ss += x.z * x.z;
164 + ss += x.w * x.w;
165 + }
166 +
167 + using BlockReduce = cub::BlockReduce<float, 1024>;
168 + __shared__ typename BlockReduce::TempStorage reduceStore;
169 + ss = BlockReduce(reduceStore).Reduce(ss, cub::Sum{}, blockDim.x);
170 +
171 + __shared__ float s_rms;
172 + if (threadIdx.x == 0) {
173 + s_rms = rsqrtf(ss / hidden_size + epsilon);
174 + }
175 + __syncthreads();
176 +
177 + *rms = s_rms;
178 +}
179 +
180 +// Vectorized version of vllm::compute_dynamic_per_token_scales
181 +// hidden_size must be a multiple of 4
182 +template <typename scalar_t, typename scalar_out_t, bool has_residual = false>
183 +__device__ void compute_dynamic_per_token_scales(
184 + float* __restrict__ token_scale, float* __restrict__ all_token_scales,
185 + scalar_t const* __restrict__ input, scalar_t const* __restrict__ weight,
186 + float const rms, float const* __restrict__ scale_ub,
187 + float const min_scaling_factor, int32_t const hidden_size,
188 + scalar_t const* __restrict__ residual = nullptr) {
189 + int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
190 + ;
191 +
192 + // Vectorized input/weight/residual to better utilize memory bandwidth.
193 + vec4_t<scalar_t> const* vec_input =
194 + reinterpret_cast<vec4_t<scalar_t> const*>(&input[token_offset]);
195 + vec4_t<scalar_t> const* vec_weight =
196 + reinterpret_cast<vec4_t<scalar_t> const*>(weight);
197 + vec4_t<scalar_t> const* vec_residual = nullptr;
198 + if constexpr (has_residual) {
199 + vec_residual =
200 + reinterpret_cast<vec4_t<scalar_t> const*>(&residual[token_offset]);
201 + }
202 +
203 + constexpr scalar_out_t qmax{std::numeric_limits<scalar_out_t>::max()};
204 +
205 + int32_t const num_vec_elems = hidden_size >> 2;
206 + float block_absmax_val_maybe = 0.0f;
207 +
208 +#pragma unroll 4
209 + for (int32_t i = threadIdx.x; i < num_vec_elems; i += blockDim.x) {
210 + vec4_t<scalar_t> in = vec_input[i];
211 + vec4_t<scalar_t> const w = vec_weight[i];
212 +
213 + vec4_t<float> x;
214 + x.x = static_cast<float>(in.x);
215 + x.y = static_cast<float>(in.y);
216 + x.z = static_cast<float>(in.z);
217 + x.w = static_cast<float>(in.w);
218 + if constexpr (has_residual) {
219 + vec4_t<scalar_t> r = vec_residual[i];
220 + x.x += static_cast<float>(r.x);
221 + x.y += static_cast<float>(r.y);
222 + x.z += static_cast<float>(r.z);
223 + x.w += static_cast<float>(r.w);
224 + }
225 +
226 + block_absmax_val_maybe = fmaxf(
227 + block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.x * rms) * w.x));
228 + block_absmax_val_maybe = fmaxf(
229 + block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.y * rms) * w.y));
230 + block_absmax_val_maybe = fmaxf(
231 + block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.z * rms) * w.z));
232 + block_absmax_val_maybe = fmaxf(
233 + block_absmax_val_maybe, fabs(static_cast<scalar_t>(x.w * rms) * w.w));
234 + }
235 +
236 + using BlockReduce = cub::BlockReduce<float, 1024>;
237 + __shared__ typename BlockReduce::TempStorage reduceStore;
238 + block_absmax_val_maybe =
239 + BlockReduce(reduceStore)
240 + .Reduce(block_absmax_val_maybe, cub::Max{}, blockDim.x);
241 +
242 + __shared__ float s_token_scale;
243 + if (threadIdx.x == 0) {
244 + float scale = 0.0f;
245 + if (scale_ub) {
246 + scale = min(block_absmax_val_maybe, *scale_ub);
247 + } else {
248 + scale = block_absmax_val_maybe;
249 + }
250 + // token scale computation
251 + scale = max(scale / qmax, min_scaling_factor);
252 + s_token_scale = scale; // shared memory store
253 + all_token_scales[blockIdx.x] = scale; // global output store
254 + }
255 + __syncthreads();
256 +
257 + *token_scale = s_token_scale;
258 +}
259 +
260 +// hidden_size must be a multiple of 4
261 +template <typename scalar_t, typename scalar_out_t, bool is_scale_inverted,
262 + bool has_residual = false>
263 +__device__ void norm_and_quant(scalar_out_t* __restrict__ output,
264 + scalar_t const* __restrict__ input,
265 + scalar_t const* __restrict__ weight,
266 + float const rms, float const scale,
267 + int32_t const hidden_size,
268 + scalar_t* __restrict__ residual = nullptr) {
269 + int64_t const token_offset = blockIdx.x * static_cast<int64_t>(hidden_size);
270 + ;
271 +
272 + // Vectorized input/output/weight/residual to better utilize memory bandwidth.
273 + vec4_t<scalar_t> const* vec_input =
274 + reinterpret_cast<vec4_t<scalar_t> const*>(&input[token_offset]);
275 + vec4_t<scalar_t> const* vec_weight =
276 + reinterpret_cast<vec4_t<scalar_t> const*>(weight);
277 + q8x4_t<scalar_out_t>* vec_output =
278 + reinterpret_cast<q8x4_t<scalar_out_t>*>(&output[token_offset]);
279 + vec4_t<scalar_t>* vec_residual = nullptr;
280 + if constexpr (has_residual) {
281 + vec_residual = reinterpret_cast<vec4_t<scalar_t>*>(&residual[token_offset]);
282 + }
283 +
284 + int32_t const num_vec_elems = hidden_size >> 2;
285 +
286 +// TODO(luka/varun) extract into type-agnostic vectorized quant function to
287 +// replace scaled_fp8_conversion_vec
288 +#pragma unroll 4
289 + for (int32_t i = threadIdx.x; i < num_vec_elems; i += blockDim.x) {
290 + vec4_t<scalar_t> const in = vec_input[i];
291 + vec4_t<scalar_t> const w = vec_weight[i];
292 +
293 + vec4_t<float> x;
294 + x.x = static_cast<float>(in.x);
295 + x.y = static_cast<float>(in.y);
296 + x.z = static_cast<float>(in.z);
297 + x.w = static_cast<float>(in.w);
298 + if constexpr (has_residual) {
299 + vec4_t<scalar_t> r = vec_residual[i];
300 + x.x += static_cast<float>(r.x);
301 + x.y += static_cast<float>(r.y);
302 + x.z += static_cast<float>(r.z);
303 + x.w += static_cast<float>(r.w);
304 + // Update residual
305 + r.x = static_cast<scalar_t>(x.x);
306 + r.y = static_cast<scalar_t>(x.y);
307 + r.z = static_cast<scalar_t>(x.z);
308 + r.w = static_cast<scalar_t>(x.w);
309 + vec_residual[i] = r;
310 + }
311 +
312 + q8x4_t<scalar_out_t> out;
313 + out.x = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
314 + static_cast<scalar_t>(x.x * rms) * w.x, scale);
315 + out.y = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
316 + static_cast<scalar_t>(x.y * rms) * w.y, scale);
317 + out.z = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
318 + static_cast<scalar_t>(x.z * rms) * w.z, scale);
319 + out.w = ScaledQuant<scalar_out_t, is_scale_inverted>::quant_fn(
320 + static_cast<scalar_t>(x.w * rms) * w.w, scale);
321 + vec_output[i] = out;
322 + }
323 +}
324 +
325 +} // namespace vectorized
326 +
327 +} // namespace vllm
csrc/quantization/fused_kernels/quant_conversions.cuh new
+81
@@ -0,0 +1,81 @@
1 +#pragma once
2 +
3 +/**
4 + * __device__ helper functions to deal with float -> quant datatype conversion
5 + */
6 +
7 +#include "quantization/vectorization.cuh"
8 +// TODO(luka/varun):refactor common.cuh to use this file instead
9 +#include "quantization/fp8/common.cuh"
10 +
11 +namespace vllm {
12 +
13 +// TODO(luka/varun): combine into common utilities for int8
14 +// (with int8_quant_kernels.cu)
15 +static __device__ __forceinline__ int8_t float_to_int8_rn(float const x) {
16 +#ifdef USE_ROCM
17 + static const float i8_min =
18 + static_cast<float>(std::numeric_limits<int8_t>::min());
19 + static const float i8_max =
20 + static_cast<float>(std::numeric_limits<int8_t>::max());
21 + // round
22 + float dst = std::nearbyint(x);
23 + // saturate
24 + dst = std::clamp(dst, i8_min, i8_max);
25 + return static_cast<int8_t>(dst);
26 +#else
27 + // CUDA path
28 + uint32_t dst;
29 + asm volatile("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(dst) : "f"(x));
30 + return reinterpret_cast<const int8_t&>(dst);
31 +#endif
32 +}
33 +
34 +static __device__ __forceinline__ FP8_TYPE float_to_fp8(float const x) {
35 + float const r = fmax(-FP8_E4M3_MAX, fmin(x, FP8_E4M3_MAX));
36 + return static_cast<FP8_TYPE>(r);
37 +}
38 +
39 +template <typename quant_type_t, bool is_scale_inverted, typename enable = void>
40 +struct ScaledQuant;
41 +
42 +template <typename quant_type_t, bool is_scale_inverted>
43 +struct ScaledQuant<
44 + quant_type_t, is_scale_inverted,
45 + typename std::enable_if_t<std::is_same_v<quant_type_t, int8_t>>> {
46 + static __device__ __forceinline__ quant_type_t quant_fn(float const x,
47 + float const scale) {
48 + if constexpr (is_scale_inverted) {
49 + return float_to_int8_rn(x * scale);
50 + } else {
51 + return float_to_int8_rn(x / scale);
52 + }
53 + }
54 +};
55 +
56 +template <typename quant_type_t, bool is_scale_inverted>
57 +struct ScaledQuant<
58 + quant_type_t, is_scale_inverted,
59 + typename std::enable_if_t<std::is_same_v<quant_type_t, FP8_TYPE>>> {
60 + static __device__ __forceinline__ quant_type_t quant_fn(float const x,
61 + float const scale) {
62 + if constexpr (is_scale_inverted) {
63 + return float_to_fp8(x * scale);
64 + } else {
65 + return float_to_fp8(x / scale);
66 + }
67 + }
68 +};
69 +
70 +template <typename scalar_t, typename quant_type_t, bool is_scale_inverted>
71 +__device__ void scaled_quant_conversion(quant_type_t* __restrict__ output,
72 + scalar_t const* __restrict__ input,
73 + float const scale, int const tid,
74 + int const num_elements,
75 + int const step) {
76 + for (int i = tid; i < num_elements; i += step) {
77 + output[i] = ScaledQuant<quant_type_t, is_scale_inverted>(input[i], scale);
78 + }
79 +}
80 +
81 +} // namespace vllm
csrc/quantization/vectorization.cuh new
+33
@@ -0,0 +1,33 @@
1 +#pragma once
2 +/**
3 + * __device__ datatypes vectorized by 4
4 + */
5 +
6 +// Include both AMD and NVIDIA fp8 types to avoid circular import
7 +// TODO(luka/varun) use FP8_TYPE instead after refactoring
8 +#include <c10/util/Float8_e4m3fnuz.h>
9 +#include <c10/util/Float8_e4m3fn.h>
10 +
11 +namespace vllm {
12 +
13 +// Vectorization containers
14 +template <typename scalar_t>
15 +struct __align__(8) vec4_t {
16 + scalar_t x;
17 + scalar_t y;
18 + scalar_t z;
19 + scalar_t w;
20 +};
21 +
22 +template <typename quant_type_t>
23 +struct __align__(4) q8x4_t {
24 + static_assert(std::is_same_v<quant_type_t, int8_t> ||
25 + std::is_same_v<quant_type_t, c10::Float8_e4m3fn> ||
26 + std::is_same_v<quant_type_t, c10::Float8_e4m3fnuz>);
27 + quant_type_t x;
28 + quant_type_t y;
29 + quant_type_t z;
30 + quant_type_t w;
31 +};
32 +
33 +} // namespace vllm
csrc/torch_bindings.cpp
+8
@@ -128,6 +128,14 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
128 ops.impl("fused_add_rms_norm_static_fp8_quant", torch::kCUDA,
129 &fused_add_rms_norm_static_fp8_quant);
130
131 + // Fused Layernorm + Quant kernels
132 + ops.def(
133 + "rms_norm_dynamic_per_token_quant(Tensor! result, Tensor input, "
134 + "Tensor weight, Tensor! scale, float epsilon, "
135 + "Tensor? scale_ub, Tensor!? residual) -> ()");
136 + ops.impl("rms_norm_dynamic_per_token_quant", torch::kCUDA,
137 + &rms_norm_dynamic_per_token_quant);
138 +
139 // Rotary embedding
140 // Apply GPT-NeoX or GPT-J style rotary embedding to query and key.
141 ops.def(
tests/compile/test_functionalization.py
+13 -8
@@ -4,10 +4,10 @@ import torch
4 import vllm.envs as envs
5 from vllm import LLM, SamplingParams
6 from vllm.compilation.fix_functionalization import FixFunctionalizationPass
7 -from vllm.compilation.fusion import (FusionPass, find_auto_fn,
8 - find_auto_fn_maybe)
7 +from vllm.compilation.fusion import (FUSED_OPS, FusionPass, QuantKey,
8 + kFp8DynamicTokenSym, kFp8StaticTensorSym)
9 +from vllm.compilation.fx_utils import find_auto_fn, find_auto_fn_maybe, is_func
10 from vllm.compilation.reshapes import RedundantReshapesPass
10 -from vllm.compilation.vllm_inductor_pass import is_func
11 from vllm.config import CompilationConfig
12
13 from .backend import TestBackend
@@ -35,12 +35,16 @@ prompts = [
35 ]
36
37
38 -@pytest.mark.parametrize("model",
39 - ["nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8-e2e"])
38 +@pytest.mark.parametrize(
39 + "model, quant_key",
40 + [("nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8-e2e", kFp8StaticTensorSym),
41 + ("nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8_DYNAMIC-e2e",
42 + kFp8DynamicTokenSym)])
43 @pytest.mark.parametrize("do_fusion", [True, False])
44 @pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
45 reason="Only test on CUDA")
43 -def test_fix_functionalization(model: str, do_fusion: bool):
46 +def test_fix_functionalization(model: str, quant_key: QuantKey,
47 + do_fusion: bool):
48 torch.set_default_device("cuda")
49
50 config = CompilationConfig.PassConfig(enable_fusion=do_fusion,
@@ -78,8 +82,9 @@ def test_fix_functionalization(model: str, do_fusion: bool):
82
83 # OPS_IN_MODEL always appear. RMS_OP is fused away if we run fusion,
84 # and replaced by fused quantized ops in RMS_QUANT_OPS.
81 - ops = OPS_IN_MODEL + (RMS_QUANT_OPS["static_fp8"]
82 - if do_fusion else [RMS_OP])
85 + rms_ops = [FUSED_OPS[(quant_key, True)], FUSED_OPS[(quant_key, False)]
86 + ] if do_fusion else [RMS_OP]
87 + ops = OPS_IN_MODEL + rms_ops
88
89 for op in ops:
90 find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
tests/compile/test_fusion.py
+43 -18
@@ -3,8 +3,9 @@ import torch
3 from compressed_tensors.quantization import FP8_DTYPE
4
5 import vllm.envs as envs
6 -from vllm.compilation.fusion import (FusionPass, find_auto_fn,
7 - find_auto_fn_maybe)
6 +from vllm.compilation.fusion import (FUSED_OPS, QUANT_OPS, FusedRMSQuantKey,
7 + FusionPass, QuantKey)
8 +from vllm.compilation.fx_utils import find_auto_fn, find_auto_fn_maybe
9 from vllm.compilation.reshapes import RedundantReshapesPass
10 from vllm.config import CompilationConfig
11 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -16,24 +17,37 @@ from .backend import TestBackend
17
18 class TestModel(torch.nn.Module):
19
19 - def __init__(self, hidden_size: int, eps: float, *args, **kwargs):
20 + def __init__(self, hidden_size: int, eps: float, static: bool, *args,
21 + **kwargs):
22 super().__init__(*args, **kwargs)
23 self.norm = [RMSNorm(hidden_size, eps) for _ in range(3)]
22 - self.scale = [torch.rand(1, dtype=torch.float32) for _ in range(4)]
24 + self.wscale = [torch.rand(1, dtype=torch.float32) for _ in range(2)]
25 + if static:
26 + self.scale = [torch.rand(1, dtype=torch.float32) for _ in range(2)]
27 + else:
28 + self.scale = [None for _ in range(2)]
29 self.w = [
30 torch.rand(hidden_size, hidden_size).to(dtype=FP8_DTYPE).t()
31 for _ in range(2)
32 ]
33
34 def forward(self, x):
29 - resid = torch.relu(x)
35 + resid = torch.sqrt(x)
36 y = self.norm[0](x)
37
32 - x2 = apply_fp8_linear(y, self.w[0], self.scale[0], self.scale[1])
38 + x2 = apply_fp8_linear(y,
39 + self.w[0],
40 + self.wscale[0],
41 + self.scale[0],
42 + use_per_token_if_dynamic=True)
43 # make sure resid is used for replacement to work
44 y2, resid = self.norm[1](x2, resid)
45
36 - x3 = apply_fp8_linear(y2, self.w[1], self.scale[2], self.scale[3])
46 + x3 = apply_fp8_linear(y2,
47 + self.w[1],
48 + self.wscale[1],
49 + self.scale[1],
50 + use_per_token_if_dynamic=True)
51 y3, resid = self.norm[2](x3, resid) # use resid here
52 return y3
53
@@ -42,14 +56,13 @@ class TestModel(torch.nn.Module):
56 @pytest.mark.parametrize("hidden_size", [64, 3392, 4096])
57 @pytest.mark.parametrize("num_tokens", [7, 256, 533, 2048, 2049])
58 @pytest.mark.parametrize("eps", [1e-5, 1e-6])
59 +@pytest.mark.parametrize("static", [True, False])
60 @pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
61 reason="Only test on CUDA")
47 -def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
62 +def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps, static):
63 torch.set_default_device("cuda")
49 - torch.set_default_dtype(torch.float16)
50 -
51 - if eps != 1e-5:
52 - pytest.skip("Only test eps=1e-5 for now")
64 + torch.set_default_dtype(dtype)
65 + torch.manual_seed(1)
66
67 # Reshape pass is needed for the fusion pass to work
68 config = CompilationConfig.PassConfig(enable_fusion=True,
@@ -58,7 +71,7 @@ def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
71 fusion_pass = FusionPass.instance(config)
72
73 backend = TestBackend(reshape_pass, fusion_pass)
61 - model = TestModel(hidden_size, eps)
74 + model = TestModel(hidden_size, eps, static)
75
76 # First dimension dynamic
77 x = torch.rand(num_tokens, hidden_size)
@@ -69,16 +82,28 @@ def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
82 model2 = torch.compile(model, backend=backend)
83 result2 = model2(x)
84
72 - # Check that it gives the same answer
73 - torch.testing.assert_close(result, result2, atol=1e-3, rtol=1e-3)
85 + # Higher tol for dynamic, even higher for bfloat16
86 + if static:
87 + ATOL, RTOL = (1e-3, 1e-3)
88 + elif dtype == torch.float16:
89 + ATOL, RTOL = (2e-3, 2e-3)
90 + else:
91 + ATOL, RTOL = (1e-2, 1e-2)
92 +
93 + torch.testing.assert_close(result, result2, atol=ATOL, rtol=RTOL)
94
95 # Check substitution worked
96 pre_nodes = backend.graph_pre_pass.nodes
97 post_nodes = backend.graph_post_pass.nodes
98
79 - rms_quant = torch.ops._C.rms_norm_static_fp8_quant.default
80 - add_rms_quant = torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
81 - fp8_quant = torch.ops._C.static_scaled_fp8_quant.default
99 + # static is per-tensor, dynamic is per-token
100 + key = QuantKey(dtype=FP8_DTYPE,
101 + static=static,
102 + per_tensor=static,
103 + symmetric=True)
104 + rms_quant = FUSED_OPS[FusedRMSQuantKey(key, False)]
105 + add_rms_quant = FUSED_OPS[FusedRMSQuantKey(key, True)]
106 + fp8_quant = QUANT_OPS[key]
107
108 # In pre-nodes, fp8 quant should be present and fused kernels should not
109 assert find_auto_fn_maybe(pre_nodes, rms_quant) is None
tests/kernels/test_fused_quant_layernorm.py new
+171
@@ -0,0 +1,171 @@
1 +from typing import Optional, Tuple, Union
2 +
3 +import pytest
4 +import torch
5 +
6 +import vllm._custom_ops as ops
7 +from tests.kernels.utils import opcheck
8 +from vllm.model_executor.layers.layernorm import RMSNorm
9 +
10 +DTYPES = [torch.bfloat16, torch.float]
11 +QUANT_DTYPES = [torch.int8, torch.float8_e4m3fn]
12 +VEC_HIDDEN_SIZES = range(1024, 1030)
13 +# Avoid combinatorial explosion with full Cartesian product
14 +NUM_TOKENS_HIDDEN_SIZES = [
15 + *[(1, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5120, 5137]],
16 + *[(83, i) for i in [1, 1033, 2048, 5120]],
17 + *[(2048, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5137]],
18 + *[(4096, i) for i in [1, 64, 5137]],
19 +]
20 +
21 +ADD_RESIDUAL = [False, True]
22 +SCALE_UBS = [True, False]
23 +SEEDS = [0]
24 +CUDA_DEVICES = [
25 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
26 +]
27 +
28 +EPS = 1e-6
29 +
30 +## Helpers
31 +
32 +
33 +def as_float32_tensor(x: Union[float, torch.tensor]) -> torch.tensor:
34 + return torch.as_tensor(x, dtype=torch.float32, device='cuda')
35 +
36 +
37 +def ref_rms_norm(rms_norm_layer: RMSNorm,
38 + x: torch.Tensor,
39 + residual: Optional[torch.Tensor]) \
40 + -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
41 + if residual is not None:
42 + residual = residual.clone()
43 + out, residual = rms_norm_layer.forward_native(x, residual)
44 + else:
45 + out = rms_norm_layer.forward_native(x)
46 +
47 + return out, residual
48 +
49 +
50 +def ref_dynamic_per_token_quant(rms_norm_layer: RMSNorm,
51 + x: torch.Tensor,
52 + quant_dtype: torch.dtype,
53 + residual: Optional[torch.Tensor],
54 + scale_ub: Optional[torch.Tensor]) \
55 + -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
56 + if scale_ub is not None:
57 + assert quant_dtype == torch.float8_e4m3fn
58 +
59 + # Norm
60 + torch_out, residual = ref_rms_norm(rms_norm_layer, x, residual)
61 +
62 + # Quant
63 + if quant_dtype == torch.float8_e4m3fn:
64 + torch_out, scales = ops.scaled_fp8_quant(torch_out,
65 + scale_ub=scale_ub,
66 + use_per_token_if_dynamic=True)
67 + else:
68 + assert quant_dtype == torch.int8
69 + torch_out, scales = ops.scaled_int8_quant(torch_out)
70 +
71 + return torch_out, scales, residual
72 +
73 +
74 +def ref_impl(rms_norm_layer: RMSNorm,
75 + x: torch.Tensor,
76 + quant_dtype: torch.dtype,
77 + residual: Optional[torch.Tensor],
78 + scale_ub: Optional[torch.Tensor]) \
79 + -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
80 + return ref_dynamic_per_token_quant(rms_norm_layer, x, quant_dtype,
81 + residual, scale_ub)
82 +
83 +
84 +def ops_dynamic_per_token_quant(weight: torch.Tensor,
85 + x: torch.Tensor,
86 + quant_dtype: torch.dtype,
87 + residual: Optional[torch.Tensor],
88 + scale_ub: Optional[torch.Tensor]) \
89 + -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
90 + if residual is not None:
91 + residual = residual.clone()
92 + out, scales = ops.rms_norm_dynamic_per_token_quant(x, weight, EPS,
93 + quant_dtype, scale_ub,
94 + residual)
95 + return out, scales, residual
96 +
97 +
98 +def ops_impl(weight: torch.Tensor,
99 + x: torch.Tensor,
100 + quant_dtype: torch.dtype,
101 + residual: Optional[torch.Tensor],
102 + scale_ub: Optional[torch.Tensor]) \
103 + -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
104 + return ops_dynamic_per_token_quant(weight, x, quant_dtype, residual,
105 + scale_ub)
106 +
107 +
108 +@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
109 +@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
110 +@pytest.mark.parametrize("scale_ub", SCALE_UBS)
111 +@pytest.mark.parametrize("dtype", DTYPES)
112 +@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
113 +@pytest.mark.parametrize("seed", SEEDS)
114 +@pytest.mark.parametrize("device", CUDA_DEVICES)
115 +@torch.inference_mode()
116 +def test_rms_norm(
117 + num_tokens: int,
118 + hidden_size: int,
119 + add_residual: bool,
120 + scale_ub: bool,
121 + dtype: torch.dtype,
122 + quant_dtype: torch.dtype,
123 + seed: int,
124 + device: str,
125 +) -> None:
126 + torch.random.manual_seed(seed)
127 + if torch.cuda.is_available():
128 + torch.cuda.manual_seed(seed)
129 + torch.set_default_device(device)
130 +
131 + if scale_ub is not None and quant_dtype != torch.float8_e4m3fn:
132 + # skip
133 + return
134 +
135 + layer = RMSNorm(hidden_size, EPS).to(dtype=dtype)
136 +
137 + # Make weights
138 + layer.weight.data.normal_(mean=1.0, std=0.1)
139 +
140 + # Make inputs
141 + scale = 1 / (hidden_size)
142 + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale
143 + residual = torch.randn_like(x) * scale if add_residual else None
144 + if scale_ub is not None:
145 + rms_x, _ = ref_rms_norm(layer, x, residual)
146 + scale_ub = torch.mean(rms_x).to(dtype=torch.float32, device='cuda')
147 +
148 + ref_out, ref_scales, ref_residual = \
149 + ref_impl(layer, x, quant_dtype, residual, scale_ub)
150 + ops_out, ops_scales, ops_residual = \
151 + ops_impl(layer.weight, x, quant_dtype, residual, scale_ub)
152 +
153 + assert ref_out.dtype == quant_dtype
154 + assert ops_out.dtype == quant_dtype
155 + assert torch.allclose(ref_scales, ops_scales)
156 + if quant_dtype == torch.int8:
157 + # big atol to account for round-off errors.
158 + assert torch.allclose(ref_out, ops_out, atol=1)
159 + else:
160 + assert torch.allclose(ref_out.to(dtype=torch.float32),
161 + ops_out.to(dtype=torch.float32))
162 + if add_residual:
163 + assert torch.allclose(ref_residual, ops_residual)
164 +
165 + output = torch.empty_like(x, dtype=quant_dtype)
166 + scales = torch.empty((x.numel() // x.shape[-1], 1),
167 + device=x.device,
168 + dtype=torch.float32)
169 +
170 + opcheck(torch.ops._C.rms_norm_dynamic_per_token_quant,
171 + (output, x, layer.weight, scales, 1e-5, scale_ub, residual))
vllm/_custom_ops.py
+20
@@ -249,6 +249,26 @@ def advance_step_flashinfer(num_seqs: int, num_queries: int, block_size: int,
249 block_table_bound)
250
251
252 +# fused quant layer norm ops
253 +def rms_norm_dynamic_per_token_quant(
254 + input: torch.Tensor,
255 + weight: torch.Tensor,
256 + epsilon: float,
257 + quant_dtype: torch.dtype,
258 + scale_ub: Optional[torch.Tensor] = None,
259 + residual: Optional[torch.Tensor] = None
260 +) -> Tuple[torch.Tensor, torch.Tensor]:
261 + output = torch.empty_like(input, dtype=quant_dtype)
262 + scales = torch.empty((input.numel() // input.shape[-1], 1),
263 + device=input.device,
264 + dtype=torch.float32)
265 +
266 + torch.ops._C.rms_norm_dynamic_per_token_quant(output, input, weight,
267 + scales, epsilon, scale_ub,
268 + residual)
269 + return output, scales
270 +
271 +
272 # quantization ops
273 # awq
274 def awq_dequantize(qweight: torch.Tensor, scales: torch.Tensor,
vllm/compilation/fix_functionalization.py
+6 -3
@@ -6,7 +6,8 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
7 from vllm.logger import init_logger
8
9 -from .vllm_inductor_pass import VllmInductorPass, is_func
9 +from .fx_utils import is_func
10 +from .vllm_inductor_pass import VllmInductorPass
11
12 logger = init_logger(__name__)
13
@@ -53,14 +54,16 @@ class FixFunctionalizationPass(VllmInductorPass):
54 self.insert_defunctionalized(graph, node)
55 self._remove(node)
56
56 - # These 2 replacements avoid the most copies for LLaMa.
57 + # rms_norm replacements avoid the most copies for LLaMa.
58 elif at_target == torch.ops._C.fused_add_rms_norm.default:
59 mutated_args = {1: 'input', 2: 'residual'}
60 self.defunctionalize(graph, node, mutated_args)
61 elif at_target == torch.ops._C.fused_add_rms_norm_static_fp8_quant.default: # noqa: E501
62 mutated_args = {1: 'result', 2: 'residual'}
63 self.defunctionalize(graph, node, mutated_args)
63 -
64 + elif at_target == torch.ops._C.rms_norm_dynamic_per_token_quant.default: # noqa: E501
65 + mutated_args = {1: 'result', 2: 'scale', 3: 'residual'}
66 + self.defunctionalize(graph, node, mutated_args)
67 elif at_target in [
68 torch.ops._C.rms_norm.default,
69 torch.ops._C.rms_norm_static_fp8_quant.default
vllm/compilation/fusion.py
+521 -198
@@ -1,129 +1,517 @@
1 -import operator
2 -from typing import Iterable, List, Optional
1 +from typing import Callable, Dict, List, NamedTuple, Optional, Tuple
2
3 import torch
4 +import torch._inductor.pattern_matcher as pm
5 +# TODO(luka) use vllm.utils once #10836 landed
6 +from compressed_tensors.quantization import FP8_DTYPE
7 +from torch import fx
8 from torch._higher_order_ops.auto_functionalize import auto_functionalized
6 -from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7 - fwd_only, register_replacement)
9 +from torch._inductor.pattern_matcher import PatternMatcherPass
10 +from torch._ops import OpOverload
11
12 from vllm.config import CompilationConfig
13 from vllm.logger import init_logger
14
12 -from .vllm_inductor_pass import VllmInductorPass, is_func
15 +from .fx_utils import find_getitem_maybe
16 +from .multi_output_match import MultiOutputMatch
17 +from .vllm_inductor_pass import VllmInductorPass
18
19 logger = init_logger(__name__)
20
21
17 -def rms_pattern_static(result: torch.Tensor, result_rms: torch.Tensor,
18 - input: torch.Tensor, weight: torch.Tensor,
19 - scale: torch.Tensor):
20 - at1 = auto_functionalized(torch.ops._C.rms_norm.default,
21 - result=result_rms,
22 - input=input,
23 - weight=weight,
24 - epsilon=1e-5)
25 - at2 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
26 - result=result,
27 - input=at1[1],
28 - scale=scale)
29 -
30 - # result
31 - return at2[1]
32 -
33 -
34 -def rms_replacement_static(result: torch.Tensor, result_rms: torch.Tensor,
35 - input: torch.Tensor, weight: torch.Tensor,
36 - scale: torch.Tensor):
37 - at = auto_functionalized(torch.ops._C.rms_norm_static_fp8_quant.default,
38 - result=result,
39 - input=input,
40 - weight=weight,
41 - scale=scale,
42 - epsilon=1e-5)
43 -
44 - # result
45 - return at[1]
46 -
47 -
48 -def rms_pattern_residual_static(result: torch.Tensor, input: torch.Tensor,
49 - residual: torch.Tensor, weight: torch.Tensor,
50 - scale: torch.Tensor):
51 - at = auto_functionalized(torch.ops._C.fused_add_rms_norm.default,
52 - input=input,
53 - residual=residual,
54 - weight=weight,
55 - epsilon=1e-5)
56 - at1 = auto_functionalized(torch.ops._C.static_scaled_fp8_quant.default,
57 - result=result,
58 - input=at[1],
59 - scale=scale)
60 -
61 - # result, residual
62 - return at1[1], at[2]
63 -
64 -
65 -def rms_replacement_residual_static(result: torch.Tensor, input: torch.Tensor,
66 - residual: torch.Tensor,
67 - weight: torch.Tensor, scale: torch.Tensor):
68 - at = auto_functionalized(
69 - torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
70 - result=result,
71 - input=input,
72 - residual=residual,
73 - weight=weight,
74 - scale=scale,
75 - epsilon=1e-5)
76 - # result, residual
77 - return at[1], at[2]
78 -
79 -
22 def empty_bf16(*args, **kwargs):
23 return torch.empty(*args, **kwargs, dtype=torch.bfloat16, device="cuda")
24
25
84 -def empty_fp8(*args, **kwargs):
85 - fp8 = torch.float8_e4m3fn
86 - return torch.empty(*args, **kwargs, dtype=fp8, device="cuda")
87 -
88 -
26 def empty_fp32(*args, **kwargs):
27 return torch.empty(*args, **kwargs, dtype=torch.float32, device="cuda")
28
29
93 -# Utilities for post-processing multi-output matches
30 +RMS_OP = torch.ops._C.rms_norm.default
31 +RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
32 +
33 +
34 +class QuantKey(NamedTuple):
35 + """
36 + Named tuple for identifying the type of quantization.
37 + dtype: quantized data type
38 + static: static quantization if True, dynamic if False
39 + per_tensor: per-tensor quantization if True, per-token if False
40 + symmetric: symmetric if True, asymmetric if False
41 + """
42 + dtype: torch.dtype
43 + static: bool
44 + per_tensor: bool = True
45 + symmetric: bool = True
46 +
47 + def __str__(self):
48 + return (f"QuantKey({'static' if self.static else 'dynamic'},"
49 + f"{fx.graph.dtype_abbrs[self.dtype]},"
50 + f"{'per_tensor' if self.per_tensor else 'per_token'},"
51 + f"{'a' if not self.symmetric else ''}symmetric)")
52 +
53 +
54 +kFp8StaticTensorSym = QuantKey(FP8_DTYPE, True, True, True)
55 +kFp8DynamicTensorSym = QuantKey(FP8_DTYPE, False, True, True)
56 +kFp8DynamicTokenSym = QuantKey(FP8_DTYPE, False, False, True)
57 +
58 +QUANT_OPS: Dict[QuantKey, OpOverload] = {
59 + kFp8StaticTensorSym: torch.ops._C.static_scaled_fp8_quant.default, # noqa
60 + kFp8DynamicTensorSym:
61 + torch.ops._C.dynamic_scaled_fp8_quant.default, # noqa
62 + kFp8DynamicTokenSym:
63 + torch.ops._C.dynamic_per_token_scaled_fp8_quant.default, # noqa
64 +}
65 +
66 +
67 +class FusedRMSQuantKey(NamedTuple):
68 + """
69 + Named tuple for identifying the type of RMSNorm + quant fusion.
70 + quant: type of quantization
71 + fused_add: does the op also perform the residual add
72 + """
73 + quant: QuantKey
74 + fused_add: bool
75 +
76 + def __str__(self):
77 + return (f"FusedQuantKey({self.quant}, with"
78 + f"{'' if self.fused_add else 'out'} residual)")
79 +
80 +
81 +FUSED_OPS: Dict[FusedRMSQuantKey, OpOverload] = {
82 + FusedRMSQuantKey(kFp8StaticTensorSym, False):
83 + torch.ops._C.rms_norm_static_fp8_quant.default, # noqa
84 + FusedRMSQuantKey(kFp8StaticTensorSym, True):
85 + torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, # noqa
86 + FusedRMSQuantKey(kFp8DynamicTokenSym, False):
87 + torch.ops._C.rms_norm_dynamic_per_token_quant.default, # noqa
88 + FusedRMSQuantKey(kFp8DynamicTokenSym, True):
89 + torch.ops._C.rms_norm_dynamic_per_token_quant.default, # noqa
90 +}
91 +
92 +
93 +class QuantMultiOutputMatch(MultiOutputMatch):
94 +
95 + def __init__(self, match: pm.Match, quant_op, fused_op):
96 + super().__init__(match)
97 + assert isinstance(quant_op, OpOverload)
98 + assert isinstance(fused_op, OpOverload)
99 + self.QUANT_OP = quant_op # in-place quant op
100 + self.FUSED_OP = fused_op # in-place fused quant op
101 +
102 + def insert_fused_node(self, fused_return_mapping: Dict[int, Tuple[fx.Node,
103 + int]],
104 + **kwargs):
105 + """
106 + This utility function inserts an auto-functionalized node for FUSED_OP.
107 + It also correctly sets its meta value and rebinds the users of the
108 + unfused nodes to use the fused node instead.
109 +
110 + :param fused_return_mapping: A dictionary, mapping from getitem indices
111 + of the fused node result to a tuple of the old node and a getitem index.
112 + :param kwargs: kwargs that get directly forwarded to the auto_fn node
113 +
114 + Example:
115 + If we want to replace this graph:
116 + _, x1, x2 = auto_fn(op1)
117 + _, y1, y2 = auto_fn(op2)
118 +
119 + with
120 + _, x1, y2, x2 = auto_fn(FUSED_OP)
121 +
122 + we would call:
123 + insert_fused_node({1: (op1_node, 1), 2: (op2_node, 2), 3: (op1_node, 2)}
124 +
125 + Note that the 0th element is None for auto-functionalized in-place ops.
126 + Hence, others appear 1-indexed.
127 + """
128 + fused_node = self.insert_auto_fn(self.FUSED_OP, kwargs)
129 + indices = fused_return_mapping.keys()
130 + getitem_nodes = self.insert_getitems(fused_node, indices)
131 +
132 + # Prepare the meta value, use a list so it's mutable
133 + meta_val = [None] * (max(indices) + 1)
134 +
135 + # Iterate through elements of the tuple produced by fused_node
136 + for idx, getitem_node in zip(indices, getitem_nodes):
137 + old_node, old_idx = fused_return_mapping[idx]
138 +
139 + # If the old value was never used, the old_getitem might not exist
140 + old_getitem = find_getitem_maybe(old_node, old_idx)
141 + if old_getitem is not None:
142 + # Rebind the users of match getitem nodes to use the new nodes.
143 + # The old nodes will be removed by DCE at the end of the pass.
144 + old_getitem.replace_all_uses_with(getitem_node)
145 + getitem_node.meta["val"] = old_getitem.meta["val"]
146 +
147 + # Extract the appropriate meta value
148 + # It is present even if the getitem node does not exist
149 + meta_val[idx] = old_node.meta["val"][old_idx]
150 +
151 + # Fix the meta value on the new fused node
152 + fused_node.meta["val"] = tuple(meta_val)
153 +
154 +
155 +class RMSNormQuantPattern:
156 +
157 + def __init__(self, epsilon: float, key: FusedRMSQuantKey):
158 + self.epsilon = epsilon
159 + self.quant_dtype = key.quant.dtype
160 +
161 + assert key.quant in QUANT_OPS, \
162 + f"unsupported quantization scheme {key.quant}"
163 + self.QUANT_OP = QUANT_OPS[key.quant]
164 +
165 + assert key in FUSED_OPS, \
166 + f"unsupported fused rmsnorm+quant op for {key}"
167 + self.FUSED_OP = FUSED_OPS[key]
168 +
169 +
170 +class RMSNormStaticQuantPattern(RMSNormQuantPattern):
171 +
172 + def __init__(self,
173 + epsilon: float,
174 + quant_dtype: torch.dtype,
175 + symmetric=True):
176 + fused_key = FusedRMSQuantKey(fused_add=False,
177 + quant=QuantKey(dtype=quant_dtype,
178 + static=True,
179 + per_tensor=True,
180 + symmetric=symmetric))
181 + super().__init__(epsilon, fused_key)
182 +
183 + def register(self, pm_pass: PatternMatcherPass):
184 + # Cannot use methods, as the self argument affects tracing
185 + def pattern(result: torch.Tensor, result_rms: torch.Tensor,
186 + input: torch.Tensor, weight: torch.Tensor,
187 + scale: torch.Tensor):
188 + at1 = auto_functionalized(RMS_OP,
189 + result=result_rms,
190 + input=input,
191 + weight=weight,
192 + epsilon=self.epsilon)
193 + at2 = auto_functionalized(self.QUANT_OP,
194 + result=result,
195 + input=at1[1],
196 + scale=scale)
197 +
198 + # result
199 + return at2[1]
200 +
201 + def replacement(result: torch.Tensor, result_rms: torch.Tensor,
202 + input: torch.Tensor, weight: torch.Tensor,
203 + scale: torch.Tensor):
204 + at = auto_functionalized(self.FUSED_OP,
205 + result=result,
206 + input=input,
207 + weight=weight,
208 + scale=scale,
209 + epsilon=self.epsilon)
210 +
211 + # result
212 + return at[1]
213 +
214 + inputs = [
215 + torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
216 + empty_bf16(5, 4), # result_rms
217 + empty_bf16(5, 4), # input
218 + empty_bf16(1, 5), # weight
219 + empty_fp32(1, 1) # scale
220 + ]
221 +
222 + pm.register_replacement(pattern, replacement, inputs, pm.fwd_only,
223 + pm_pass)
224 +
225 +
226 +class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
227 +
228 + def __init__(self,
229 + epsilon: float,
230 + quant_dtype: torch.dtype,
231 + symmetric=True):
232 + key = FusedRMSQuantKey(fused_add=True,
233 + quant=QuantKey(dtype=quant_dtype,
234 + static=True,
235 + per_tensor=True,
236 + symmetric=symmetric))
237 + super().__init__(epsilon, key)
238 +
239 + def register(self, pm_pass: PatternMatcherPass,
240 + record_match: Callable[[MultiOutputMatch], bool]):
241 +
242 + def pattern(result: torch.Tensor, input: torch.Tensor,
243 + residual: torch.Tensor, weight: torch.Tensor,
244 + scale: torch.Tensor):
245 + at = auto_functionalized(RMS_ADD_OP,
246 + input=input,
247 + residual=residual,
248 + weight=weight,
249 + epsilon=self.epsilon)
250 + at1 = auto_functionalized(self.QUANT_OP,
251 + result=result,
252 + input=at[1],
253 + scale=scale)
254 +
255 + # result, residual
256 + return at1[1], at[2]
257 +
258 + def replacement(result: torch.Tensor, input: torch.Tensor,
259 + residual: torch.Tensor, weight: torch.Tensor,
260 + scale: torch.Tensor):
261 + at = auto_functionalized(self.FUSED_OP,
262 + result=result,
263 + input=input,
264 + residual=residual,
265 + weight=weight,
266 + scale=scale,
267 + epsilon=self.epsilon)
268 +
269 + # result, residual
270 + return at[1], at[2]
271 +
272 + inputs = [
273 + torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
274 + empty_bf16(5, 4), # input
275 + empty_bf16(5, 4), # residual
276 + empty_bf16(1, 5), # weight
277 + empty_fp32(1, 1) # scale
278 + ]
279 +
280 + pm.register_replacement(
281 + pattern,
282 + replacement,
283 + inputs,
284 + pm.fwd_only,
285 + pm_pass,
286 + extra_check=lambda m: record_match(
287 + self.Match(m, self.QUANT_OP, self.FUSED_OP)))
288 +
289 + class Match(QuantMultiOutputMatch):
290 +
291 + def process(self):
292 + # Find the nodes in the match that we need to rebind
293 + rms_node = self.find_auto_fn(RMS_ADD_OP)
294 + quant_node = self.find_auto_fn(self.QUANT_OP)
295 +
296 + assert len(rms_node.users) == 2
297 + assert len(quant_node.users) == 1
298 +
299 + # First, insert a new auto_functionalized node for the fused op,
300 + # as well as getitem nodes to extract the result and residual.
301 + # The auto_fn node returns a tuple of (None, result, residual).
302 + #
303 + # The resulting graph looks like this:
304 + # at = auto_functionalized(torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, ...) # noqa
305 + # result_node_new = at[1]
306 + # residual_node_new = at[2]
307 + with self.inserting_after_match():
308 + # Missing epsilon, scalars cannot be inputs to the pattern
309 + kwargs = self.match.kwargs.copy()
310 +
311 + # 0 is always None
312 + fused_return_mapping = {1: (quant_node, 1), 2: (rms_node, 2)}
313 + self.insert_fused_node(fused_return_mapping,
314 + epsilon=rms_node.kwargs["epsilon"],
315 + **kwargs)
316 +
317 +
318 +class RMSNormDynamicQuantPattern(RMSNormQuantPattern):
319 +
320 + def __init__(self,
321 + epsilon: float,
322 + quant_dtype: torch.dtype,
323 + per_tensor: bool,
324 + symmetric=True):
325 + key = FusedRMSQuantKey(fused_add=False,
326 + quant=QuantKey(dtype=quant_dtype,
327 + static=False,
328 + per_tensor=per_tensor,
329 + symmetric=symmetric))
330 + super().__init__(epsilon, key)
331 +
332 + def register(self, pm_pass: PatternMatcherPass,
333 + record_match: Callable[[MultiOutputMatch], bool]):
334 +
335 + def pattern(result: torch.Tensor, result_rms: torch.Tensor,
336 + input: torch.Tensor, weight: torch.Tensor,
337 + scale: torch.Tensor):
338 + at1 = auto_functionalized(RMS_OP,
339 + result=result_rms,
340 + input=input,
341 + weight=weight,
342 + epsilon=self.epsilon)
343 + at2 = auto_functionalized(self.QUANT_OP,
344 + result=result,
345 + input=at1[1],
346 + scale=scale,
347 + scale_ub=None)
348 +
349 + # result, scale
350 + return at2[1], at2[2]
351 +
352 + def replacement(result: torch.Tensor, result_rms: torch.Tensor,
353 + input: torch.Tensor, weight: torch.Tensor,
354 + scale: torch.Tensor):
355 + at = auto_functionalized(self.FUSED_OP,
356 + result=result,
357 + input=input,
358 + weight=weight,
359 + scale=scale,
360 + epsilon=self.epsilon,
361 + scale_ub=None,
362 + residual=None)
363 +
364 + # result, scale
365 + return at[1], at[2]
366 +
367 + inputs = [
368 + torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
369 + empty_bf16(5, 4), # result_rms
370 + empty_bf16(5, 4), # input
371 + empty_bf16(1, 5), # weight
372 + empty_fp32(1, 1) # scale
373 + ]
374 +
375 + pm.register_replacement(
376 + pattern,
377 + replacement,
378 + inputs,
379 + pm.fwd_only,
380 + pm_pass,
381 + extra_check=lambda m: record_match(
382 + self.Match(m, self.QUANT_OP, self.FUSED_OP)))
383
384 + class Match(QuantMultiOutputMatch):
385
96 -# Returns the first auto_functionalized node with the given op (if it exists)
97 -def find_auto_fn_maybe(nodes: Iterable[torch.fx.Node],
98 - op) -> Optional[torch.fx.Node]:
99 - for node in nodes:
100 - if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
101 - return node
102 - return None
386 + def process(self):
387 + # Find the nodes in the match that we need to rebind
388 + rms_node = self.find_auto_fn(RMS_OP)
389 + quant_node = self.find_auto_fn(self.QUANT_OP)
390
391 + assert len(rms_node.users) == 1
392 + assert len(quant_node.users) == 2
393
105 -# Returns the first auto_functionalized node with the given op
106 -def find_auto_fn(nodes: Iterable[torch.fx.Node], op) -> torch.fx.Node:
107 - node = find_auto_fn_maybe(nodes, op)
108 - assert node is not None, f"Could not find {op} in nodes {nodes}"
109 - return node
394 + # First, insert a new auto_functionalized node for the fused op,
395 + # as well as getitem nodes to extract the result and scale.
396 + # The auto_fn node returns a tuple of (None, result, scale).
397 + #
398 + # The resulting graph looks like this:
399 + # at = auto_functionalized(torch.ops._C.rms_norm_dynamic_per_token_quant.default, ...) # noqa
400 + # result_node_new = at[1]
401 + # scale_node_new = at[2]
402 + with self.inserting_after_match():
403 + # Missing epsilon, scalars cannot be inputs to the pattern
404 + kwargs = self.match.kwargs.copy()
405 + del kwargs["result_rms"] # not used in the fused op
406 +
407 + fused_return_mapping = {1: (quant_node, 1), 2: (quant_node, 2)}
408 + self.insert_fused_node(
409 + fused_return_mapping,
410 + epsilon=rms_node.kwargs["epsilon"],
411 + scale_ub=None, # not used but required
412 + residual=None, # not used but required
413 + **kwargs)
414 +
415 +
416 +class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
417 +
418 + def __init__(self,
419 + epsilon: float,
420 + quant_dtype: torch.dtype,
421 + per_tensor: bool = True,
422 + symmetric=True):
423 + key = FusedRMSQuantKey(fused_add=True,
424 + quant=QuantKey(dtype=quant_dtype,
425 + static=False,
426 + per_tensor=per_tensor,
427 + symmetric=symmetric))
428 + super().__init__(epsilon, key)
429 +
430 + def register(self, pm_pass: PatternMatcherPass,
431 + record_match: Callable[[MultiOutputMatch], bool]):
432 +
433 + def pattern(result: torch.Tensor, input: torch.Tensor,
434 + residual: torch.Tensor, weight: torch.Tensor,
435 + scale: torch.Tensor):
436 + at = auto_functionalized(RMS_ADD_OP,
437 + input=input,
438 + residual=residual,
439 + weight=weight,
440 + epsilon=self.epsilon)
441 + at1 = auto_functionalized(self.QUANT_OP,
442 + result=result,
443 + input=at[1],
444 + scale=scale,
445 + scale_ub=None)
446 +
447 + # result, residual, scale
448 + return at1[1], at[2], at1[2]
449 +
450 + def replacement(result: torch.Tensor, input: torch.Tensor,
451 + residual: torch.Tensor, weight: torch.Tensor,
452 + scale: torch.Tensor):
453 + at = auto_functionalized(self.FUSED_OP,
454 + result=result,
455 + input=input,
456 + weight=weight,
457 + scale=scale,
458 + epsilon=self.epsilon,
459 + scale_ub=None,
460 + residual=residual)
461 +
462 + # result, residual, scale
463 + return at[1], at[3], at[2]
464
465 + inputs = [
466 + torch.empty(5, 4, device="cuda", dtype=self.quant_dtype), # result
467 + empty_bf16(5, 4), # input
468 + empty_bf16(5, 4), # residual
469 + empty_bf16(1, 5), # weight
470 + empty_fp32(1, 1) # scale
471 + ]
472
112 -# Returns the getitem node that extracts the idx-th element from node
113 -# (if it exists)
114 -def find_getitem_maybe(node: torch.fx.Node,
115 - idx: int) -> Optional[torch.fx.Node]:
116 - for user in node.users:
117 - if is_func(user, operator.getitem) and user.args[1] == idx:
118 - return user
119 - return None
473 + pm.register_replacement(
474 + pattern,
475 + replacement,
476 + inputs,
477 + pm.fwd_only,
478 + pm_pass,
479 + extra_check=lambda m: record_match(
480 + self.Match(m, self.QUANT_OP, self.FUSED_OP)))
481
482 + class Match(QuantMultiOutputMatch):
483
122 -# Returns the getitem node that extracts the idx-th element from node
123 -def find_getitem(node: torch.fx.Node, idx: int) -> torch.fx.Node:
124 - ret = find_getitem_maybe(node, idx)
125 - assert ret is not None, f"Could not find getitem {idx} in node {node}"
126 - return ret
484 + def process(self):
485 + # Find the nodes in the match that we need to rebind
486 + rms_node = self.find_auto_fn(RMS_ADD_OP)
487 + quant_node = self.find_auto_fn(self.QUANT_OP)
488 +
489 + assert len(rms_node.users) == 2
490 + assert len(quant_node.users) == 2
491 +
492 + # First, insert a new auto_functionalized node for the fused op,
493 + # as well as getitem nodes to extract result, scale, and residual.
494 + # The auto_fn node returns a tuple (None, result, scale, residual).
495 + #
496 + # The resulting graph looks like this:
497 + # at = auto_functionalized(torch.ops._C.rms_norm_dynamic_per_token_quant.default, ...) # noqa
498 + # result_node_new = at[1]
499 + # scale_node_new = at[2]
500 + # residual_node_new = at[3]
501 + with self.inserting_after_match():
502 + # Missing epsilon, scalars cannot be inputs to the pattern
503 + kwargs = self.match.kwargs.copy()
504 +
505 + fused_return_mapping = {
506 + 1: (quant_node, 1), # result
507 + 2: (quant_node, 2), # scale
508 + 3: (rms_node, 2), # residual
509 + }
510 + self.insert_fused_node(
511 + fused_return_mapping,
512 + epsilon=rms_node.kwargs["epsilon"],
513 + scale_ub=None, # not used but required
514 + **kwargs)
515
516
517 class FusionPass(VllmInductorPass):
@@ -158,41 +546,39 @@ class FusionPass(VllmInductorPass):
546 "FusionPass singleton instance already exists"
547 super().__init__(config)
548
161 - self.matches: List[Match] = []
549 + self.matches: List[MultiOutputMatch] = []
550 self.patterns: PatternMatcherPass = PatternMatcherPass(
551 pass_name="fusion_pass")
552
165 - # Fuse rms_norm + static_scaled_fp8_quant into
166 - # rms_norm_static_fp8_quant
167 - inputs = [
168 - empty_fp8(5, 4),
169 - empty_bf16(5, 4),
170 - empty_bf16(5, 4),
171 - empty_bf16(1, 5),
172 - empty_fp32(1, 1)
173 - ]
174 - register_replacement(rms_pattern_static, rms_replacement_static,
175 - inputs, fwd_only, self.patterns)
553 + for epsilon in [1e-5, 1e-6]:
554 + # Fuse rms_norm + static fp8 quant
555 + RMSNormStaticQuantPattern(epsilon,
556 + FP8_DTYPE).register(self.patterns)
557
177 - # Fuse fused_add_rms_norm + static_scaled_fp8_quant into
178 - # fused_add_rms_norm_static_fp8_quant
179 - # Because pattern has 2 outputs, we need to manually process the match
180 - # (see process_matches)
181 - inputs = [
182 - empty_fp8(5, 4),
183 - empty_bf16(5, 4),
184 - empty_bf16(5, 4),
185 - empty_bf16(1, 5),
186 - empty_fp32(1, 1)
187 - ]
188 - register_replacement(rms_pattern_residual_static,
189 - rms_replacement_residual_static,
190 - inputs,
191 - fwd_only,
192 - self.patterns,
193 - extra_check=lambda m: self.record_match(m))
194 -
195 - def record_match(self, match: Match) -> bool:
558 + # Matches for patterns below have 2 or more outputs,
559 + # so we need to process them manually (see process_matches)
560 +
561 + # Fuse rms_norm + static fp8 quant
562 + FusedAddRMSNormStaticQuantPattern(epsilon, FP8_DTYPE).register(
563 + self.patterns, self.record_match)
564 +
565 + # Fuse rms_norm + dynamic per-token fp8 quant
566 + RMSNormDynamicQuantPattern(epsilon, FP8_DTYPE,
567 + per_tensor=False).register(
568 + self.patterns, self.record_match)
569 +
570 + # Fuse fused_add_rms_norm + dynamic per-token fp8 quant
571 + FusedAddRMSNormDynamicQuantPattern(epsilon,
572 + FP8_DTYPE,
573 + per_tensor=False).register(
574 + self.patterns,
575 + self.record_match)
576 +
577 + # WARNING: This is a hack to clear the pattern matcher cache
578 + # and allow multiple values of epsilon.
579 + torch._inductor.pattern_matcher._seen_patterns.clear()
580 +
581 + def record_match(self, match: MultiOutputMatch) -> bool:
582 # Hijack the extra_check to record the match and
583 # save it for post-processing.
584 self.matches.append(match)
@@ -200,83 +586,20 @@ class FusionPass(VllmInductorPass):
586 # Return False to prevent automatic replacement.
587 return False
588
203 - def process_matches(self, graph: torch.fx.Graph):
589 + def process_matches(self, graph: fx.Graph):
590 """
591 Manually process multi-output matches and replace them with fused nodes.
206 - This is necessary because the automatic replacement for multi-output
207 - matches is broken: https://github.com/pytorch/pytorch/issues/137280
592 + See MultiOutputMatch for more details.
593 """
594 for match in self.matches:
210 - # To avoid use-before-definition errors, insert replacement nodes
211 - # after the last node in the match.
212 - # match.nodes is not guaranteed to be sorted.
213 - # Find the last node in the match.
214 - for last_node_in_match in reversed(graph.nodes):
215 - if last_node_in_match in match.nodes:
216 - break
217 - else:
218 - raise ValueError("No nodes in graph")
219 -
220 - # Insert a new auto_functionalized node for the fused operation,
221 - # as well as getitem nodes to extract the result and residual.
222 - # The auto_functionalized node returns a tuple of
223 - # (None, result, residual) - None is the function return value.
224 - # The resulting graph looks like this:
225 - # at = auto_functionalized(torch.ops._C.fused_add_rms_norm_static_fp8_quant.default, ...) # noqa
226 - # result_node_new = at[1]
227 - # residual_node_new = at[2]
228 - with graph.inserting_after(last_node_in_match):
229 - kwargs = match.kwargs
230 - kwargs["epsilon"] = 1e-5 # Currently hard-coded in RMSNorm
231 -
232 - fused_node = graph.call_function(
233 - auto_functionalized,
234 - (torch.ops._C.fused_add_rms_norm_static_fp8_quant.default,
235 - ),
236 - kwargs=kwargs)
237 -
238 - graph.inserting_after(fused_node)
239 - result_node_new = graph.call_function(operator.getitem,
240 - (fused_node, 1))
241 - residual_node_new = graph.call_function(
242 - operator.getitem, (fused_node, 2))
243 -
244 - # Last part of replacement is rebinding the users of nodes in the
245 - # match to use the new nodes.
246 -
247 - # Find the nodes in the match that we need to rebind
248 - rms_node = find_auto_fn(match.nodes,
249 - torch.ops._C.fused_add_rms_norm.default)
250 - quant_node = find_auto_fn(
251 - match.nodes, torch.ops._C.static_scaled_fp8_quant.default)
252 -
253 - assert len(rms_node.users) == 2
254 - assert len(quant_node.users) == 1
255 -
256 - # meta["val"] is used by de-functionalization and has to contain the
257 - # value of the node (tuple of tensors) that would be returned by the
258 - # functionalized node during tracing.
259 -
260 - rms_tup = rms_node.meta["val"]
261 - quant_tup = quant_node.meta["val"]
262 -
263 - # The result of fused_node must be a tuple with the first element
264 - # None (the function return value) and the remaining elements
265 - # representing the mutated inputs.
266 - fused_tup = (None, quant_tup[1], rms_tup[1], rms_tup[2])
267 - fused_node.meta["val"] = fused_tup
268 -
269 - # Find the getitem nodes and replace their uses with the new nodes.
270 - # The old nodes will be removed by DCE at the end of the pass.
271 - find_getitem(rms_node, 2).replace_all_uses_with(residual_node_new)
272 - find_getitem(quant_node, 1).replace_all_uses_with(result_node_new)
595 + match.process()
596
597 # Finally, remove matched nodes
598 graph.eliminate_dead_code()
599 assert all(node not in graph.nodes for match in self.matches
277 - for node in match.nodes)
600 + for node in match.match.nodes)
601
279 - def __call__(self, graph: torch.fx.Graph):
602 + def __call__(self, graph: fx.Graph):
603 self.begin()
604 self.dump_graph(graph, "before_fusion")
605
vllm/compilation/fx_utils.py new
+42
@@ -0,0 +1,42 @@
1 +import operator
2 +from typing import Iterable, Optional
3 +
4 +from torch import fx
5 +from torch._higher_order_ops.auto_functionalize import auto_functionalized
6 +from torch._ops import OpOverload
7 +
8 +
9 +def is_func(node: fx.Node, target) -> bool:
10 + return node.op == "call_function" and node.target == target
11 +
12 +
13 +# Returns the first auto_functionalized node with the given op (if it exists)
14 +def find_auto_fn_maybe(nodes: Iterable[fx.Node],
15 + op: OpOverload) -> Optional[fx.Node]:
16 + for node in nodes:
17 + if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
18 + return node
19 + return None
20 +
21 +
22 +# Returns the first auto_functionalized node with the given op
23 +def find_auto_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node:
24 + node = find_auto_fn_maybe(nodes, op)
25 + assert node is not None, f"Could not find {op} in nodes {nodes}"
26 + return node
27 +
28 +
29 +# Returns the getitem node that extracts the idx-th element from node
30 +# (if it exists)
31 +def find_getitem_maybe(node: fx.Node, idx: int) -> Optional[fx.Node]:
32 + for user in node.users:
33 + if is_func(user, operator.getitem) and user.args[1] == idx:
34 + return user
35 + return None
36 +
37 +
38 +# Returns the getitem node that extracts the idx-th element from node
39 +def find_getitem(node: fx.Node, idx: int) -> fx.Node:
40 + ret = find_getitem_maybe(node, idx)
41 + assert ret is not None, f"Could not find getitem {idx} in node {node}"
42 + return ret
vllm/compilation/multi_output_match.py new
+105
@@ -0,0 +1,105 @@
1 +import abc
2 +import operator
3 +from abc import abstractmethod
4 +from typing import Iterable, List, Tuple
5 +
6 +from torch import fx
7 +from torch._higher_order_ops.auto_functionalize import auto_functionalized
8 +from torch._inductor import pattern_matcher as pm
9 +from torch._ops import OpOverload
10 +
11 +from vllm.compilation.fx_utils import find_auto_fn
12 +
13 +
14 +class MultiOutputMatch(abc.ABC):
15 + """
16 + This class provides utilities to process multi-output matches and
17 + manually insert replacements.
18 +
19 + This is necessary because the automatic replacement for multi-output
20 + matches is broken: https://github.com/pytorch/pytorch/issues/137280
21 + """
22 +
23 + def __init__(self, match: pm.Match):
24 + self.match = match
25 +
26 + @abstractmethod
27 + def process(self):
28 + """
29 + Process a multi-output match and manually insert the replacement.
30 +
31 + This method should:
32 + 1. Insert the replacement nodes after the last node in the match.
33 + 2. Rebind the users of nodes in the match to use the new nodes.
34 + 3. Set meta["val"] for de-functionalization.
35 +
36 + The result of an auto-functionalized node is a tuple of tensors.
37 + The first element is the return value of the function, usually None.
38 + The remaining elements are the mutated args of the function.
39 +
40 + All auto-functionalized nodes must contain a proper meta["val"],
41 + as it is used by de-functionalization. meta["val"] has to contain the
42 + value of the node (tuple of tensors) that would be returned by the
43 + functionalized node during tracing.
44 +
45 + Existing nodes in the graph all have this property set, but we have
46 + to set it manually for new nodes we insert.
47 +
48 + Example:
49 + # op schema: foo(a: Tensor!, b: Tensor, c: Tensor!) -> None
50 + at = auto_functionalized(torch.ops._C.foo.default, a, b, c)
51 + # at.meta["val"] = (None, a, c)
52 + """
53 + raise NotImplementedError
54 +
55 + @property
56 + def nodes(self) -> List[fx.Node]:
57 + return self.match.nodes
58 +
59 + @property
60 + def graph(self) -> fx.Graph:
61 + return self.match.graph
62 +
63 + def find_auto_fn(self, op) -> fx.Node:
64 + """
65 + Find the first auto_functionalized node with the given op in the match.
66 + """
67 + return find_auto_fn(self.nodes, op)
68 +
69 + def inserting_after_match(self):
70 + """
71 + Insert nodes after the last node in the match.
72 + This is done to avoid use-before-definition errors after inserting
73 + replacement nodes.
74 + """
75 +
76 + # match.nodes is not guaranteed to be sorted.
77 + # Find the last node in the match.
78 + for last_node_in_match in reversed(self.graph.nodes):
79 + if last_node_in_match in self.match.nodes:
80 + break
81 + else:
82 + raise ValueError("No nodes in graph")
83 +
84 + return self.graph.inserting_after(last_node_in_match)
85 +
86 + def insert_getitems(self, tuple_node: fx.Node,
87 + indices: Iterable[int]) -> Tuple[fx.Node, ...]:
88 + """
89 + Insert operator.getitem nodes to extract elements from a tuple node.
90 +
91 + :param tuple_node: The tuple node to extract elements from.
92 + :param indices: The indices of the elements to extract.
93 + :return: Tuple of the new getitem nodes, corresponding to the indices.
94 + """
95 + with self.graph.inserting_after(tuple_node):
96 + return tuple(
97 + self.graph.call_function(operator.getitem, (tuple_node, idx))
98 + for idx in indices)
99 +
100 + def insert_auto_fn(self, op: OpOverload, kwargs):
101 + """
102 + Insert an auto_functionalized node with the given op and kwargs.
103 + """
104 + return self.graph.call_function(auto_functionalized, (op, ),
105 + kwargs=kwargs)
vllm/compilation/reshapes.py
+2 -1
@@ -5,7 +5,8 @@ from torch import SymInt
5
6 from vllm.logger import init_logger
7
8 -from .vllm_inductor_pass import VllmInductorPass, is_func
8 +from .fx_utils import is_func
9 +from .vllm_inductor_pass import VllmInductorPass
10
11 logger = init_logger(__name__)
12
vllm/compilation/vllm_inductor_pass.py
-4
@@ -16,10 +16,6 @@ from .inductor_pass import InductorPass
16 logger = init_logger(__name__)
17
18
19 -def is_func(node: torch.fx.Node, target) -> bool:
20 - return node.op == "call_function" and node.target == target
21 -
22 -
19 class VllmInductorPass(InductorPass):
20 """
21 An inductor pass with access to vLLM PassConfig.