A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[feat]: CUTLASS block scaled group gemm for SM100 (#19757)
Signed-off-by: Duncan Moss <djm.moss@gmail.com> Co-authored-by: Duncan Moss <dmoss@nvidia.com>
Duncan Moss committed
Jul 4, 2025 at 11:58 UTC
3d184b95b83bd1c755811238b013b51c49da3951
13 files changed
+726
-30
CMakeLists.txt
+21
-1
@@ -259,7 +259,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
259
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
260
261
# Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building.
262
- set(CUTLASS_REVISION "v3.9.2" CACHE STRING "CUTLASS revision to use")
262
+ set(CUTLASS_REVISION "v4.0.0" CACHE STRING "CUTLASS revision to use")
263
264
# Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
265
if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
@@ -615,6 +615,26 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
615
"in CUDA target architectures.")
616
endif()
617
endif()
618
+
619
+ cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a" "${CUDA_ARCHS}")
620
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
621
+ set(SRCS "csrc/quantization/cutlass_w8a8/moe/blockwise_scaled_group_mm_sm100.cu")
622
+ set_gencode_flags_for_srcs(
623
+ SRCS "${SRCS}"
624
+ CUDA_ARCHS "${SCALED_MM_ARCHS}")
625
+ list(APPEND VLLM_EXT_SRC "${SRCS}")
626
+ list(APPEND VLLM_GPU_FLAGS "-DENABLE_CUTLASS_MOE_SM100=1")
627
+ message(STATUS "Building blockwise_scaled_group_mm_sm100 for archs: ${SCALED_MM_ARCHS}")
628
+ else()
629
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
630
+ message(STATUS "Not building blockwise_scaled_group_mm_sm100 kernels as CUDA Compiler version is "
631
+ "not >= 12.8, we recommend upgrading to CUDA 12.8 or later "
632
+ "if you intend on running FP8 quantized MoE models on Blackwell.")
633
+ else()
634
+ message(STATUS "Not building blockwise_scaled_group_mm_sm100 as no compatible archs found "
635
+ "in CUDA target architectures")
636
+ endif()
637
+ endif()
638
639
#
640
# Machete kernels
csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp
-1
@@ -45,7 +45,6 @@
45
#include "cute/algorithm/functional.hpp"
46
#include "cute/atom/mma_atom.hpp"
47
#include "cute/algorithm/gemm.hpp"
48
-#include "cute/tensor_predicate.hpp"
48
#include "cute/numeric/arithmetic_tuple.hpp"
49
50
#include "cutlass_extensions/gemm/dispatch_policy.hpp"
csrc/ops.h
+5
@@ -239,6 +239,11 @@ void cutlass_moe_mm(
239
torch::Tensor const& b_strides, torch::Tensor const& c_strides,
240
bool per_act_token, bool per_out_ch);
241
242
+void cutlass_blockwise_scaled_grouped_mm(
243
+ torch::Tensor& output, const torch::Tensor& a, const torch::Tensor& b,
244
+ const torch::Tensor& scales_a, const torch::Tensor& scales_b,
245
+ const torch::Tensor& problem_sizes, const torch::Tensor& expert_offsets);
246
+
247
void cutlass_fp4_group_mm(
248
torch::Tensor& output, const torch::Tensor& a, const torch::Tensor& b,
249
const torch::Tensor& a_blockscale, const torch::Tensor& b_blockscales,
csrc/quantization/cutlass_w8a8/c3x/scaled_mm.cuh
+2
-1
@@ -51,7 +51,8 @@ struct cutlass_3x_gemm {
51
// These are the minimum alignments needed for the kernels to compile
52
static constexpr int AlignmentAB =
53
128 / cutlass::sizeof_bits<ElementAB>::value;
54
- static constexpr int AlignmentCD = 4;
54
+ static constexpr int AlignmentCD =
55
+ 128 / cutlass::sizeof_bits<ElementD>::value;
56
57
using CollectiveEpilogue =
58
typename cutlass::epilogue::collective::CollectiveBuilder<
csrc/quantization/cutlass_w8a8/moe/blockwise_scaled_group_mm_sm100.cu
new
+367
@@ -0,0 +1,367 @@
1
+#include <torch/all.h>
2
+#include <cutlass/arch/arch.h>
3
+
4
+#include <ATen/cuda/CUDAContext.h>
5
+#include <c10/cuda/CUDAGuard.h>
6
+#include <c10/cuda/CUDAStream.h>
7
+
8
+#include "cute/tensor.hpp"
9
+#include "cutlass/tensor_ref.h"
10
+#include "cutlass/epilogue/collective/default_epilogue.hpp"
11
+#include "cutlass/epilogue/thread/linear_combination.h"
12
+#include "cutlass/gemm/dispatch_policy.hpp"
13
+#include "cutlass/gemm/group_array_problem_shape.hpp"
14
+#include "cutlass/gemm/collective/collective_builder.hpp"
15
+#include "cutlass/epilogue/collective/collective_builder.hpp"
16
+#include "cutlass/gemm/device/gemm_universal_adapter.h"
17
+#include "cutlass/gemm/kernel/gemm_universal.hpp"
18
+
19
+#include "cutlass/util/command_line.h"
20
+#include "cutlass/util/distribution.h"
21
+#include "cutlass/util/host_tensor.h"
22
+#include "cutlass/util/packed_stride.hpp"
23
+#include "cutlass/util/tensor_view_io.h"
24
+#include "cutlass/util/reference/device/gemm.h"
25
+#include "cutlass/util/reference/device/tensor_compare.h"
26
+#include "cutlass/util/reference/host/tensor_fill.h"
27
+#include "cutlass/util/reference/host/gett.hpp"
28
+#include "cutlass/util/reference/host/tensor_norm.h"
29
+#include "cutlass/util/reference/host/tensor_compare.h"
30
+#include <cassert>
31
+
32
+using namespace cute;
33
+
34
+template <typename ElementAB, typename ElementC, typename ElementAccumulator,
35
+ typename LayoutSFA, typename LayoutSFB, typename ScaleConfig>
36
+__global__ void get_ggemm_starts(
37
+ int32_t* expert_offsets, ElementAB** a_offsets, ElementAB** b_offsets,
38
+ ElementC** out_offsets, ElementAccumulator** a_scale_offsets,
39
+ ElementAccumulator** b_scale_offsets, ElementAB* a_base_as_int,
40
+ ElementAB* b_base_as_int, ElementC* out_base_as_int,
41
+ ElementAccumulator* a_scale_base_as_int,
42
+ ElementAccumulator* b_scale_base_as_int, LayoutSFA* layout_sfa_base_as_int,
43
+ LayoutSFB* layout_sfb_base_as_int, int* problem_sizes) {
44
+ int expert_id = threadIdx.x;
45
+
46
+ if (expert_id >= gridDim.x * blockDim.x) {
47
+ return;
48
+ }
49
+
50
+ int m = problem_sizes[expert_id * 3];
51
+ int n = problem_sizes[expert_id * 3 + 1];
52
+ int k = problem_sizes[expert_id * 3 + 2];
53
+
54
+ int32_t expert_offset = expert_offsets[expert_id];
55
+ int a_stride = expert_offset * k;
56
+ int b_stride = expert_id * k * n;
57
+ int a_scale_stride = expert_offset * k / 128;
58
+ int b_scale_stride = expert_id * k * n / 128 / 128;
59
+
60
+ a_offsets[expert_id] = a_base_as_int + a_stride;
61
+ b_offsets[expert_id] = b_base_as_int + b_stride;
62
+ out_offsets[expert_id] = out_base_as_int + expert_offset * n;
63
+ a_scale_offsets[expert_id] = a_scale_base_as_int + a_scale_stride;
64
+ b_scale_offsets[expert_id] = b_scale_base_as_int + b_scale_stride;
65
+
66
+ LayoutSFA* layout_sfa_ptr = layout_sfa_base_as_int + expert_id;
67
+ LayoutSFB* layout_sfb_ptr = layout_sfb_base_as_int + expert_id;
68
+
69
+ *layout_sfa_ptr =
70
+ ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1));
71
+ *layout_sfb_ptr =
72
+ ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1));
73
+}
74
+
75
+#define __CALL_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, \
76
+ ScaleConfig) \
77
+ else if (out_tensors.dtype() == TENSOR_C_TYPE) { \
78
+ get_ggemm_starts<cutlass::float_e4m3_t, C_TYPE, float, LayoutSFA, \
79
+ LayoutSFB, ScaleConfig><<<1, num_experts, 0, stream>>>( \
80
+ static_cast<int32_t*>(expert_offsets.data_ptr()), \
81
+ static_cast<cutlass::float_e4m3_t**>(a_ptrs.data_ptr()), \
82
+ static_cast<cutlass::float_e4m3_t**>(b_ptrs.data_ptr()), \
83
+ static_cast<C_TYPE**>(out_ptrs.data_ptr()), \
84
+ static_cast<float**>(a_scales_ptrs.data_ptr()), \
85
+ static_cast<float**>(b_scales_ptrs.data_ptr()), \
86
+ static_cast<cutlass::float_e4m3_t*>(a_tensors.data_ptr()), \
87
+ static_cast<cutlass::float_e4m3_t*>(b_tensors.data_ptr()), \
88
+ static_cast<C_TYPE*>(out_tensors.data_ptr()), \
89
+ static_cast<float*>(a_scales.data_ptr()), \
90
+ static_cast<float*>(b_scales.data_ptr()), \
91
+ reinterpret_cast<LayoutSFA*>(layout_sfa.data_ptr()), \
92
+ reinterpret_cast<LayoutSFB*>(layout_sfb.data_ptr()), \
93
+ static_cast<int*>(problem_sizes.data_ptr())); \
94
+ }
95
+
96
+template <typename LayoutSFA, typename LayoutSFB, typename ScaleConfig>
97
+void run_get_ggemm_starts(
98
+ torch::Tensor const& expert_offsets, torch::Tensor& a_ptrs,
99
+ torch::Tensor& b_ptrs, torch::Tensor& out_ptrs,
100
+ torch::Tensor& a_scales_ptrs, torch::Tensor& b_scales_ptrs,
101
+ torch::Tensor const& a_tensors, torch::Tensor const& b_tensors,
102
+ torch::Tensor out_tensors, torch::Tensor const& a_scales,
103
+ torch::Tensor const& b_scales, torch::Tensor const& layout_sfa,
104
+ torch::Tensor const& layout_sfb, torch::Tensor const& problem_sizes) {
105
+ TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn);
106
+ TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn);
107
+ TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
108
+ TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
109
+ TORCH_CHECK(out_tensors.size(1) % 128 == 0 or out_tensors.size(0) % 128 == 0);
110
+ TORCH_CHECK(a_tensors.size(1) % 128 == 0 or a_tensors.size(0) % 128 == 0);
111
+
112
+ int num_experts = (int)expert_offsets.size(0);
113
+ auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index());
114
+
115
+ if (false) {
116
+ }
117
+ __CALL_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t, LayoutSFA,
118
+ LayoutSFB, ScaleConfig)
119
+ __CALL_GET_STARTS_KERNEL(torch::kFloat16, cutlass::half_t, LayoutSFA,
120
+ LayoutSFB, ScaleConfig)
121
+ else {
122
+ TORCH_CHECK(false, "Unsupported output tensor type");
123
+ }
124
+}
125
+
126
+template <typename OutType, typename ScheduleConfig, typename LayoutD>
127
+void run_blockwise_scaled_group_mm(
128
+ torch::Tensor& out_ptrs, const torch::Tensor& a_ptrs,
129
+ const torch::Tensor& b_ptrs, const torch::Tensor& a_scales_ptrs,
130
+ const torch::Tensor& b_scales_ptrs, const torch::Tensor& stride_a,
131
+ const torch::Tensor& stride_b, const torch::Tensor& stride_c,
132
+ const torch::Tensor& layout_sfa, const torch::Tensor& layout_sfb,
133
+ const torch::Tensor& problem_sizes, const torch::Tensor& expert_offsets) {
134
+ using ProblemShape = cutlass::gemm::GroupProblemShape<Shape<int, int, int>>;
135
+
136
+ // Types
137
+ using ElementA = cutlass::float_e4m3_t;
138
+ using ElementB = cutlass::float_e4m3_t;
139
+ using ElementC = OutType;
140
+ using ElementD = ElementC;
141
+ using ElementAccumulator = float;
142
+ using LayoutA = cutlass::layout::RowMajor;
143
+ using LayoutB = cutlass::layout::ColumnMajor;
144
+ using LayoutC = LayoutD;
145
+
146
+ // Alignments
147
+ static constexpr int AlignmentA = 128 / cutlass::sizeof_bits<ElementA>::value;
148
+ static constexpr int AlignmentB = 128 / cutlass::sizeof_bits<ElementB>::value;
149
+ static constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
150
+
151
+ using ArchTag = cutlass::arch::Sm100;
152
+ using OperatorClass = cutlass::arch::OpClassTensorOp;
153
+
154
+ using CollectiveEpilogue =
155
+ typename cutlass::epilogue::collective::CollectiveBuilder<
156
+ ArchTag, OperatorClass, typename ScheduleConfig::MmaTileShape,
157
+ typename ScheduleConfig::ClusterShape,
158
+ cutlass::epilogue::collective::EpilogueTileAuto, ElementAccumulator,
159
+ ElementAccumulator, void, LayoutC*, AlignmentC, ElementD, LayoutC*,
160
+ AlignmentC, typename ScheduleConfig::EpilogueSchedule>::CollectiveOp;
161
+
162
+ using CollectiveMainloop =
163
+ typename cutlass::gemm::collective::CollectiveBuilder<
164
+ ArchTag, OperatorClass, ElementA,
165
+ cute::tuple<LayoutA*, typename ScheduleConfig::LayoutSFA*>,
166
+ AlignmentA, ElementB,
167
+ cute::tuple<LayoutB*, typename ScheduleConfig::LayoutSFB*>,
168
+ AlignmentB, ElementAccumulator, typename ScheduleConfig::MmaTileShape,
169
+ typename ScheduleConfig::ClusterShape,
170
+ cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
171
+ sizeof(typename CollectiveEpilogue::SharedStorage))>,
172
+ typename ScheduleConfig::KernelSchedule>::CollectiveOp;
173
+
174
+ using GemmKernel =
175
+ cutlass::gemm::kernel::GemmUniversal<ProblemShape, CollectiveMainloop,
176
+ CollectiveEpilogue, void>;
177
+
178
+ using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
179
+ using StrideA = typename Gemm::GemmKernel::InternalStrideA;
180
+ using StrideB = typename Gemm::GemmKernel::InternalStrideB;
181
+ using StrideC = typename Gemm::GemmKernel::InternalStrideC;
182
+ using StrideD = typename Gemm::GemmKernel::InternalStrideD;
183
+
184
+ using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape;
185
+ int num_experts = (int)expert_offsets.size(0);
186
+
187
+ Gemm gemm_op;
188
+
189
+ // Mainloop Arguments
190
+ typename GemmKernel::MainloopArguments mainloop_args{
191
+ static_cast<const ElementA**>(a_ptrs.data_ptr()),
192
+ static_cast<StrideA*>(stride_a.data_ptr()),
193
+ static_cast<const ElementB**>(b_ptrs.data_ptr()),
194
+ static_cast<StrideB*>(stride_b.data_ptr()),
195
+ static_cast<const ElementAccumulator**>(a_scales_ptrs.data_ptr()),
196
+ reinterpret_cast<typename ScheduleConfig::LayoutSFA*>(
197
+ layout_sfa.data_ptr()),
198
+ static_cast<const ElementAccumulator**>(b_scales_ptrs.data_ptr()),
199
+ reinterpret_cast<typename ScheduleConfig::LayoutSFB*>(
200
+ layout_sfb.data_ptr())};
201
+
202
+ cutlass::KernelHardwareInfo hw_info;
203
+ hw_info.device_id = a_ptrs.get_device();
204
+ hw_info.sm_count =
205
+ cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
206
+ hw_info.device_id);
207
+
208
+ // Epilogue Arguments
209
+ typename GemmKernel::EpilogueArguments epilogue_args{
210
+ {}, // epilogue.thread
211
+ nullptr,
212
+ static_cast<StrideC*>(stride_c.data_ptr()),
213
+ static_cast<ElementD**>(out_ptrs.data_ptr()),
214
+ static_cast<StrideC*>(stride_c.data_ptr())};
215
+
216
+ UnderlyingProblemShape* problem_sizes_as_shapes =
217
+ static_cast<UnderlyingProblemShape*>(problem_sizes.data_ptr());
218
+
219
+ // Gemm Arguments
220
+ typename GemmKernel::Arguments args{
221
+ cutlass::gemm::GemmUniversalMode::kGrouped,
222
+ {num_experts, problem_sizes_as_shapes, nullptr},
223
+ mainloop_args,
224
+ epilogue_args,
225
+ hw_info};
226
+
227
+ at::cuda::CUDAGuard device_guard{(char)a_ptrs.device().index()};
228
+ const cudaStream_t stream =
229
+ at::cuda::getCurrentCUDAStream(a_ptrs.get_device());
230
+
231
+ auto can_implement_status = gemm_op.can_implement(args);
232
+ TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess,
233
+ "Failed to implement GEMM");
234
+
235
+ size_t workspace_size = gemm_op.get_workspace_size(args);
236
+ auto const workspace_options =
237
+ torch::TensorOptions().dtype(torch::kUInt8).device(a_ptrs.device());
238
+ auto workspace = torch::empty(workspace_size, workspace_options);
239
+
240
+ auto status = gemm_op.initialize(args, workspace.data_ptr(), stream);
241
+ TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM");
242
+
243
+ status = gemm_op.run(stream);
244
+ TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM");
245
+}
246
+
247
+template <typename OutType>
248
+void blockwise_scaled_group_mm_dispatch_shape(
249
+ torch::Tensor& output, const torch::Tensor& a, const torch::Tensor& b,
250
+ const torch::Tensor& scales_a, const torch::Tensor& scales_b,
251
+ const torch::Tensor& problem_sizes, const torch::Tensor& expert_offsets) {
252
+ struct MmaConfig {
253
+ using ElementA = cutlass::float_e4m3_t;
254
+ using KernelSchedule =
255
+ cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise1SmSm100;
256
+ using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm;
257
+ using ScaleConfig = cutlass::detail::Sm100BlockwiseScaleConfig<
258
+ 1, 128, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>;
259
+ using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA());
260
+ using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB());
261
+ using LayoutC = cutlass::layout::RowMajor;
262
+ using MmaTileShape = Shape<_128, _128, _128>;
263
+ using ClusterShape = Shape<_1, _1, _1>;
264
+ };
265
+
266
+ int num_experts = (int)expert_offsets.size(0);
267
+
268
+ auto a_ptrs = torch::empty(
269
+ {num_experts},
270
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
271
+ auto b_ptrs = torch::empty(
272
+ {num_experts},
273
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
274
+ auto out_ptrs = torch::empty(
275
+ {num_experts},
276
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
277
+ auto a_scales_ptrs = torch::empty(
278
+ {num_experts},
279
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
280
+ auto b_scales_ptrs = torch::empty(
281
+ {num_experts},
282
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
283
+
284
+ auto layout_sfa = torch::empty(
285
+ {num_experts, 5},
286
+ torch::TensorOptions().dtype(torch::kInt32).device(a.device()));
287
+ auto layout_sfb = torch::empty(
288
+ {num_experts, 5},
289
+ torch::TensorOptions().dtype(torch::kInt32).device(a.device()));
290
+
291
+ auto stride_a = torch::full(
292
+ {num_experts}, a.size(1),
293
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
294
+ auto stride_b = torch::full(
295
+ {num_experts}, a.size(1),
296
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
297
+ auto stride_c = torch::full(
298
+ {num_experts}, output.size(1),
299
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device()));
300
+
301
+ torch::TensorOptions options_int =
302
+ torch::TensorOptions().dtype(torch::kInt64).device(a.device());
303
+
304
+ run_get_ggemm_starts<typename MmaConfig::LayoutSFA,
305
+ typename MmaConfig::LayoutSFB,
306
+ typename MmaConfig::ScaleConfig>(
307
+ expert_offsets, a_ptrs, b_ptrs, out_ptrs, a_scales_ptrs, b_scales_ptrs, a,
308
+ b, output, scales_a, scales_b, layout_sfa, layout_sfb, problem_sizes);
309
+
310
+ run_blockwise_scaled_group_mm<OutType, MmaConfig,
311
+ typename MmaConfig::LayoutC>(
312
+ out_ptrs, a_ptrs, b_ptrs, a_scales_ptrs, b_scales_ptrs, stride_a,
313
+ stride_b, stride_c, layout_sfa, layout_sfb, problem_sizes,
314
+ expert_offsets);
315
+}
316
+
317
+void cutlass_blockwise_scaled_grouped_mm(
318
+ torch::Tensor& output, const torch::Tensor& a, const torch::Tensor& b,
319
+ const torch::Tensor& scales_a, const torch::Tensor& scales_b,
320
+ const torch::Tensor& problem_sizes, const torch::Tensor& expert_offsets) {
321
+ TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor");
322
+ TORCH_CHECK(problem_sizes.size(1) == 3,
323
+ "problem_sizes must have shape (num_experts, 3)");
324
+ TORCH_CHECK(problem_sizes.size(0) == expert_offsets.size(0),
325
+ "Number of experts in problem_sizes must match expert_offsets");
326
+ TORCH_CHECK(problem_sizes.dtype() == torch::kInt32,
327
+ "problem_sizes must be int32");
328
+ TORCH_CHECK(a.scalar_type() == torch::kFloat8_e4m3fn,
329
+ "a must be kFloat8_e4m3fn");
330
+ TORCH_CHECK(b.scalar_type() == torch::kFloat8_e4m3fn,
331
+ "b must be kFloat8_e4m3fn");
332
+ TORCH_CHECK(output.scalar_type() == torch::kBFloat16 ||
333
+ output.scalar_type() == torch::kHalf,
334
+ "output must be bfloat16 or half");
335
+ TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32,
336
+ "scales_a must be float32");
337
+ TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32,
338
+ "scales_b must be float32");
339
+ TORCH_CHECK(expert_offsets.scalar_type() == torch::kInt32,
340
+ "expert_offsets must be int32");
341
+
342
+ TORCH_CHECK(output.dim() == 2, "output must be 2D tensor");
343
+ TORCH_CHECK(a.dim() == 2, "a must be 2D tensor");
344
+ TORCH_CHECK(b.dim() == 3, "b must be 3D tensor");
345
+ TORCH_CHECK(scales_a.dim() == 2, "scales_a must be 2D tensor");
346
+ TORCH_CHECK(scales_b.dim() == 3, "scales_b must be 3D tensor");
347
+ TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor");
348
+ TORCH_CHECK(problem_sizes.size(1) == 3,
349
+ "problem_sizes must have shape (num_experts, 3)");
350
+ TORCH_CHECK(problem_sizes.size(0) == expert_offsets.size(0),
351
+ "Number of experts in problem_sizes must match expert_offsets");
352
+ TORCH_CHECK(problem_sizes.dtype() == torch::kInt32,
353
+ "problem_sizes must be int32");
354
+ TORCH_CHECK(expert_offsets.dim() == 1, "expert_offsets must be 1D tensor");
355
+
356
+#if defined(ENABLE_CUTLASS_MOE_SM100) && ENABLE_CUTLASS_MOE_SM100
357
+ if (output.scalar_type() == torch::kBFloat16) {
358
+ blockwise_scaled_group_mm_dispatch_shape<cutlass::bfloat16_t>(
359
+ output, a, b, scales_a, scales_b, problem_sizes, expert_offsets);
360
+ } else if (output.scalar_type() == torch::kFloat16) {
361
+ blockwise_scaled_group_mm_dispatch_shape<cutlass::half_t>(
362
+ output, a, b, scales_a, scales_b, problem_sizes, expert_offsets);
363
+ } else {
364
+ TORCH_CHECK(false, "Unsupported output tensor type");
365
+ }
366
+#endif
367
+}
\ No newline at end of file
csrc/quantization/machete/machete_mainloop.cuh
-1
@@ -38,7 +38,6 @@
38
#include "cute/atom/mma_atom.hpp"
39
#include "cute/atom/copy_traits_sm90_tma.hpp"
40
#include "cute/algorithm/gemm.hpp"
41
-#include "cute/tensor_predicate.hpp"
41
#include "cute/numeric/arithmetic_tuple.hpp"
42
#include "cutlass/pipeline/pipeline.hpp"
43
#include "cutlass/transform/collective/sm90_wgmma_transpose.hpp"
csrc/sparse/cutlass/sparse_scaled_mm_c3x.cuh
+2
-1
@@ -79,7 +79,8 @@ struct cutlass_sparse_3x_gemm {
79
// These are the minimum alignments needed for the kernels to compile
80
static constexpr int AlignmentAB =
81
128 / cutlass::sizeof_bits<ElementAB>::value;
82
- static constexpr int AlignmentCD = 4;
82
+ static constexpr int AlignmentCD =
83
+ 128 / cutlass::sizeof_bits<ElementD>::value;
84
85
using CollectiveEpilogue =
86
typename cutlass::epilogue::collective::CollectiveBuilder<
csrc/torch_bindings.cpp
+9
@@ -393,6 +393,15 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
393
{stride_tag});
394
ops.impl("cutlass_scaled_fp4_mm", torch::kCUDA, &cutlass_scaled_fp4_mm);
395
396
+ // cutlass blockwise scaledgroup GEMM
397
+ ops.def(
398
+ "cutlass_blockwise_scaled_grouped_mm(Tensor! output, Tensor a, Tensor b, "
399
+ "Tensor scales_a, Tensor scales_b, "
400
+ "Tensor problem_sizes, Tensor expert_offsets) -> ()",
401
+ {stride_tag});
402
+ ops.impl("cutlass_blockwise_scaled_grouped_mm", torch::kCUDA,
403
+ &cutlass_blockwise_scaled_grouped_mm);
404
+
405
// cutlass nvfp4 block scaled group GEMM
406
ops.def(
407
"cutlass_fp4_group_mm(Tensor! out, Tensor a, Tensor b,"
tests/kernels/moe/test_cutlass_grouped_gemm.py
new
+115
@@ -0,0 +1,115 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# DeepGEMM Style Cutlass Grouped GEMM Test
4
+# See https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_core.py
5
+
6
+import random
7
+
8
+import pytest
9
+import torch
10
+
11
+from tests.kernels.utils import baseline_scaled_mm
12
+from vllm import _custom_ops as ops
13
+from vllm.platforms import current_platform
14
+
15
+
16
+def cdiv(a, b):
17
+ return (a + b - 1) // b
18
+
19
+
20
+def per_token_cast_to_fp8(
21
+ x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
22
+ assert x.dim() == 2
23
+ m, n = x.shape
24
+ pad_size = (128 - (n % 128)) % 128
25
+ x = torch.nn.functional.pad(x,
26
+ (0, pad_size), value=0) if pad_size > 0 else x
27
+ x_view = x.view(m, -1, 128)
28
+ x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4)
29
+ fp8_data = (x_view *
30
+ (448.0 / x_amax.unsqueeze(2))).to(dtype=torch.float8_e4m3fn)
31
+ return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1)
32
+
33
+
34
+def per_block_cast_to_fp8(
35
+ x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
36
+ assert x.dim() == 2
37
+ m, n = x.shape
38
+ x_padded = torch.zeros((cdiv(m, 128) * 128, cdiv(n, 128) * 128),
39
+ device=x.device,
40
+ dtype=x.dtype)
41
+ x_padded[:m, :n] = x
42
+ x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128)
43
+ x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4)
44
+ x_scaled = (x_view * (448.0 / x_amax)).to(dtype=torch.float8_e4m3fn)
45
+ return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (
46
+ x_amax / 448.0).view(x_view.size(0), x_view.size(2))
47
+
48
+
49
+@pytest.mark.parametrize("num_groups, expected_m_per_group, k, n", [
50
+ (4, 8192, 7168, 4096),
51
+ (4, 8192, 2048, 7168),
52
+ (8, 4096, 7168, 4096),
53
+ (8, 4096, 2048, 7168),
54
+ (32, 1024, 7168, 4096),
55
+ (32, 1024, 2048, 7168),
56
+])
57
+@pytest.mark.parametrize("out_dtype", [torch.float16])
58
+@pytest.mark.skipif(
59
+ (lambda x: x is None or x.to_int() != 100)(
60
+ current_platform.get_device_capability()),
61
+ reason="Block Scaled Grouped GEMM is only supported on SM100.")
62
+def test_cutlass_grouped_gemm(
63
+ num_groups: int,
64
+ expected_m_per_group: int,
65
+ k: int,
66
+ n: int,
67
+ out_dtype: torch.dtype,
68
+):
69
+ device = "cuda"
70
+ alignment = 128
71
+ group_ms = [
72
+ int(expected_m_per_group * random.uniform(0.7, 1.3))
73
+ for _ in range(num_groups)
74
+ ]
75
+ m = sum([cdiv(m, alignment) * alignment for m in group_ms])
76
+
77
+ x = torch.randn((m, k), device=device, dtype=out_dtype)
78
+ y = torch.randn((num_groups, n, k), device=device, dtype=out_dtype)
79
+ out = torch.empty((m, n), device=device, dtype=out_dtype)
80
+ ref_out = torch.randn((m, n), device=device, dtype=out_dtype)
81
+
82
+ ep_offset = [0] + [sum(group_ms[:i]) for i in range(1, num_groups)] + [m]
83
+ pb_size = []
84
+ for i in range(num_groups):
85
+ pb_size.append([ep_offset[i + 1] - ep_offset[i], n, k])
86
+ problem_sizes = torch.tensor(pb_size, device=device, dtype=torch.int32)
87
+ expert_offsets = torch.tensor(ep_offset, device=device, dtype=torch.int32)
88
+
89
+ x_fp8 = per_token_cast_to_fp8(x)
90
+ y_fp8 = (torch.empty_like(y, dtype=torch.float8_e4m3fn),
91
+ torch.empty((num_groups, cdiv(n, 128), k // 128),
92
+ device=device,
93
+ dtype=torch.float))
94
+ for i in range(num_groups):
95
+ y_fp8[0][i], y_fp8[1][i] = per_block_cast_to_fp8(y[i])
96
+
97
+ for i in range(num_groups):
98
+ a = x_fp8[0][ep_offset[i]:ep_offset[i + 1]]
99
+ a_scale = x_fp8[1][ep_offset[i]:ep_offset[i + 1]]
100
+ b = y_fp8[0][i].t()
101
+ b_scale = y_fp8[1][i].t()
102
+ baseline = baseline_scaled_mm(a, b, a_scale, b_scale, out_dtype)
103
+ ref_out[ep_offset[i]:ep_offset[i + 1]] = baseline
104
+
105
+ ops.cutlass_blockwise_scaled_grouped_mm(
106
+ out,
107
+ x_fp8[0],
108
+ y_fp8[0],
109
+ x_fp8[1],
110
+ y_fp8[1],
111
+ problem_sizes,
112
+ expert_offsets[:-1],
113
+ )
114
+
115
+ torch.testing.assert_close(ref_out, out, atol=5e-1, rtol=1e-3)
vllm/_custom_ops.py
+14
@@ -646,6 +646,20 @@ def cutlass_scaled_mm_supports_fp4(cuda_device_capability: int) -> bool:
646
return torch.ops._C.cutlass_scaled_mm_supports_fp4(cuda_device_capability)
647
648
649
+def cutlass_blockwise_scaled_grouped_mm(
650
+ output: torch.Tensor,
651
+ a: torch.Tensor,
652
+ b: torch.Tensor,
653
+ scales_a: torch.Tensor,
654
+ scales_b: torch.Tensor,
655
+ problem_sizes: torch.Tensor,
656
+ expert_offsets: torch.Tensor,
657
+):
658
+ torch.ops._C.cutlass_blockwise_scaled_grouped_mm(output, a, b, scales_a,
659
+ scales_b, problem_sizes,
660
+ expert_offsets)
661
+
662
+
663
def cutlass_scaled_fp4_mm(a: torch.Tensor, b: torch.Tensor,
664
block_scale_a: torch.Tensor,
665
block_scale_b: torch.Tensor, alpha: torch.Tensor,
vllm/model_executor/layers/fused_moe/cutlass_moe.py
+133
-1
@@ -7,12 +7,17 @@ import torch
7
8
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
9
from vllm import _custom_ops as ops
10
+from vllm.logger import init_logger
11
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
12
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
13
MoEPrepareAndFinalizeNoEP)
13
-from vllm.model_executor.layers.fused_moe.utils import _fp8_perm, _resize_cache
14
+from vllm.model_executor.layers.fused_moe.utils import (_fp8_perm,
15
+ _fp8_quantize,
16
+ _resize_cache)
17
from vllm.scalar_type import scalar_types
18
19
+logger = init_logger(__name__)
20
+
21
22
def run_cutlass_moe_fp8(
23
output: torch.Tensor,
@@ -508,3 +513,130 @@ def cutlass_moe_fp4(a: torch.Tensor, a1_gscale: torch.Tensor,
513
out = (c2.view(m, num_topk, k) *
514
topk_weights.view(m, num_topk, 1).half()).sum(dim=1)
515
return out.to(dtype=out_dtype)
516
+
517
+
518
+def _valid_cutlass_block_scaled_grouped_gemm(hidden_states: torch.Tensor,
519
+ w1: torch.Tensor,
520
+ w2: torch.Tensor) -> bool:
521
+
522
+ def _valid_cutlass_block_scaled_grouped_gemm_shape(M: int, N: int, K: int):
523
+ return M >= 128 and N % 128 == 0 and K % 128 == 0
524
+
525
+ m = hidden_states.size(0)
526
+ _, K, N = w2.size()
527
+ if not _valid_cutlass_block_scaled_grouped_gemm_shape(m, N, K):
528
+ logger.debug(
529
+ "CutlassBlockScaledGroupedGemm disabled: unalinged problem size.")
530
+ return False
531
+
532
+ if (w1.dtype != torch.float8_e4m3fn or w2.dtype != torch.float8_e4m3fn):
533
+ logger.debug(
534
+ "CutlassBlockScaledGroupedGemm disabled: invalid weight dtype(s).")
535
+ return False
536
+
537
+ return True
538
+
539
+
540
+def run_cutlass_block_scaled_fused_experts(
541
+ a: torch.Tensor,
542
+ w1: torch.Tensor,
543
+ w2: torch.Tensor,
544
+ w1_scale: torch.Tensor,
545
+ w2_scale: torch.Tensor,
546
+ topk_weights: torch.Tensor,
547
+ topk_ids: torch.Tensor,
548
+) -> torch.Tensor:
549
+ w1_q = w1.transpose(1, 2)
550
+ w2_q = w2.transpose(1, 2)
551
+ w1_scale = w1_scale.transpose(1, 2)
552
+ w2_scale = w2_scale.transpose(1, 2)
553
+
554
+ assert topk_weights.shape == topk_ids.shape, "topk shape mismatch"
555
+ assert a.shape[0] == topk_ids.shape[
556
+ 0], "a and topk_ids must have the same batch size"
557
+ assert w1_q.dtype == torch.float8_e4m3fn, "w1_q must be float8_e4m3fn"
558
+ assert w2_q.dtype == torch.float8_e4m3fn, "w2_q must be float8_e4m3fn"
559
+ assert a.shape[1] == w1_q.shape[1], "Hidden size mismatch w1"
560
+ assert w1_q.shape[2] == w2_q.shape[1] * 2, "Hidden size mismatch w2"
561
+ assert w1_q.shape[0] == w2_q.shape[0], "Expert number mismatch"
562
+ assert w1_q.shape[0] == w1_scale.shape[
563
+ 0], "w1_scale expert number mismatch"
564
+ assert w1_q.shape[0] == w2_scale.shape[
565
+ 0], "w2_scale expert number mismatch"
566
+ assert a.dtype in [torch.half, torch.bfloat16], "Invalid output dtype"
567
+
568
+ out_dtype = a.dtype
569
+ num_experts = w1_q.size(0)
570
+ m = a.size(0)
571
+ k = w1_q.size(1)
572
+ n = w2_q.size(1)
573
+
574
+ expert_offsets = torch.empty((num_experts + 1, ),
575
+ dtype=torch.int32,
576
+ device="cuda")
577
+ problem_sizes1 = torch.empty((num_experts, 3),
578
+ dtype=torch.int32,
579
+ device="cuda")
580
+ problem_sizes2 = torch.empty((num_experts, 3),
581
+ dtype=torch.int32,
582
+ device="cuda")
583
+
584
+ topk = topk_ids.size(1)
585
+
586
+ a_q, a1_scale = _fp8_quantize(a,
587
+ A_scale=None,
588
+ per_act_token=False,
589
+ block_shape=[128, 128])
590
+ device = a_q.device
591
+
592
+ a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
593
+ c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
594
+
595
+ ops.get_cutlass_moe_mm_data(
596
+ topk_ids,
597
+ expert_offsets,
598
+ problem_sizes1,
599
+ problem_sizes2,
600
+ a_map,
601
+ c_map,
602
+ num_experts,
603
+ n,
604
+ k,
605
+ )
606
+
607
+ rep_a_q = a_q.view(dtype=torch.uint8)[a_map].view(dtype=a_q.dtype)
608
+ rep_a1_scales = a1_scale[a_map]
609
+
610
+ c1 = torch.empty((m * topk, n * 2), dtype=out_dtype, device=device)
611
+ c2 = torch.empty((m * topk, k), dtype=out_dtype, device=device)
612
+
613
+ ops.cutlass_blockwise_scaled_grouped_mm(
614
+ c1,
615
+ rep_a_q,
616
+ w1_q,
617
+ rep_a1_scales,
618
+ w1_scale,
619
+ problem_sizes1,
620
+ expert_offsets[:-1],
621
+ )
622
+
623
+ intermediate = torch.empty((m * topk, n), dtype=out_dtype, device=device)
624
+ torch.ops._C.silu_and_mul(intermediate, c1)
625
+
626
+ intermediate_q, a2_scale = _fp8_quantize(intermediate,
627
+ A_scale=None,
628
+ per_act_token=False,
629
+ block_shape=[128, 128])
630
+
631
+ ops.cutlass_blockwise_scaled_grouped_mm(
632
+ c2,
633
+ intermediate_q,
634
+ w2_q,
635
+ a2_scale,
636
+ w2_scale,
637
+ problem_sizes2,
638
+ expert_offsets[:-1],
639
+ )
640
+
641
+ return (c2[c_map].view(m, topk, k) *
642
+ topk_weights.view(m, topk, 1).to(out_dtype)).sum(dim=1)
vllm/model_executor/layers/fused_moe/fused_moe.py
+39
-23
@@ -15,6 +15,9 @@ from vllm.logger import init_logger
15
# yapf: disable
16
from vllm.model_executor.layers.fused_moe.config import (
17
FusedMoEQuantConfig, get_config_quant_dtype)
18
+from vllm.model_executor.layers.fused_moe.cutlass_moe import (
19
+ _valid_cutlass_block_scaled_grouped_gemm,
20
+ run_cutlass_block_scaled_fused_experts)
21
# yapf: enable
22
from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
23
_valid_deep_gemm, deep_gemm_moe_fp8)
@@ -1129,29 +1132,31 @@ def dispatch_fused_experts_func(inplace: bool) -> Callable[..., torch.Tensor]:
1132
1133
# TODO (bnell): replace this with modular op. Can get rid of inplace/outplace
1134
# torch ops.
1132
-def fused_experts(hidden_states: torch.Tensor,
1133
- w1: torch.Tensor,
1134
- w2: torch.Tensor,
1135
- topk_weights: torch.Tensor,
1136
- topk_ids: torch.Tensor,
1137
- inplace: bool = False,
1138
- activation: str = "silu",
1139
- apply_router_weight_on_input: bool = False,
1140
- use_fp8_w8a8: bool = False,
1141
- use_int8_w8a8: bool = False,
1142
- use_int8_w8a16: bool = False,
1143
- use_int4_w4a16: bool = False,
1144
- per_channel_quant: bool = False,
1145
- global_num_experts: int = -1,
1146
- expert_map: Optional[torch.Tensor] = None,
1147
- w1_scale: Optional[torch.Tensor] = None,
1148
- w2_scale: Optional[torch.Tensor] = None,
1149
- w1_zp: Optional[torch.Tensor] = None,
1150
- w2_zp: Optional[torch.Tensor] = None,
1151
- a1_scale: Optional[torch.Tensor] = None,
1152
- a2_scale: Optional[torch.Tensor] = None,
1153
- block_shape: Optional[list[int]] = None,
1154
- allow_deep_gemm: bool = False) -> torch.Tensor:
1135
+def fused_experts(
1136
+ hidden_states: torch.Tensor,
1137
+ w1: torch.Tensor,
1138
+ w2: torch.Tensor,
1139
+ topk_weights: torch.Tensor,
1140
+ topk_ids: torch.Tensor,
1141
+ inplace: bool = False,
1142
+ activation: str = "silu",
1143
+ apply_router_weight_on_input: bool = False,
1144
+ use_fp8_w8a8: bool = False,
1145
+ use_int8_w8a8: bool = False,
1146
+ use_int8_w8a16: bool = False,
1147
+ use_int4_w4a16: bool = False,
1148
+ per_channel_quant: bool = False,
1149
+ global_num_experts: int = -1,
1150
+ expert_map: Optional[torch.Tensor] = None,
1151
+ w1_scale: Optional[torch.Tensor] = None,
1152
+ w2_scale: Optional[torch.Tensor] = None,
1153
+ w1_zp: Optional[torch.Tensor] = None,
1154
+ w2_zp: Optional[torch.Tensor] = None,
1155
+ a1_scale: Optional[torch.Tensor] = None,
1156
+ a2_scale: Optional[torch.Tensor] = None,
1157
+ block_shape: Optional[list[int]] = None,
1158
+ allow_deep_gemm: bool = False,
1159
+ allow_cutlass_block_scaled_grouped_gemm: bool = False) -> torch.Tensor:
1160
# For now, disable DeepGemm for small N (<= 512) until better
1161
# permute/unpermute ops are available.
1162
N = w1.size(1)
@@ -1174,6 +1179,17 @@ def fused_experts(hidden_states: torch.Tensor,
1179
a2_scale=a2_scale,
1180
apply_router_weight_on_input=apply_router_weight_on_input,
1181
)
1182
+ elif (allow_cutlass_block_scaled_grouped_gemm and use_fp8_w8a8
1183
+ and _valid_cutlass_block_scaled_grouped_gemm(hidden_states, w1, w2)):
1184
+ assert apply_router_weight_on_input is False
1185
+ return run_cutlass_block_scaled_fused_experts(
1186
+ a=hidden_states,
1187
+ w1=w1,
1188
+ w2=w2,
1189
+ w1_scale=w1_scale,
1190
+ w2_scale=w2_scale,
1191
+ topk_weights=topk_weights,
1192
+ topk_ids=topk_ids)
1193
else:
1194
return dispatch_fused_experts_func(inplace)(
1195
hidden_states=hidden_states,
vllm/model_executor/layers/quantization/fp8.py
+19
-1
@@ -473,12 +473,30 @@ class Fp8MoEMethod(FusedMoEMethodBase):
473
logger.warning_once(
474
"DeepGemm not supported on the current platform.")
475
476
+ # Check for CutlassBlockScaledGroupedGemm support.
477
+ self.allow_cutlass_block_scaled_grouped_gemm = False
478
+ if not self.block_quant:
479
+ logger.warning_once("Model is not block quantized. Not using "
480
+ "CutlassBlockScaledGroupedGemm kernels")
481
+ elif (current_platform.is_cuda()
482
+ and current_platform.has_device_capability(100)):
483
+ logger.info_once(
484
+ "Using CutlassBlockScaledGroupedGemm kernels for Fp8MoEMethod."
485
+ )
486
+ self.allow_cutlass_block_scaled_grouped_gemm = True
487
+ else:
488
+ logger.warning_once(
489
+ "CutlassBlockScaledGroupedGemm not supported on the current "
490
+ "platform.")
491
+
492
self.topk_indices_dtype = None
493
self.fused_experts = functools.partial( # type: ignore
494
fused_experts,
495
use_fp8_w8a8=True,
496
block_shape=self.quant_config.weight_block_size,
481
- allow_deep_gemm=self.allow_deep_gemm)
497
+ allow_deep_gemm=self.allow_deep_gemm,
498
+ allow_cutlass_block_scaled_grouped_gemm=(
499
+ self.allow_cutlass_block_scaled_grouped_gemm))
500
501
def create_weights(self, layer: Module, num_experts: int, hidden_size: int,
502
intermediate_size_per_partition: int,