A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Build][Kernel] Update CUTLASS to v3.6.0 (#11607)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Dec 30, 2024 at 04:22 UTC
970d6d0776076f17604077ba4d484cdadd604ceb
6 files changed
+25
-31
CMakeLists.txt
+2
-2
@@ -223,13 +223,13 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
223
FetchContent_Declare(
224
cutlass
225
GIT_REPOSITORY https://github.com/nvidia/cutlass.git
226
- GIT_TAG 8aa95dbb888be6d81c6fbf7169718c5244b53227
226
+ GIT_TAG v3.6.0
227
GIT_PROGRESS TRUE
228
229
# Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
230
# Important: If GIT_SHALLOW is enabled then GIT_TAG works only with branch names and tags.
231
# So if the GIT_TAG above is updated to a commit hash, GIT_SHALLOW must be set to FALSE
232
- GIT_SHALLOW FALSE
232
+ GIT_SHALLOW TRUE
233
)
234
endif()
235
FetchContent_MakeAvailable(cutlass)
csrc/cutlass_extensions/vllm_cutlass_library_extension.py
+9
-9
@@ -14,9 +14,9 @@ class VLLMDataType(enum.Enum):
14
15
16
class MixedInputKernelScheduleType(enum.Enum):
17
- TmaWarpSpecializedMixedInput = enum_auto()
18
- TmaWarpSpecializedPingpongMixedInput = enum_auto()
19
- TmaWarpSpecializedCooperativeMixedInput = enum_auto()
17
+ TmaWarpSpecialized = enum_auto()
18
+ TmaWarpSpecializedPingpong = enum_auto()
19
+ TmaWarpSpecializedCooperative = enum_auto()
20
21
22
VLLMDataTypeNames: Dict[Union[VLLMDataType, DataType], str] = {
@@ -68,11 +68,11 @@ VLLMKernelScheduleTag: Dict[Union[
68
MixedInputKernelScheduleType, KernelScheduleType], str] = {
69
**KernelScheduleTag, # type: ignore
70
**{
71
- MixedInputKernelScheduleType.TmaWarpSpecializedMixedInput:
72
- "cutlass::gemm::KernelTmaWarpSpecializedMixedInput",
73
- MixedInputKernelScheduleType.TmaWarpSpecializedPingpongMixedInput:
74
- "cutlass::gemm::KernelTmaWarpSpecializedPingpongMixedInput",
75
- MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput:
76
- "cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput",
71
+ MixedInputKernelScheduleType.TmaWarpSpecialized:
72
+ "cutlass::gemm::KernelTmaWarpSpecialized",
73
+ MixedInputKernelScheduleType.TmaWarpSpecializedPingpong:
74
+ "cutlass::gemm::KernelTmaWarpSpecializedPingpong",
75
+ MixedInputKernelScheduleType.TmaWarpSpecializedCooperative:
76
+ "cutlass::gemm::KernelTmaWarpSpecializedCooperative",
77
}
78
}
csrc/quantization/machete/generate.py
+4
-4
@@ -189,7 +189,7 @@ using Kernel_{{type_sig}} = MacheteKernelTemplate<
189
{{DataTypeTag[t.b_group_zeropoint]}}, // GroupZeroT
190
{{DataTypeTag[t.b_channel_scale]}}, // ChannelScaleT
191
{{DataTypeTag[t.a_token_scale]}}, // TokenScaleT
192
- cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput,
192
+ cutlass::gemm::KernelTmaWarpSpecializedCooperative,
193
Sch>;
194
195
{% for sch in schs %}
@@ -223,7 +223,7 @@ torch::Tensor prepack_B_dispatch(PrepackBArgs args) {
223
{{DataTypeTag[t.convert]}}, // ElementConvert
224
{{DataTypeTag[t.accumulator]}}, // Accumulator
225
cutlass::layout::ColumnMajor,
226
- cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput>
226
+ cutlass::gemm::KernelTmaWarpSpecializedCooperative>
227
>(args.B);
228
}
229
{%- endfor %}
@@ -239,7 +239,7 @@ torch::Tensor prepack_B_dispatch(PrepackBArgs args) {
239
}; // namespace machete
240
"""
241
242
-TmaMI = MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput
242
+TmaMI = MixedInputKernelScheduleType.TmaWarpSpecializedCooperative
243
TmaCoop = EpilogueScheduleType.TmaWarpSpecializedCooperative
244
245
@@ -300,7 +300,7 @@ def generate_sch_sig(schedule_config: ScheduleConfig) -> str:
300
# mostly unique shorter sch_sig
301
def generate_terse_sch_sig(schedule_config: ScheduleConfig) -> str:
302
kernel_terse_names_replace = {
303
- "KernelTmaWarpSpecializedCooperativeMixedInput_": "TmaMI_",
303
+ "KernelTmaWarpSpecializedCooperative": "TmaMI_",
304
"TmaWarpSpecializedCooperative_": "TmaCoop_",
305
"StreamKScheduler": "streamK",
306
}
csrc/quantization/machete/machete_collective_builder.cuh
+4
-6
@@ -18,16 +18,14 @@ struct VLLMCollectiveBuilder<
18
ElementAccumulator, TileShape_MNK, ClusterShape_MNK, StageCountType,
19
KernelScheduleType,
20
cute::enable_if_t<(
21
+ cute::is_same_v<KernelScheduleType, KernelTmaWarpSpecialized> ||
22
+ cute::is_same_v<KernelScheduleType, KernelTmaWarpSpecializedPingpong> ||
23
cute::is_same_v<KernelScheduleType,
22
- KernelTmaWarpSpecializedMixedInput> ||
23
- cute::is_same_v<KernelScheduleType,
24
- KernelTmaWarpSpecializedPingpongMixedInput> ||
25
- cute::is_same_v<KernelScheduleType,
26
- KernelTmaWarpSpecializedCooperativeMixedInput>)>> {
24
+ KernelTmaWarpSpecializedCooperative>)>> {
25
using CollectiveOp = machete::MacheteCollectiveMma<
26
ElementPairA_, GmemLayoutA_, AlignmentA, ElementPairB_, GmemLayoutB_,
27
AlignmentB, ElementAccumulator, TileShape_MNK, ClusterShape_MNK,
28
StageCountType, KernelScheduleType>;
29
};
30
33
-}; // namespace cutlass::gemm::collective
\ No newline at end of file
31
+}; // namespace cutlass::gemm::collective
csrc/quantization/machete/machete_mainloop.cuh
+4
-7
@@ -66,13 +66,11 @@ struct MacheteCollectiveMma {
66
using Schedule = KernelScheduleType;
67
static_assert(
68
cute::is_same_v<Schedule, KernelTmaWarpSpecialized> ||
69
- cute::is_same_v<Schedule, KernelTmaWarpSpecializedMixedInput> ||
69
+ cute::is_same_v<Schedule, KernelTmaWarpSpecialized> ||
70
+ cute::is_same_v<Schedule, KernelTmaWarpSpecializedPingpong> ||
71
cute::is_same_v<Schedule, KernelTmaWarpSpecializedPingpong> ||
71
- cute::is_same_v<Schedule,
72
- KernelTmaWarpSpecializedPingpongMixedInput> ||
72
cute::is_same_v<Schedule, KernelTmaWarpSpecializedCooperative> ||
74
- cute::is_same_v<Schedule,
75
- KernelTmaWarpSpecializedCooperativeMixedInput>,
73
+ cute::is_same_v<Schedule, KernelTmaWarpSpecializedCooperative>,
74
"KernelSchedule must be one of the warp specialized policies");
75
76
public:
@@ -113,8 +111,7 @@ struct MacheteCollectiveMma {
111
// For coop schedules we have two warp groups cooperatively issuing wgmma
112
// instructions so we use 2 atoms along the M dim (one for each warpgroup)
113
using AtomLayoutMNK = cute::conditional_t<
116
- cute::is_same_v<KernelScheduleType,
117
- KernelTmaWarpSpecializedCooperativeMixedInput>,
114
+ cute::is_same_v<KernelScheduleType, KernelTmaWarpSpecializedCooperative>,
115
Layout<Shape<_2, _1, _1>>, Layout<Shape<_1, _1, _1>>>;
116
117
using TiledMma = decltype(cute::make_tiled_mma(
csrc/quantization/machete/machete_prepacked_layout.cuh
+2
-3
@@ -98,8 +98,7 @@ struct PrepackedLayoutBTemplate {
98
// For coop schedules we have two warp groups cooperatively issuing wgmma
99
// instructions so we use 2 atoms along the M dim (one for each warpgroup)
100
using AtomLayoutMNK = cute::conditional_t<
101
- cute::is_same_v<KernelSchedule,
102
- KernelTmaWarpSpecializedCooperativeMixedInput>,
101
+ cute::is_same_v<KernelSchedule, KernelTmaWarpSpecializedCooperative>,
102
Layout<Shape<_2, _1, _1>>, Layout<Shape<_1, _1, _1>>>;
103
104
using TiledMma = decltype(cute::make_tiled_mma(
@@ -247,4 +246,4 @@ struct PrepackedLayoutBTemplate {
246
}
247
};
248
250
-}; // namespace machete
\ No newline at end of file
249
+}; // namespace machete