@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 970d6d0776
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Build][Kernel] Update CUTLASS to v3.6.0 (#11607)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Dec 30, 2024 at 04:22 UTC 970d6d0776076f17604077ba4d484cdadd604ceb
6 files changed +25 -31
CMakeLists.txt
+2 -2
@@ -223,13 +223,13 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
223 FetchContent_Declare(
224 cutlass
225 GIT_REPOSITORY https://github.com/nvidia/cutlass.git
226 - GIT_TAG 8aa95dbb888be6d81c6fbf7169718c5244b53227
226 + GIT_TAG v3.6.0
227 GIT_PROGRESS TRUE
228
229 # Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
230 # Important: If GIT_SHALLOW is enabled then GIT_TAG works only with branch names and tags.
231 # So if the GIT_TAG above is updated to a commit hash, GIT_SHALLOW must be set to FALSE
232 - GIT_SHALLOW FALSE
232 + GIT_SHALLOW TRUE
233 )
234 endif()
235 FetchContent_MakeAvailable(cutlass)
csrc/cutlass_extensions/vllm_cutlass_library_extension.py
+9 -9
@@ -14,9 +14,9 @@ class VLLMDataType(enum.Enum):
14
15
16 class MixedInputKernelScheduleType(enum.Enum):
17 - TmaWarpSpecializedMixedInput = enum_auto()
18 - TmaWarpSpecializedPingpongMixedInput = enum_auto()
19 - TmaWarpSpecializedCooperativeMixedInput = enum_auto()
17 + TmaWarpSpecialized = enum_auto()
18 + TmaWarpSpecializedPingpong = enum_auto()
19 + TmaWarpSpecializedCooperative = enum_auto()
20
21
22 VLLMDataTypeNames: Dict[Union[VLLMDataType, DataType], str] = {
@@ -68,11 +68,11 @@ VLLMKernelScheduleTag: Dict[Union[
68 MixedInputKernelScheduleType, KernelScheduleType], str] = {
69 **KernelScheduleTag, # type: ignore
70 **{
71 - MixedInputKernelScheduleType.TmaWarpSpecializedMixedInput:
72 - "cutlass::gemm::KernelTmaWarpSpecializedMixedInput",
73 - MixedInputKernelScheduleType.TmaWarpSpecializedPingpongMixedInput:
74 - "cutlass::gemm::KernelTmaWarpSpecializedPingpongMixedInput",
75 - MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput:
76 - "cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput",
71 + MixedInputKernelScheduleType.TmaWarpSpecialized:
72 + "cutlass::gemm::KernelTmaWarpSpecialized",
73 + MixedInputKernelScheduleType.TmaWarpSpecializedPingpong:
74 + "cutlass::gemm::KernelTmaWarpSpecializedPingpong",
75 + MixedInputKernelScheduleType.TmaWarpSpecializedCooperative:
76 + "cutlass::gemm::KernelTmaWarpSpecializedCooperative",
77 }
78 }
csrc/quantization/machete/generate.py
+4 -4
@@ -189,7 +189,7 @@ using Kernel_{{type_sig}} = MacheteKernelTemplate<
189 {{DataTypeTag[t.b_group_zeropoint]}}, // GroupZeroT
190 {{DataTypeTag[t.b_channel_scale]}}, // ChannelScaleT
191 {{DataTypeTag[t.a_token_scale]}}, // TokenScaleT
192 - cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput,
192 + cutlass::gemm::KernelTmaWarpSpecializedCooperative,
193 Sch>;
194
195 {% for sch in schs %}
@@ -223,7 +223,7 @@ torch::Tensor prepack_B_dispatch(PrepackBArgs args) {
223 {{DataTypeTag[t.convert]}}, // ElementConvert
224 {{DataTypeTag[t.accumulator]}}, // Accumulator
225 cutlass::layout::ColumnMajor,
226 - cutlass::gemm::KernelTmaWarpSpecializedCooperativeMixedInput>
226 + cutlass::gemm::KernelTmaWarpSpecializedCooperative>
227 >(args.B);
228 }
229 {%- endfor %}
@@ -239,7 +239,7 @@ torch::Tensor prepack_B_dispatch(PrepackBArgs args) {
239 }; // namespace machete
240 """
241
242 -TmaMI = MixedInputKernelScheduleType.TmaWarpSpecializedCooperativeMixedInput
242 +TmaMI = MixedInputKernelScheduleType.TmaWarpSpecializedCooperative
243 TmaCoop = EpilogueScheduleType.TmaWarpSpecializedCooperative
244
245
@@ -300,7 +300,7 @@ def generate_sch_sig(schedule_config: ScheduleConfig) -> str:
300 # mostly unique shorter sch_sig
301 def generate_terse_sch_sig(schedule_config: ScheduleConfig) -> str:
302 kernel_terse_names_replace = {
303 - "KernelTmaWarpSpecializedCooperativeMixedInput_": "TmaMI_",
303 + "KernelTmaWarpSpecializedCooperative": "TmaMI_",
304 "TmaWarpSpecializedCooperative_": "TmaCoop_",
305 "StreamKScheduler": "streamK",
306 }
csrc/quantization/machete/machete_collective_builder.cuh
+4 -6
@@ -18,16 +18,14 @@ struct VLLMCollectiveBuilder<
18 ElementAccumulator, TileShape_MNK, ClusterShape_MNK, StageCountType,
19 KernelScheduleType,
20 cute::enable_if_t<(
21 + cute::is_same_v<KernelScheduleType, KernelTmaWarpSpecialized> ||
22 + cute::is_same_v<KernelScheduleType, KernelTmaWarpSpecializedPingpong> ||
23 cute::is_same_v<KernelScheduleType,
22 - KernelTmaWarpSpecializedMixedInput> ||
23 - cute::is_same_v<KernelScheduleType,
24 - KernelTmaWarpSpecializedPingpongMixedInput> ||
25 - cute::is_same_v<KernelScheduleType,
26 - KernelTmaWarpSpecializedCooperativeMixedInput>)>> {
24 + KernelTmaWarpSpecializedCooperative>)>> {
25 using CollectiveOp = machete::MacheteCollectiveMma<
26 ElementPairA_, GmemLayoutA_, AlignmentA, ElementPairB_, GmemLayoutB_,
27 AlignmentB, ElementAccumulator, TileShape_MNK, ClusterShape_MNK,
28 StageCountType, KernelScheduleType>;
29 };
30
33 -}; // namespace cutlass::gemm::collective
\ No newline at end of file
31 +}; // namespace cutlass::gemm::collective
csrc/quantization/machete/machete_mainloop.cuh
+4 -7
@@ -66,13 +66,11 @@ struct MacheteCollectiveMma {
66 using Schedule = KernelScheduleType;
67 static_assert(
68 cute::is_same_v<Schedule, KernelTmaWarpSpecialized> ||
69 - cute::is_same_v<Schedule, KernelTmaWarpSpecializedMixedInput> ||
69 + cute::is_same_v<Schedule, KernelTmaWarpSpecialized> ||
70 + cute::is_same_v<Schedule, KernelTmaWarpSpecializedPingpong> ||
71 cute::is_same_v<Schedule, KernelTmaWarpSpecializedPingpong> ||
71 - cute::is_same_v<Schedule,
72 - KernelTmaWarpSpecializedPingpongMixedInput> ||
72 cute::is_same_v<Schedule, KernelTmaWarpSpecializedCooperative> ||
74 - cute::is_same_v<Schedule,
75 - KernelTmaWarpSpecializedCooperativeMixedInput>,
73 + cute::is_same_v<Schedule, KernelTmaWarpSpecializedCooperative>,
74 "KernelSchedule must be one of the warp specialized policies");
75
76 public:
@@ -113,8 +111,7 @@ struct MacheteCollectiveMma {
111 // For coop schedules we have two warp groups cooperatively issuing wgmma
112 // instructions so we use 2 atoms along the M dim (one for each warpgroup)
113 using AtomLayoutMNK = cute::conditional_t<
116 - cute::is_same_v<KernelScheduleType,
117 - KernelTmaWarpSpecializedCooperativeMixedInput>,
114 + cute::is_same_v<KernelScheduleType, KernelTmaWarpSpecializedCooperative>,
115 Layout<Shape<_2, _1, _1>>, Layout<Shape<_1, _1, _1>>>;
116
117 using TiledMma = decltype(cute::make_tiled_mma(
csrc/quantization/machete/machete_prepacked_layout.cuh
+2 -3
@@ -98,8 +98,7 @@ struct PrepackedLayoutBTemplate {
98 // For coop schedules we have two warp groups cooperatively issuing wgmma
99 // instructions so we use 2 atoms along the M dim (one for each warpgroup)
100 using AtomLayoutMNK = cute::conditional_t<
101 - cute::is_same_v<KernelSchedule,
102 - KernelTmaWarpSpecializedCooperativeMixedInput>,
101 + cute::is_same_v<KernelSchedule, KernelTmaWarpSpecializedCooperative>,
102 Layout<Shape<_2, _1, _1>>, Layout<Shape<_1, _1, _1>>>;
103
104 using TiledMma = decltype(cute::make_tiled_mma(
@@ -247,4 +246,4 @@ struct PrepackedLayoutBTemplate {
246 }
247 };
248
250 -}; // namespace machete
\ No newline at end of file
249 +}; // namespace machete