A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[CI/Build] Per file CUDA Archs (improve wheel size and dev build times) (#8845)
Lucas Wilkinson committed
Oct 3, 2024 at 22:55 UTC
aeb37c2a725554791ff6f258b1e18830867a3ab9
22 files changed
+828
-370
CMakeLists.txt
+169
-55
@@ -143,6 +143,19 @@ else()
143
message(FATAL_ERROR "Can't find CUDA or HIP installation.")
144
endif()
145
146
+
147
+#
148
+# For cuda we want to be able to control which architectures we compile for on
149
+# a per-file basis in order to cut down on compile time. So here we extract
150
+# the set of architectures we want to compile for and remove the from the
151
+# CMAKE_CUDA_FLAGS so that they are not applied globally.
152
+#
153
+if(VLLM_GPU_LANG STREQUAL "CUDA")
154
+ clear_cuda_arches(CUDA_ARCH_FLAGS)
155
+ extract_unique_cuda_archs_ascending(CUDA_ARCHS "${CUDA_ARCH_FLAGS}")
156
+ message(STATUS "CUDA target architectures: ${CUDA_ARCHS}")
157
+endif()
158
+
159
#
160
# Override the GPU architectures detected by cmake/torch and filter them by
161
# the supported versions for the current language.
@@ -223,30 +236,89 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
236
"csrc/mamba/causal_conv1d/causal_conv1d.cu"
237
"csrc/quantization/aqlm/gemm_kernels.cu"
238
"csrc/quantization/awq/gemm_kernels.cu"
226
- "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
227
- "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
228
- "csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu"
229
- "csrc/quantization/gptq_marlin/gptq_marlin.cu"
230
- "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
231
- "csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
239
"csrc/quantization/gguf/gguf_kernel.cu"
233
- "csrc/quantization/fp8/fp8_marlin.cu"
240
"csrc/custom_all_reduce.cu"
241
"csrc/permute_cols.cu"
236
- "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
237
- "csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu"
238
- "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
242
+ "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu")
243
+
244
+ set_gencode_flags_for_srcs(
245
+ SRCS "${VLLM_EXT_SRC}"
246
+ CUDA_ARCHS "${CUDA_ARCHS}")
247
+
248
+ # Only build Marlin kernels if we are building for at least some compatible archs.
249
+ # Keep building Marlin for 9.0 as there are some group sizes and shapes that
250
+ # are not supported by Machete yet.
251
+ cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.9;9.0" ${CUDA_ARCHS})
252
+ if (MARLIN_ARCHS)
253
+ set(MARLIN_SRCS
254
+ "csrc/quantization/fp8/fp8_marlin.cu"
255
+ "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
256
+ "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
257
+ "csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu"
258
+ "csrc/quantization/gptq_marlin/gptq_marlin.cu"
259
+ "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
260
+ "csrc/quantization/gptq_marlin/awq_marlin_repack.cu")
261
+ set_gencode_flags_for_srcs(
262
+ SRCS "${MARLIN_SRCS}"
263
+ CUDA_ARCHS "${MARLIN_ARCHS}")
264
+ list(APPEND VLLM_EXT_SRC "${MARLIN_SRCS}")
265
+ message(STATUS "Building Marlin kernels for archs: ${MARLIN_ARCHS}")
266
+ else()
267
+ message(STATUS "Not building Marlin kernels as no compatible archs found"
268
+ "in CUDA target architectures")
269
+ endif()
270
+
271
+ #
272
+ # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
273
+ # CUDA 12.0 or later (and only work on Hopper, 9.0/9.0a for now).
274
+ cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0;9.0a" "${CUDA_ARCHS}")
275
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
276
+ set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
277
+ set_gencode_flags_for_srcs(
278
+ SRCS "${SRCS}"
279
+ CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
280
+ list(APPEND VLLM_EXT_SRC "${SRCS}")
281
+ list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_C3X=1")
282
+ message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
283
+ else()
284
+ # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
285
+ # build any 3x kernels
286
+ set(SCALED_MM_3X_ARCHS)
287
+
288
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
289
+ message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
290
+ "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
291
+ "later if you intend on running FP8 quantized models on "
292
+ "Hopper.")
293
+ else()
294
+ message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
295
+ "in CUDA target architectures")
296
+ endif()
297
+ endif()
298
299
#
241
- # The CUTLASS kernels for Hopper require sm90a to be enabled.
242
- # This is done via the below gencode option, BUT that creates kernels for both sm90 and sm90a.
243
- # That adds an extra 17MB to compiled binary, so instead we selectively enable it.
244
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
245
- set_source_files_properties(
246
- "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
247
- PROPERTIES
248
- COMPILE_FLAGS
249
- "-gencode arch=compute_90a,code=sm_90a")
300
+ # For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
301
+ # kernels for the remaining archs that are not already built for 3x.
302
+ cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
303
+ "7.5;8.0;8.6;8.9;9.0;9.0a" "${CUDA_ARCHS}")
304
+ # subtract out the archs that are already built for 3x
305
+ list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
306
+ if (SCALED_MM_2X_ARCHS)
307
+ set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu")
308
+ set_gencode_flags_for_srcs(
309
+ SRCS "${SRCS}"
310
+ CUDA_ARCHS "${SCALED_MM_2X_ARCHS}")
311
+ list(APPEND VLLM_EXT_SRC "${SRCS}")
312
+ list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_C2X=1")
313
+ message(STATUS "Building scaled_mm_c2x for archs: ${SCALED_MM_2X_ARCHS}")
314
+ else()
315
+ if (SCALED_MM_3X_ARCHS)
316
+ message(STATUS "Not building scaled_mm_c2x as all archs are already built"
317
+ " for and covered by scaled_mm_c3x")
318
+ else()
319
+ message(STATUS "Not building scaled_mm_c2x as no compatible archs found "
320
+ "in CUDA target architectures")
321
+ endif()
322
endif()
323
324
@@ -254,47 +326,72 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
326
# Machete kernels
327
328
# The machete kernels only work on hopper and require CUDA 12.0 or later.
257
- if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
329
+ # Only build Machete kernels if we are building for something compatible with sm90a
330
+ cuda_archs_loose_intersection(MACHETE_ARCHS "9.0a" "${CUDA_ARCHS}")
331
+ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND MACHETE_ARCHS)
332
#
333
# For the Machete kernels we automatically generate sources for various
334
# preselected input type pairs and schedules.
335
# Generate sources:
262
- execute_process(
263
- COMMAND ${CMAKE_COMMAND} -E env
264
- PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
265
- ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py
266
- RESULT_VARIABLE machete_generation_result
267
- OUTPUT_VARIABLE machete_generation_output
268
- OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
269
- ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
270
- )
271
-
272
- if (NOT machete_generation_result EQUAL 0)
273
- message(FATAL_ERROR "Machete generation failed."
274
- " Result: \"${machete_generation_result}\""
275
- "\nCheck the log for details: "
276
- "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
336
+ set(MACHETE_GEN_SCRIPT
337
+ ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py)
338
+ file(MD5 ${MACHETE_GEN_SCRIPT} MACHETE_GEN_SCRIPT_HASH)
339
+
340
+ message(STATUS "Machete generation script hash: ${MACHETE_GEN_SCRIPT_HASH}")
341
+ message(STATUS "Last run machete generate script hash: $CACHE{MACHETE_GEN_SCRIPT_HASH}")
342
+
343
+ if (NOT DEFINED CACHE{MACHETE_GEN_SCRIPT_HASH}
344
+ OR NOT $CACHE{MACHETE_GEN_SCRIPT_HASH} STREQUAL ${MACHETE_GEN_SCRIPT_HASH})
345
+ execute_process(
346
+ COMMAND ${CMAKE_COMMAND} -E env
347
+ PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
348
+ ${Python_EXECUTABLE} ${MACHETE_GEN_SCRIPT}
349
+ RESULT_VARIABLE machete_generation_result
350
+ OUTPUT_VARIABLE machete_generation_output
351
+ OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
352
+ ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
353
+ )
354
+
355
+ if (NOT machete_generation_result EQUAL 0)
356
+ message(FATAL_ERROR "Machete generation failed."
357
+ " Result: \"${machete_generation_result}\""
358
+ "\nCheck the log for details: "
359
+ "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
360
+ else()
361
+ set(MACHETE_GEN_SCRIPT_HASH ${MACHETE_GEN_SCRIPT_HASH}
362
+ CACHE STRING "Last run machete generate script hash" FORCE)
363
+ message(STATUS "Machete generation completed successfully.")
364
+ endif()
365
else()
278
- message(STATUS "Machete generation completed successfully.")
366
+ message(STATUS "Machete generation script has not changed, skipping generation.")
367
endif()
368
369
# Add machete generated sources
370
file(GLOB MACHETE_GEN_SOURCES "csrc/quantization/machete/generated/*.cu")
371
list(APPEND VLLM_EXT_SRC ${MACHETE_GEN_SOURCES})
284
- message(STATUS "Machete generated sources: ${MACHETE_GEN_SOURCES}")
372
286
- set_source_files_properties(
287
- ${MACHETE_GEN_SOURCES}
288
- PROPERTIES
289
- COMPILE_FLAGS
290
- "-gencode arch=compute_90a,code=sm_90a")
373
+ # forward compatible
374
+ set_gencode_flags_for_srcs(
375
+ SRCS "${MACHETE_GEN_SOURCES}"
376
+ CUDA_ARCHS "${MACHETE_ARCHS}")
377
+
378
+ list(APPEND VLLM_EXT_SRC
379
+ csrc/quantization/machete/machete_pytorch.cu)
380
+
381
+ message(STATUS "Building Machete kernels for archs: ${MACHETE_ARCHS}")
382
+ else()
383
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
384
+ AND MACHETE_ARCHS)
385
+ message(STATUS "Not building Machete kernels as CUDA Compiler version is "
386
+ "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
387
+ "later if you intend on running w4a16 quantized models on "
388
+ "Hopper.")
389
+ else()
390
+ message(STATUS "Not building Machete kernels as no compatible archs "
391
+ "found in CUDA target architectures")
392
+ endif()
393
endif()
292
-
293
- # Add pytorch binding for machete (add on even CUDA < 12.0 so that we can
294
- # raise an error if the user that this was built with an incompatible
295
- # CUDA version)
296
- list(APPEND VLLM_EXT_SRC
297
- csrc/quantization/machete/machete_pytorch.cu)
394
+# if CUDA endif
395
endif()
396
397
message(STATUS "Enabling C extension.")
@@ -323,14 +420,31 @@ set(VLLM_MOE_EXT_SRC
420
"csrc/moe/torch_bindings.cpp"
421
"csrc/moe/topk_softmax_kernels.cu")
422
423
+set_gencode_flags_for_srcs(
424
+ SRCS "${VLLM_MOE_EXT_SRC}"
425
+ CUDA_ARCHS "${CUDA_ARCHS}")
426
+
427
if(VLLM_GPU_LANG STREQUAL "CUDA")
327
- list(APPEND VLLM_MOE_EXT_SRC
328
- "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
329
- "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h"
330
- "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
331
- "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
332
- "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
333
- "csrc/moe/marlin_moe_ops.cu")
428
+ cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
429
+ if (MARLIN_MOE_ARCHS)
430
+ set(MARLIN_MOE_SRC
431
+ "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
432
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h"
433
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
434
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
435
+ "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
436
+ "csrc/moe/marlin_moe_ops.cu")
437
+
438
+ set_gencode_flags_for_srcs(
439
+ SRCS "${MARLIN_MOE_SRC}"
440
+ CUDA_ARCHS "${MARLIN_MOE_ARCHS}")
441
+
442
+ list(APPEND VLLM_MOE_EXT_SRC "${MARLIN_MOE_SRC}")
443
+ message(STATUS "Building Marlin MOE kernels for archs: ${MARLIN_MOE_ARCHS}")
444
+ else()
445
+ message(STATUS "Not building Marlin MOE kernels as no compatible archs found"
446
+ "in CUDA target architectures")
447
+ endif()
448
endif()
449
450
message(STATUS "Enabling moe extension.")
cmake/utils.cmake
+172
-103
@@ -133,10 +133,181 @@ macro(string_to_ver OUT_VER IN_STR)
133
string(REGEX REPLACE "\([0-9]+\)\([0-9]\)" "\\1.\\2" ${OUT_VER} ${IN_STR})
134
endmacro()
135
136
+#
137
+# Clear all `-gencode` flags from `CMAKE_CUDA_FLAGS` and store them in
138
+# `CUDA_ARCH_FLAGS`.
139
+#
140
+# Example:
141
+# CMAKE_CUDA_FLAGS="-Wall -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75"
142
+# clear_cuda_arches(CUDA_ARCH_FLAGS)
143
+# CUDA_ARCH_FLAGS="-gencode arch=compute_70,code=sm_70;-gencode arch=compute_75,code=sm_75"
144
+# CMAKE_CUDA_FLAGS="-Wall"
145
+#
146
+macro(clear_cuda_arches CUDA_ARCH_FLAGS)
147
+ # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS`
148
+ string(REGEX MATCHALL "-gencode arch=[^ ]+" CUDA_ARCH_FLAGS
149
+ ${CMAKE_CUDA_FLAGS})
150
+
151
+ # Remove all `-gencode` flags from `CMAKE_CUDA_FLAGS` since they will be modified
152
+ # and passed back via the `CUDA_ARCHITECTURES` property.
153
+ string(REGEX REPLACE "-gencode arch=[^ ]+ *" "" CMAKE_CUDA_FLAGS
154
+ ${CMAKE_CUDA_FLAGS})
155
+endmacro()
156
+
157
+#
158
+# Extract unique CUDA architectures from a list of compute capabilities codes in
159
+# the form `<major><minor>[<letter>]`, convert them to the form sort
160
+# `<major>.<minor>`, dedupes them and then sorts them in ascending order and
161
+# stores them in `OUT_ARCHES`.
162
+#
163
+# Example:
164
+# CUDA_ARCH_FLAGS="-gencode arch=compute_75,code=sm_75;...;-gencode arch=compute_90a,code=sm_90a"
165
+# extract_unique_cuda_archs_ascending(OUT_ARCHES CUDA_ARCH_FLAGS)
166
+# OUT_ARCHES="7.5;...;9.0"
167
+function(extract_unique_cuda_archs_ascending OUT_ARCHES CUDA_ARCH_FLAGS)
168
+ set(_CUDA_ARCHES)
169
+ foreach(_ARCH ${CUDA_ARCH_FLAGS})
170
+ string(REGEX MATCH "arch=compute_\([0-9]+a?\)" _COMPUTE ${_ARCH})
171
+ if (_COMPUTE)
172
+ set(_COMPUTE ${CMAKE_MATCH_1})
173
+ endif()
174
+
175
+ string_to_ver(_COMPUTE_VER ${_COMPUTE})
176
+ list(APPEND _CUDA_ARCHES ${_COMPUTE_VER})
177
+ endforeach()
178
+
179
+ list(REMOVE_DUPLICATES _CUDA_ARCHES)
180
+ list(SORT _CUDA_ARCHES COMPARE NATURAL ORDER ASCENDING)
181
+ set(${OUT_ARCHES} ${_CUDA_ARCHES} PARENT_SCOPE)
182
+endfunction()
183
+
184
+#
185
+# For a specific file set the `-gencode` flag in compile options conditionally
186
+# for the CUDA language.
187
+#
188
+# Example:
189
+# set_gencode_flag_for_srcs(
190
+# SRCS "foo.cu"
191
+# ARCH "compute_75"
192
+# CODE "sm_75")
193
+# adds: "-gencode arch=compute_75,code=sm_75" to the compile options for
194
+# `foo.cu` (only for the CUDA language).
195
+#
196
+macro(set_gencode_flag_for_srcs)
197
+ set(options)
198
+ set(oneValueArgs ARCH CODE)
199
+ set(multiValueArgs SRCS)
200
+ cmake_parse_arguments(arg "${options}" "${oneValueArgs}"
201
+ "${multiValueArgs}" ${ARGN} )
202
+ set(_FLAG -gencode arch=${arg_ARCH},code=${arg_CODE})
203
+ set_property(
204
+ SOURCE ${arg_SRCS}
205
+ APPEND PROPERTY
206
+ COMPILE_OPTIONS "$<$<COMPILE_LANGUAGE:CUDA>:${_FLAG}>"
207
+ )
208
+
209
+ message(DEBUG "Setting gencode flag for ${arg_SRCS}: ${_FLAG}")
210
+endmacro(set_gencode_flag_for_srcs)
211
+
212
+#
213
+# For a list of source files set the `-gencode` flags in the files specific
214
+# compile options (specifically for the CUDA language).
215
+#
216
+# arguments are:
217
+# SRCS: list of source files
218
+# CUDA_ARCHS: list of CUDA architectures in the form `<major>.<minor>[letter]`
219
+# BUILD_PTX_FOR_ARCH: if set to true, then the PTX code will be built
220
+# for architecture `BUILD_PTX_FOR_ARCH` if there is a CUDA_ARCH in CUDA_ARCHS
221
+# that is larger than BUILD_PTX_FOR_ARCH.
222
+#
223
+macro(set_gencode_flags_for_srcs)
224
+ set(options)
225
+ set(oneValueArgs BUILD_PTX_FOR_ARCH)
226
+ set(multiValueArgs SRCS CUDA_ARCHS)
227
+ cmake_parse_arguments(arg "${options}" "${oneValueArgs}"
228
+ "${multiValueArgs}" ${ARGN} )
229
+
230
+ foreach(_ARCH ${arg_CUDA_ARCHS})
231
+ string(REPLACE "." "" _ARCH "${_ARCH}")
232
+ set_gencode_flag_for_srcs(
233
+ SRCS ${arg_SRCS}
234
+ ARCH "compute_${_ARCH}"
235
+ CODE "sm_${_ARCH}")
236
+ endforeach()
237
+
238
+ if (${arg_BUILD_PTX_FOR_ARCH})
239
+ list(SORT arg_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
240
+ list(GET arg_CUDA_ARCHS -1 _HIGHEST_ARCH)
241
+ if (_HIGHEST_ARCH VERSION_GREATER_EQUAL ${arg_BUILD_PTX_FOR_ARCH})
242
+ string(REPLACE "." "" _PTX_ARCH "${arg_BUILD_PTX_FOR_ARCH}")
243
+ set_gencode_flag_for_srcs(
244
+ SRCS ${arg_SRCS}
245
+ ARCH "compute_${_PTX_ARCH}"
246
+ CODE "compute_${_PTX_ARCH}")
247
+ endif()
248
+ endif()
249
+endmacro()
250
+
251
+#
252
+# For the given `SRC_CUDA_ARCHS` list of gencode versions in the form
253
+# `<major>.<minor>[letter]` compute the "loose intersection" with the
254
+# `TGT_CUDA_ARCHS` list of gencodes.
255
+# The loose intersection is defined as:
256
+# { max{ x \in tgt | x <= y } | y \in src, { x \in tgt | x <= y } != {} }
257
+# where `<=` is the version comparison operator.
258
+# In other words, for each version in `TGT_CUDA_ARCHS` find the highest version
259
+# in `SRC_CUDA_ARCHS` that is less or equal to the version in `TGT_CUDA_ARCHS`.
260
+# We have special handling for 9.0a, if 9.0a is in `SRC_CUDA_ARCHS` and 9.0 is
261
+# in `TGT_CUDA_ARCHS` then we should remove 9.0a from `SRC_CUDA_ARCHS` and add
262
+# 9.0a to the result.
263
+# The result is stored in `OUT_CUDA_ARCHS`.
264
+#
265
+# Example:
266
+# SRC_CUDA_ARCHS="7.5;8.0;8.6;9.0;9.0a"
267
+# TGT_CUDA_ARCHS="8.0;8.9;9.0"
268
+# cuda_archs_loose_intersection(OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_ARCHS)
269
+# OUT_CUDA_ARCHS="8.0;8.6;9.0;9.0a"
270
+#
271
+function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_ARCHS)
272
+ list(REMOVE_DUPLICATES SRC_CUDA_ARCHS)
273
+
274
+ # if 9.0a is in SRC_CUDA_ARCHS and 9.0 is in CUDA_ARCHS then we should
275
+ # remove 9.0a from SRC_CUDA_ARCHS and add 9.0a to _CUDA_ARCHS
276
+ set(_CUDA_ARCHS)
277
+ if ("9.0a" IN_LIST SRC_CUDA_ARCHS)
278
+ list(REMOVE_ITEM SRC_CUDA_ARCHS "9.0a")
279
+ if ("9.0" IN_LIST TGT_CUDA_ARCHS)
280
+ set(_CUDA_ARCHS "9.0a")
281
+ endif()
282
+ endif()
283
+
284
+ list(SORT SRC_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
285
+
286
+ # for each ARCH in CUDA_ARCHS find the highest arch in SRC_CUDA_ARCHS that is
287
+ # less or eqault to ARCH
288
+ foreach(_ARCH ${CUDA_ARCHS})
289
+ set(_TMP_ARCH)
290
+ foreach(_SRC_ARCH ${SRC_CUDA_ARCHS})
291
+ if (_SRC_ARCH VERSION_LESS_EQUAL _ARCH)
292
+ set(_TMP_ARCH ${_SRC_ARCH})
293
+ else()
294
+ break()
295
+ endif()
296
+ endforeach()
297
+ if (_TMP_ARCH)
298
+ list(APPEND _CUDA_ARCHS ${_TMP_ARCH})
299
+ endif()
300
+ endforeach()
301
+
302
+ list(REMOVE_DUPLICATES _CUDA_ARCHS)
303
+ set(${OUT_CUDA_ARCHS} ${_CUDA_ARCHS} PARENT_SCOPE)
304
+endfunction()
305
+
306
#
307
# Override the GPU architectures detected by cmake/torch and filter them by
308
# `GPU_SUPPORTED_ARCHES`. Sets the final set of architectures in
139
-# `GPU_ARCHES`.
309
+# `GPU_ARCHES`. This only applies to the HIP language since for CUDA we set
310
+# the architectures on a per file basis.
311
#
312
# Note: this is defined as a macro since it updates `CMAKE_CUDA_FLAGS`.
313
#
@@ -174,109 +345,7 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
345
"None of the detected ROCm architectures: ${HIP_ARCHITECTURES} is"
346
" supported. Supported ROCm architectures are: ${_GPU_SUPPORTED_ARCHES_LIST}.")
347
endif()
177
-
178
- elseif(${GPU_LANG} STREQUAL "CUDA")
179
- #
180
- # Setup/process CUDA arch flags.
181
- #
182
- # The torch cmake setup hardcodes the detected architecture flags in
183
- # `CMAKE_CUDA_FLAGS`. Since `CMAKE_CUDA_FLAGS` is a "global" variable, it
184
- # can't modified on a per-target basis.
185
- # So, all the `-gencode` flags need to be extracted and removed from
186
- # `CMAKE_CUDA_FLAGS` for processing so they can be passed by another method.
187
- # Since it's not possible to use `target_compiler_options` for adding target
188
- # specific `-gencode` arguments, the target's `CUDA_ARCHITECTURES` property
189
- # must be used instead. This requires repackaging the architecture flags
190
- # into a format that cmake expects for `CUDA_ARCHITECTURES`.
191
- #
192
- # This is a bit fragile in that it depends on torch using `-gencode` as opposed
193
- # to one of the other nvcc options to specify architectures.
194
- #
195
- # Note: torch uses the `TORCH_CUDA_ARCH_LIST` environment variable to override
196
- # detected architectures.
197
- #
198
- message(DEBUG "initial CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
199
-
200
- # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS`
201
- string(REGEX MATCHALL "-gencode arch=[^ ]+" _CUDA_ARCH_FLAGS
202
- ${CMAKE_CUDA_FLAGS})
203
-
204
- # Remove all `-gencode` flags from `CMAKE_CUDA_FLAGS` since they will be modified
205
- # and passed back via the `CUDA_ARCHITECTURES` property.
206
- string(REGEX REPLACE "-gencode arch=[^ ]+ *" "" CMAKE_CUDA_FLAGS
207
- ${CMAKE_CUDA_FLAGS})
208
-
209
- # If this error is triggered, it might mean that torch has changed how it sets
210
- # up nvcc architecture code generation flags.
211
- if (NOT _CUDA_ARCH_FLAGS)
212
- message(FATAL_ERROR
213
- "Could not find any architecture related code generation flags in "
214
- "CMAKE_CUDA_FLAGS. (${CMAKE_CUDA_FLAGS})")
215
- endif()
216
-
217
- message(DEBUG "final CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
218
- message(DEBUG "arch flags: ${_CUDA_ARCH_FLAGS}")
219
-
220
- # Initialize the architecture lists to empty.
221
- set(${GPU_ARCHES})
222
-
223
- # Process each `gencode` flag.
224
- foreach(_ARCH ${_CUDA_ARCH_FLAGS})
225
- # For each flag, extract the version number and whether it refers to PTX
226
- # or native code.
227
- # Note: if a regex matches then `CMAKE_MATCH_1` holds the binding
228
- # for that match.
229
-
230
- string(REGEX MATCH "arch=compute_\([0-9]+a?\)" _COMPUTE ${_ARCH})
231
- if (_COMPUTE)
232
- set(_COMPUTE ${CMAKE_MATCH_1})
233
- endif()
234
-
235
- string(REGEX MATCH "code=sm_\([0-9]+a?\)" _SM ${_ARCH})
236
- if (_SM)
237
- set(_SM ${CMAKE_MATCH_1})
238
- endif()
239
-
240
- string(REGEX MATCH "code=compute_\([0-9]+a?\)" _CODE ${_ARCH})
241
- if (_CODE)
242
- set(_CODE ${CMAKE_MATCH_1})
243
- endif()
244
-
245
- # Make sure the virtual architecture can be matched.
246
- if (NOT _COMPUTE)
247
- message(FATAL_ERROR
248
- "Could not determine virtual architecture from: ${_ARCH}.")
249
- endif()
250
-
251
- # One of sm_ or compute_ must exist.
252
- if ((NOT _SM) AND (NOT _CODE))
253
- message(FATAL_ERROR
254
- "Could not determine a codegen architecture from: ${_ARCH}.")
255
- endif()
256
-
257
- if (_SM)
258
- # -real suffix let CMake to only generate elf code for the kernels.
259
- # we want this, otherwise the added ptx (default) will increase binary size.
260
- set(_VIRT "-real")
261
- set(_CODE_ARCH ${_SM})
262
- else()
263
- # -virtual suffix let CMake to generate ptx code for the kernels.
264
- set(_VIRT "-virtual")
265
- set(_CODE_ARCH ${_CODE})
266
- endif()
267
-
268
- # Check if the current version is in the supported arch list.
269
- string_to_ver(_CODE_VER ${_CODE_ARCH})
270
- if (NOT _CODE_VER IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
271
- message(STATUS "discarding unsupported CUDA arch ${_VER}.")
272
- continue()
273
- endif()
274
-
275
- # Add it to the arch list.
276
- list(APPEND ${GPU_ARCHES} "${_CODE_ARCH}${_VIRT}")
277
- endforeach()
348
endif()
279
- message(STATUS "${GPU_LANG} target arches: ${${GPU_ARCHES}}")
349
endmacro()
350
351
#
csrc/core/registration.h
+5
@@ -12,6 +12,11 @@
12
// could be a macro instead of a literal token.
13
#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE)
14
15
+// A version of the TORCH_LIBRARY_IMPL macro that expands the NAME, i.e. so NAME
16
+// could be a macro instead of a literal token.
17
+#define TORCH_LIBRARY_IMPL_EXPAND(NAME, DEVICE, MODULE) \
18
+ TORCH_LIBRARY_IMPL(NAME, DEVICE, MODULE)
19
+
20
// REGISTER_EXTENSION allows the shared library to be loaded and initialized
21
// via python's import statement.
22
#define REGISTER_EXTENSION(NAME) \
csrc/moe/marlin_moe_ops.cu
+5
@@ -27,6 +27,7 @@
27
28
#include "core/exception.hpp"
29
#include "core/scalar_type.hpp"
30
+#include "core/registration.h"
31
#include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
32
#include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
33
@@ -552,3 +553,7 @@ torch::Tensor marlin_gemm_moe(
553
thread_n, sms, max_par, replicate_input, apply_weights);
554
return c;
555
}
556
+
557
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
558
+ m.impl("marlin_gemm_moe", &marlin_gemm_moe);
559
+}
csrc/moe/marlin_moe_ops.h
deleted
-15
@@ -1,15 +0,0 @@
1
-#pragma once
2
-
3
-#include <torch/all.h>
4
-
5
-#include "core/scalar_type.hpp"
6
-
7
-torch::Tensor marlin_gemm_moe(
8
- const torch::Tensor& a, const torch::Tensor& b_q_weights,
9
- const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
10
- const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
11
- const torch::Tensor& g_idx, const torch::Tensor& perm,
12
- torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
13
- int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
14
- int64_t num_experts, int64_t topk, int64_t moe_block_size,
15
- bool replicate_input, bool apply_weights);
csrc/moe/torch_bindings.cpp
+1
-2
@@ -1,6 +1,5 @@
1
#include "core/registration.h"
2
#include "moe_ops.h"
3
-#include "marlin_moe_ops.h"
3
4
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5
// Apply topk softmax to the gating outputs.
@@ -18,7 +17,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
17
"int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
18
"int moe_block_size, bool replicate_input, bool apply_weights)"
19
" -> Tensor");
21
- m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
20
+ // conditionally compiled so impl registration is in source file
21
#endif
22
}
23
csrc/ops.h
-68
@@ -90,63 +90,8 @@ torch::Tensor awq_dequantize(torch::Tensor _kernel,
90
torch::Tensor _zeros, int64_t split_k_iters,
91
int64_t thx, int64_t thy);
92
93
-torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
94
- torch::Tensor& b_scales, torch::Tensor& workspace,
95
- int64_t size_m, int64_t size_n, int64_t size_k);
96
-
97
-namespace machete {
98
-
99
-std::vector<std::string> supported_schedules(
100
- vllm::ScalarTypeTorchPtr const& btype);
101
-
102
-torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
103
- vllm::ScalarTypeTorchPtr const& btype,
104
- c10::optional<torch::Tensor> const& scales,
105
- c10::optional<torch::Tensor> const& zeros,
106
- c10::optional<int64_t> group_size,
107
- c10::optional<torch::Tensor> const& C,
108
- c10::optional<double> alpha, c10::optional<double> beta,
109
- c10::optional<std::string> schedule);
110
-
111
-torch::Tensor prepack_B(torch::Tensor const& B,
112
- vllm::ScalarTypeTorchPtr const& btype);
113
-
114
-}; // namespace machete
115
-
93
torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
94
118
-torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
119
- torch::Tensor& b_meta,
120
- torch::Tensor& b_scales,
121
- torch::Tensor& workspace,
122
- vllm::ScalarTypeTorchPtr const& b_q_type,
123
- int64_t size_m, int64_t size_n,
124
- int64_t size_k);
125
-
126
-torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
127
- torch::Tensor& b_scales, torch::Tensor& b_zeros,
128
- torch::Tensor& g_idx, torch::Tensor& perm,
129
- torch::Tensor& workspace,
130
- vllm::ScalarTypeTorchPtr const& b_q_type,
131
- int64_t size_m, int64_t size_n, int64_t size_k,
132
- bool is_k_full, bool has_zp,
133
- bool use_fp32_reduce);
134
-
135
-torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
136
- int64_t size_k, int64_t size_n,
137
- int64_t num_bits);
138
-
139
-torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
140
- torch::Tensor& perm, c10::SymInt size_k,
141
- c10::SymInt size_n, int64_t num_bits);
142
-
143
-torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
144
- int64_t size_n, int64_t num_bits);
145
-
146
-torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
147
- c10::SymInt size_k, c10::SymInt size_n,
148
- int64_t num_bits);
149
-
95
torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
96
int64_t n);
97
@@ -156,11 +101,6 @@ torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X,
101
torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
102
int64_t row);
103
159
-torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
160
- torch::Tensor& b_scales, torch::Tensor& workspace,
161
- int64_t num_bits, int64_t size_m, int64_t size_n,
162
- int64_t size_k);
163
-
104
bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
105
106
void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
@@ -175,14 +115,6 @@ void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
115
torch::Tensor const& azp_adj,
116
c10::optional<torch::Tensor> const& azp,
117
c10::optional<torch::Tensor> const& bias);
178
-
179
-torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
180
- torch::Tensor const& b_q_weight,
181
- torch::Tensor const& s_tok,
182
- torch::Tensor const& s_ch,
183
- torch::Tensor const& s_group,
184
- torch::Tensor& workspace, int64_t size_m,
185
- int64_t size_n, int64_t size_k);
118
#endif
119
120
void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+51
-25
@@ -21,7 +21,7 @@ void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
21
torch::Tensor const& b_scales,
22
c10::optional<torch::Tensor> const& bias);
23
24
-#if defined CUDA_VERSION && CUDA_VERSION >= 12000
24
+#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
25
void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
26
torch::Tensor const& b,
27
torch::Tensor const& a_scales,
@@ -114,26 +114,39 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
114
115
at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
116
int32_t version_num = get_sm_version_num();
117
- if (version_num >= 90) {
118
- // Hopper
117
+ // Hopper
118
120
- // Guard against compilation issues for sm90 kernels
121
-#if defined CUDA_VERSION && CUDA_VERSION >= 12000
119
+ // Guard against compilation issues for sm90 kernels
120
+#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
121
+ if (version_num >= 90) {
122
cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales, bias);
123
-#else
124
- cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
123
+ return;
124
+ }
125
#endif
126
- } else if (version_num == 89) {
126
+
127
+#if defined ENABLE_SCALED_MM_C2X && ENABLE_SCALED_MM_C2X
128
+ if (version_num == 89) {
129
// Ada Lovelace
130
cutlass_scaled_mm_sm89(c, a, b, a_scales, b_scales, bias);
129
- } else if (version_num >= 80) {
131
+ return;
132
+ }
133
+
134
+ if (version_num >= 80) {
135
// Ampere
136
cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
132
- } else {
133
- // Turing
134
- TORCH_CHECK(version_num >= 75);
135
- cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
137
+ return;
138
}
139
+
140
+ // Turing
141
+ TORCH_CHECK(version_num >= 75);
142
+ cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
143
+#endif
144
+
145
+ TORCH_CHECK_NOT_IMPLEMENTED(
146
+ false,
147
+ "No compiled cutlass_scaled_mm for a compute capability less than "
148
+ "CUDA device capability: ",
149
+ version_num);
150
}
151
152
void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
@@ -174,25 +187,38 @@ void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
187
"currently bias dtype must match output dtype ", c.dtype());
188
189
at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
190
+
191
int32_t version_num = get_sm_version_num();
178
- if (version_num >= 90) {
179
- // Hopper
192
181
- // Guard against compilation issues for sm90 kernels
182
-#if defined CUDA_VERSION && CUDA_VERSION >= 12000
193
+#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
194
+ if (version_num >= 90) {
195
cutlass_scaled_mm_azp_sm90(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
184
-#else
185
- cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
196
+ return;
197
+ }
198
#endif
187
- } else if (version_num == 89) {
199
+
200
+#if defined ENABLE_SCALED_MM_C2X && ENABLE_SCALED_MM_C2X
201
+ if (version_num == 89) {
202
// Ada Lovelace
203
cutlass_scaled_mm_azp_sm89(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
190
- } else if (version_num >= 80) {
204
+ return;
205
+ }
206
+
207
+ if (version_num >= 80) {
208
// Ampere
209
cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
193
- } else {
194
- // Turing
195
- TORCH_CHECK(version_num >= 75);
196
- cutlass_scaled_mm_azp_sm75(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
210
+ return;
211
}
212
+
213
+ // Turing
214
+ TORCH_CHECK(version_num >= 75);
215
+ cutlass_scaled_mm_azp_sm75(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
216
+ return;
217
+#endif
218
+
219
+ TORCH_CHECK_NOT_IMPLEMENTED(
220
+ false,
221
+ "No compiled cutlass_scaled_mm_azp for a compute capability less than "
222
+ "CUDA device capability: ",
223
+ version_num);
224
}
\ No newline at end of file
csrc/quantization/fp8/fp8_marlin.cu
+6
@@ -22,6 +22,8 @@
22
#include "../gptq_marlin/marlin.cuh"
23
#include "../gptq_marlin/marlin_dtypes.cuh"
24
25
+#include "core/registration.h"
26
+
27
using namespace marlin;
28
29
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
@@ -1303,3 +1305,7 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1305
}
1306
1307
#endif
1308
+
1309
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1310
+ m.impl("fp8_marlin_gemm", &fp8_marlin_gemm);
1311
+}
\ No newline at end of file
csrc/quantization/gptq_marlin/awq_marlin_repack.cu
+24
-37
@@ -1,25 +1,6 @@
1
#include "marlin.cuh"
2
3
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4
-
5
-namespace marlin {
6
-
7
-template <int const num_threads, int const num_bits, bool const has_perm>
8
-__global__ void awq_marlin_repack_kernel(
9
- uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr,
10
- int size_k, int size_n) {}
11
-
12
-} // namespace marlin
13
-
14
-torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
15
- int64_t size_k, int64_t size_n,
16
- int64_t num_bits) {
17
- TORCH_CHECK_NOT_IMPLEMENTED(
18
- false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
19
- return torch::empty({1, 1});
20
-}
21
-
22
-#else
3
+#include "core/registration.h"
4
5
namespace marlin {
6
@@ -122,7 +103,7 @@ __global__ void awq_marlin_repack_kernel(
103
}
104
105
uint32_t vals[8];
125
- #pragma unroll
106
+#pragma unroll
107
for (int i = 0; i < 4; i++) {
108
int cur_elem = tc_row + tc_offsets[i];
109
@@ -143,7 +124,7 @@ __global__ void awq_marlin_repack_kernel(
124
constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
125
126
uint32_t res = 0;
146
- #pragma unroll
127
+#pragma unroll
128
for (int i = 0; i < 8; i++) {
129
res |= vals[pack_idx[i]] << (i * 4);
130
}
@@ -155,7 +136,7 @@ __global__ void awq_marlin_repack_kernel(
136
137
uint32_t res1 = 0;
138
uint32_t res2 = 0;
158
- #pragma unroll
139
+#pragma unroll
140
for (int i = 0; i < 4; i++) {
141
res1 |= vals[pack_idx[i]] << (i * 8);
142
res2 |= vals[4 + pack_idx[i]] << (i * 8);
@@ -167,21 +148,21 @@ __global__ void awq_marlin_repack_kernel(
148
};
149
150
auto start_pipes = [&](int k_tile_id, int n_tile_id) {
170
- #pragma unroll
151
+#pragma unroll
152
for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
153
fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
154
}
155
156
wait_for_stage();
157
};
177
- #pragma unroll
158
+#pragma unroll
159
for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
160
int n_tile_id = 0;
161
162
start_pipes(k_tile_id, n_tile_id);
163
164
while (n_tile_id < n_tiles) {
184
- #pragma unroll
165
+#pragma unroll
166
for (int pipe = 0; pipe < repack_stages; pipe++) {
167
fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
168
n_tile_id + pipe + repack_stages - 1);
@@ -195,15 +176,15 @@ __global__ void awq_marlin_repack_kernel(
176
177
} // namespace marlin
178
198
- #define CALL_IF(NUM_BITS) \
199
- else if (num_bits == NUM_BITS) { \
200
- cudaFuncSetAttribute( \
201
- marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS>, \
202
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
203
- marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS> \
204
- <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
205
- b_q_weight_ptr, out_ptr, size_k, size_n); \
206
- }
179
+#define CALL_IF(NUM_BITS) \
180
+ else if (num_bits == NUM_BITS) { \
181
+ cudaFuncSetAttribute( \
182
+ marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS>, \
183
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
184
+ marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS> \
185
+ <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
186
+ b_q_weight_ptr, out_ptr, size_k, size_n); \
187
+ }
188
189
torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
190
int64_t size_n, int64_t num_bits) {
@@ -266,8 +247,6 @@ torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
247
return out;
248
}
249
269
-#endif
270
-
250
torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
251
c10::SymInt size_k, c10::SymInt size_n,
252
int64_t num_bits) {
@@ -279,3 +258,11 @@ torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
258
{size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
259
options);
260
}
261
+
262
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
263
+ m.impl("awq_marlin_repack", &awq_marlin_repack);
264
+}
265
+
266
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, Meta, m) {
267
+ m.impl("awq_marlin_repack", &awq_marlin_repack_meta);
268
+}
\ No newline at end of file
csrc/quantization/gptq_marlin/gptq_marlin.cu
+6
@@ -23,6 +23,8 @@
23
#include "marlin_dtypes.cuh"
24
#include "core/scalar_type.hpp"
25
26
+#include "core/registration.h"
27
+
28
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
29
static_assert(std::is_same<scalar_t, half>::value || \
30
std::is_same<scalar_t, nv_bfloat16>::value, \
@@ -2297,3 +2299,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2299
}
2300
2301
#endif
2302
+
2303
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
2304
+ m.impl("gptq_marlin_gemm", &gptq_marlin_gemm);
2305
+}
\ No newline at end of file
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+27
-41
@@ -1,26 +1,6 @@
1
#include "marlin.cuh"
2
3
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4
-
5
-namespace marlin {
6
-
7
-template <int const num_threads, int const num_bits, bool const has_perm>
8
-__global__ void gptq_marlin_repack_kernel(
9
- uint32_t const* __restrict__ b_q_weight_ptr,
10
- uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
11
- int size_k, int size_n) {}
12
-
13
-} // namespace marlin
14
-
15
-torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
16
- int64_t size_k, int64_t size_n,
17
- int64_t num_bits) {
18
- TORCH_CHECK_NOT_IMPLEMENTED(
19
- false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
20
- return torch::empty({1, 1});
21
-}
22
-
23
-#else
3
+#include "core/registration.h"
4
5
namespace marlin {
6
@@ -174,13 +154,13 @@ __global__ void gptq_marlin_repack_kernel(
154
uint32_t b1_vals[tile_ints];
155
uint32_t b2_vals[tile_ints];
156
177
- #pragma unroll
157
+#pragma unroll
158
for (int i = 0; i < tile_ints; i++) {
159
b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
160
b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
161
}
162
183
- #pragma unroll
163
+#pragma unroll
164
for (int i = 0; i < 4; i++) {
165
int cur_elem = tc_row + tc_offsets[i];
166
int cur_int = cur_elem / pack_factor;
@@ -200,7 +180,7 @@ __global__ void gptq_marlin_repack_kernel(
180
constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
181
182
uint32_t res = 0;
203
- #pragma unroll
183
+#pragma unroll
184
for (int i = 0; i < 8; i++) {
185
res |= vals[pack_idx[i]] << (i * 4);
186
}
@@ -212,7 +192,7 @@ __global__ void gptq_marlin_repack_kernel(
192
193
uint32_t res1 = 0;
194
uint32_t res2 = 0;
215
- #pragma unroll
195
+#pragma unroll
196
for (int i = 0; i < 4; i++) {
197
res1 |= vals[pack_idx[i]] << (i * 8);
198
res2 |= vals[4 + pack_idx[i]] << (i * 8);
@@ -224,14 +204,14 @@ __global__ void gptq_marlin_repack_kernel(
204
};
205
206
auto start_pipes = [&](int k_tile_id, int n_tile_id) {
227
- #pragma unroll
207
+#pragma unroll
208
for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
209
fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
210
}
211
212
wait_for_stage();
213
};
234
- #pragma unroll
214
+#pragma unroll
215
for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
216
int n_tile_id = 0;
217
@@ -242,7 +222,7 @@ __global__ void gptq_marlin_repack_kernel(
222
start_pipes(k_tile_id, n_tile_id);
223
224
while (n_tile_id < n_tiles) {
245
- #pragma unroll
225
+#pragma unroll
226
for (int pipe = 0; pipe < repack_stages; pipe++) {
227
fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
228
n_tile_id + pipe + repack_stages - 1);
@@ -256,17 +236,17 @@ __global__ void gptq_marlin_repack_kernel(
236
237
} // namespace marlin
238
259
- #define CALL_IF(NUM_BITS, HAS_PERM) \
260
- else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
261
- cudaFuncSetAttribute( \
262
- marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
263
- HAS_PERM>, \
264
- cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
265
- marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
266
- HAS_PERM> \
267
- <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
268
- b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
269
- }
239
+#define CALL_IF(NUM_BITS, HAS_PERM) \
240
+ else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
241
+ cudaFuncSetAttribute( \
242
+ marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
243
+ HAS_PERM>, \
244
+ cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
245
+ marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
246
+ HAS_PERM> \
247
+ <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
248
+ b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
249
+ }
250
251
torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
252
int64_t size_k, int64_t size_n,
@@ -341,8 +321,6 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
321
return out;
322
}
323
344
-#endif
345
-
324
torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
325
torch::Tensor& perm, c10::SymInt size_k,
326
c10::SymInt size_n, int64_t num_bits) {
@@ -354,3 +332,11 @@ torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
332
{size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
333
options);
334
}
335
+
336
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
337
+ m.impl("gptq_marlin_repack", &gptq_marlin_repack);
338
+}
339
+
340
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, Meta, m) {
341
+ m.impl("gptq_marlin_repack", &gptq_marlin_repack_meta);
342
+}
\ No newline at end of file
csrc/quantization/machete/generate.py
+1
-1
@@ -284,7 +284,7 @@ mm_impl_template = create_template(IMPL_TEMPLATE)
284
prepack_dispatch_template = create_template(PREPACK_TEMPLATE)
285
286
287
-def create_sources(impl_config: ImplConfig, num_impl_files=2):
287
+def create_sources(impl_config: ImplConfig, num_impl_files=1):
288
sources = []
289
290
type_name = generate_type_signature(impl_config.type_config)
csrc/quantization/machete/machete_prepack_kernel.cuh
+3
-4
@@ -34,10 +34,9 @@ static __global__ void prepack_B_kernel(BInTensor B_in,
34
}
35
36
template <typename PrepackedLayoutB, typename InLayout>
37
-static void prepack_B(cudaStream_t stream,
38
- typename PrepackedLayoutB::ElementB const* B_in_ptr,
39
- InLayout B_layout,
40
- typename PrepackedLayoutB::ElementB* B_out_ptr) {
37
+static void prepack_B_template(
38
+ cudaStream_t stream, typename PrepackedLayoutB::ElementB const* B_in_ptr,
39
+ InLayout B_layout, typename PrepackedLayoutB::ElementB* B_out_ptr) {
40
using TileShapeNKL =
41
decltype(append(typename PrepackedLayoutB::PPBlockShape_NK{}, _1{}));
42
auto ilvd_NKbNbKL_to_offset =
csrc/quantization/machete/machete_prepack_launcher.cuh
+2
-2
@@ -55,8 +55,8 @@ torch::Tensor prepack_impl(torch::Tensor const B) {
55
// Allocate output
56
torch::Tensor D = torch::empty_like(B, {}, at::MemoryFormat::Contiguous);
57
58
- prepack_B<PrepackedLayoutB>(stream, B_ptr, layout_Bt,
59
- static_cast<ElementB*>(D.mutable_data_ptr()));
58
+ prepack_B_template<PrepackedLayoutB>(
59
+ stream, B_ptr, layout_Bt, static_cast<ElementB*>(D.mutable_data_ptr()));
60
61
return D;
62
};
csrc/quantization/machete/machete_pytorch.cu
+9
-5
@@ -2,6 +2,8 @@
2
#include "machete_prepack_launcher.cuh"
3
#include "core/scalar_type.hpp"
4
5
+#include "core/registration.h"
6
+
7
namespace machete {
8
9
using namespace vllm;
@@ -78,14 +80,16 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
80
}
81
82
torch::Tensor prepack_B(torch::Tensor const& B,
81
- ScalarTypeTorchPtr const& btype) {
82
-#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
83
+ vllm::ScalarTypeTorchPtr const& btype) {
84
return scalar_type_dispatch(*btype, [&](auto BType) {
85
return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
86
});
86
-#else
87
- TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
88
-#endif
87
+}
88
+
89
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
90
+ m.impl("machete_prepack_B", &prepack_B);
91
+ m.impl("machete_gemm", &gemm);
92
+ m.impl("machete_supported_schedules", &supported_schedules);
93
}
94
95
}; // namespace machete
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+5
@@ -26,6 +26,7 @@
26
#include <iostream>
27
28
#include "common/base.h"
29
+#include "core/registration.h"
30
31
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
32
#include "common/mem.h"
@@ -1066,3 +1067,7 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1067
1068
return c;
1069
}
1070
+
1071
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1072
+ m.impl("marlin_gemm", &marlin_gemm);
1073
+}
csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu
+5
@@ -30,6 +30,7 @@
30
#include <iostream>
31
32
#include "../dense/common/base.h"
33
+#include "core/registration.h"
34
35
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
36
#include "../dense/common/mem.h"
@@ -1241,3 +1242,7 @@ torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
1242
1243
return d;
1244
}
1245
+
1246
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1247
+ m.impl("marlin_qqq_gemm", &marlin_qqq_gemm);
1248
+}
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+5
@@ -28,6 +28,7 @@
28
29
#include "common/base.h"
30
#include "core/scalar_type.hpp"
31
+#include "core/registration.h"
32
33
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
34
@@ -1134,3 +1135,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1135
1136
return c;
1137
}
1138
+
1139
+TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1140
+ m.impl("gptq_marlin_24_gemm", &gptq_marlin_24_gemm);
1141
+}
csrc/torch_bindings.cpp
+12
-12
@@ -167,7 +167,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
167
ops.def(
168
"marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
169
"Tensor! workspace, int size_m, int size_n, int size_k) -> Tensor");
170
- ops.impl("marlin_gemm", torch::kCUDA, &marlin_gemm);
170
+ // conditionally compiled so impl in source file
171
172
// Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
173
ops.def(
@@ -175,22 +175,24 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
175
"Tensor b_scales, Tensor workspace, "
176
"__torch__.torch.classes._core_C.ScalarType b_q_type, "
177
"int size_m, int size_n, int size_k) -> Tensor");
178
- ops.impl("gptq_marlin_24_gemm", torch::kCUDA, &gptq_marlin_24_gemm);
178
+ // conditionally compiled so impl in source file
179
180
// Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
181
- ops.def("machete_supported_schedules", &machete::supported_schedules);
181
+ ops.def(
182
+ "machete_supported_schedules("
183
+ " __torch__.torch.classes._core_C.ScalarType btype"
184
+ ") -> str[]");
185
ops.def(
186
"machete_gemm(Tensor A, Tensor B,"
187
" __torch__.torch.classes._core_C.ScalarType btype,"
188
" Tensor? scales, Tensor? zeros, int? group_size,"
189
" Tensor? C, float? alpha, float? beta, str? schedule)"
190
"-> Tensor");
188
- ops.impl("machete_gemm", torch::kCUDA, &machete::gemm);
191
ops.def(
192
"machete_prepack_B(Tensor B,"
193
" __torch__.torch.classes._core_C.ScalarType btype)"
194
"-> Tensor");
193
- ops.impl("machete_prepack_B", torch::kCUDA, &machete::prepack_B);
195
+ // conditionally compiled so impl registration is in source file
196
197
ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
198
ops.impl("permute_cols", torch::kCUDA, &permute_cols);
@@ -202,21 +204,19 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
204
"__torch__.torch.classes._core_C.ScalarType b_q_type, "
205
"int size_m, int size_n, int size_k, bool is_k_full, "
206
"bool has_zp, bool use_fp32_reduce) -> Tensor");
205
- ops.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm);
207
+ // conditionally compiled so impl registration is in source file
208
209
// gptq_marlin repack from GPTQ.
210
ops.def(
211
"gptq_marlin_repack(Tensor b_q_weight, Tensor perm, "
212
"SymInt size_k, SymInt size_n, int num_bits) -> Tensor");
211
- ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
212
- ops.impl("gptq_marlin_repack", torch::kMeta, &gptq_marlin_repack_meta);
213
+ // conditionally compiled so impl registrations are in source file
214
215
// awq_marlin repack from AWQ.
216
ops.def(
217
"awq_marlin_repack(Tensor b_q_weight, SymInt size_k, "
218
"SymInt size_n, int num_bits) -> Tensor");
218
- ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
219
- ops.impl("awq_marlin_repack", torch::kMeta, &awq_marlin_repack_meta);
219
+ // conditionally compiled so impl registrations are in source file
220
221
// Dequantization for GGML.
222
ops.def("ggml_dequantize(Tensor W, int type, int m, int n) -> Tensor");
@@ -237,7 +237,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
237
"fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
238
"Tensor! workspace, int num_bits, int size_m, int size_n, "
239
"int size_k) -> Tensor");
240
- ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
240
+ // conditionally compiled so impl registration is in source file
241
242
// marlin_qqq_gemm for QQQ.
243
ops.def(
@@ -245,7 +245,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
245
"Tensor s_tok, Tensor s_ch, Tensor s_group, "
246
"Tensor! workspace, int size_m, int size_n, "
247
"int size_k) -> Tensor");
248
- ops.impl("marlin_qqq_gemm", torch::kCUDA, &marlin_qqq_gemm);
248
+ // conditionally compiled so impl registration is in source file
249
250
// CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
251
// quantization, as well as bias
tools/report_build_time_ninja.py
new
+311
@@ -0,0 +1,311 @@
1
+#!/usr/bin/env python3
2
+# Copyright (c) 2018 The Chromium Authors. All rights reserved.
3
+# Use of this source code is governed by a BSD-style license that can be
4
+# found in the LICENSE file.
5
+
6
+# Modified version of: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/refs/heads/main/post_build_ninja_summary.py
7
+"""Summarize the last ninja build, invoked with ninja's -C syntax.
8
+
9
+> python3 tools/report_build_time_ninja.py -C build/..
10
+
11
+Typical output looks like this:
12
+```
13
+ Longest build steps for .cpp.o:
14
+ 1.0 weighted s to build ...torch_bindings.cpp.o (12.4 s elapsed time)
15
+ 2.0 weighted s to build ..._attn_c.dir/csrc... (23.5 s elapsed time)
16
+ 2.6 weighted s to build ...torch_bindings.cpp.o (31.5 s elapsed time)
17
+ 3.2 weighted s to build ...torch_bindings.cpp.o (38.5 s elapsed time)
18
+ Longest build steps for .so (linking):
19
+ 0.1 weighted s to build _core_C.abi3.so (0.7 s elapsed time)
20
+ 0.1 weighted s to build _moe_C.abi3.so (1.0 s elapsed time)
21
+ 0.5 weighted s to build ...flash_attn_c.abi3.so (1.1 s elapsed time)
22
+ 6.2 weighted s to build _C.abi3.so (6.2 s elapsed time)
23
+ Longest build steps for .cu.o:
24
+ 15.3 weighted s to build ...machete_mm_... (183.5 s elapsed time)
25
+ 15.3 weighted s to build ...machete_mm_... (183.5 s elapsed time)
26
+ 15.3 weighted s to build ...machete_mm_... (183.6 s elapsed time)
27
+ 15.3 weighted s to build ...machete_mm_... (183.7 s elapsed time)
28
+ 15.5 weighted s to build ...machete_mm_... (185.6 s elapsed time)
29
+ 15.5 weighted s to build ...machete_mm_... (185.9 s elapsed time)
30
+ 15.5 weighted s to build ...machete_mm_... (186.2 s elapsed time)
31
+ 37.4 weighted s to build ...scaled_mm_c3x.cu... (449.0 s elapsed time)
32
+ 43.9 weighted s to build ...scaled_mm_c2x.cu... (527.4 s elapsed time)
33
+ 344.8 weighted s to build ...attention_...cu.o (1087.2 s elapsed time)
34
+ 1110.0 s weighted time (10120.4 s elapsed time sum, 9.1x parallelism)
35
+ 134 build steps completed, average of 0.12/s
36
+```
37
+"""
38
+
39
+import argparse
40
+import errno
41
+import fnmatch
42
+import os
43
+import sys
44
+from collections import defaultdict
45
+
46
+# The number of long build times to report:
47
+long_count = 10
48
+# The number of long times by extension to report
49
+long_ext_count = 10
50
+
51
+
52
+class Target:
53
+ """Represents a single line read for a .ninja_log file."""
54
+
55
+ def __init__(self, start, end):
56
+ """Creates a target object by passing in the start/end times in seconds
57
+ as a float."""
58
+ self.start = start
59
+ self.end = end
60
+ # A list of targets, appended to by the owner of this object.
61
+ self.targets = []
62
+ self.weighted_duration = 0.0
63
+
64
+ def Duration(self):
65
+ """Returns the task duration in seconds as a float."""
66
+ return self.end - self.start
67
+
68
+ def SetWeightedDuration(self, weighted_duration):
69
+ """Sets the duration, in seconds, passed in as a float."""
70
+ self.weighted_duration = weighted_duration
71
+
72
+ def WeightedDuration(self):
73
+ """Returns the task's weighted duration in seconds as a float.
74
+
75
+ Weighted_duration takes the elapsed time of the task and divides it
76
+ by how many other tasks were running at the same time. Thus, it
77
+ represents the approximate impact of this task on the total build time,
78
+ with serialized or serializing steps typically ending up with much
79
+ longer weighted durations.
80
+ weighted_duration should always be the same or shorter than duration.
81
+ """
82
+ # Allow for modest floating-point errors
83
+ epsilon = 0.000002
84
+ if (self.weighted_duration > self.Duration() + epsilon):
85
+ print('%s > %s?' % (self.weighted_duration, self.Duration()))
86
+ assert (self.weighted_duration <= self.Duration() + epsilon)
87
+ return self.weighted_duration
88
+
89
+ def DescribeTargets(self):
90
+ """Returns a printable string that summarizes the targets."""
91
+ # Some build steps generate dozens of outputs - handle them sanely.
92
+ # The max_length was chosen so that it can fit most of the long
93
+ # single-target names, while minimizing word wrapping.
94
+ result = ', '.join(self.targets)
95
+ max_length = 65
96
+ if len(result) > max_length:
97
+ result = result[:max_length] + '...'
98
+ return result
99
+
100
+
101
+# Copied with some modifications from ninjatracing
102
+def ReadTargets(log, show_all):
103
+ """Reads all targets from .ninja_log file |log_file|, sorted by duration.
104
+
105
+ The result is a list of Target objects."""
106
+ header = log.readline()
107
+ assert header == '# ninja log v5\n', \
108
+ 'unrecognized ninja log version %r' % header
109
+ targets_dict = {}
110
+ last_end_seen = 0.0
111
+ for line in log:
112
+ parts = line.strip().split('\t')
113
+ if len(parts) != 5:
114
+ # If ninja.exe is rudely halted then the .ninja_log file may be
115
+ # corrupt. Silently continue.
116
+ continue
117
+ start, end, _, name, cmdhash = parts # Ignore restat.
118
+ # Convert from integral milliseconds to float seconds.
119
+ start = int(start) / 1000.0
120
+ end = int(end) / 1000.0
121
+ if not show_all and end < last_end_seen:
122
+ # An earlier time stamp means that this step is the first in a new
123
+ # build, possibly an incremental build. Throw away the previous
124
+ # data so that this new build will be displayed independently.
125
+ # This has to be done by comparing end times because records are
126
+ # written to the .ninja_log file when commands complete, so end
127
+ # times are guaranteed to be in order, but start times are not.
128
+ targets_dict = {}
129
+ target = None
130
+ if cmdhash in targets_dict:
131
+ target = targets_dict[cmdhash]
132
+ if not show_all and (target.start != start or target.end != end):
133
+ # If several builds in a row just run one or two build steps
134
+ # then the end times may not go backwards so the last build may
135
+ # not be detected as such. However in many cases there will be a
136
+ # build step repeated in the two builds and the changed
137
+ # start/stop points for that command, identified by the hash,
138
+ # can be used to detect and reset the target dictionary.
139
+ targets_dict = {}
140
+ target = None
141
+ if not target:
142
+ targets_dict[cmdhash] = target = Target(start, end)
143
+ last_end_seen = end
144
+ target.targets.append(name)
145
+ return list(targets_dict.values())
146
+
147
+
148
+def GetExtension(target, extra_patterns):
149
+ """Return the file extension that best represents a target.
150
+
151
+ For targets that generate multiple outputs it is important to return a
152
+ consistent 'canonical' extension. Ultimately the goal is to group build steps
153
+ by type."""
154
+ for output in target.targets:
155
+ if extra_patterns:
156
+ for fn_pattern in extra_patterns.split(';'):
157
+ if fnmatch.fnmatch(output, '*' + fn_pattern + '*'):
158
+ return fn_pattern
159
+ # Not a true extension, but a good grouping.
160
+ if output.endswith('type_mappings'):
161
+ extension = 'type_mappings'
162
+ break
163
+
164
+ # Capture two extensions if present. For example: file.javac.jar should
165
+ # be distinguished from file.interface.jar.
166
+ root, ext1 = os.path.splitext(output)
167
+ _, ext2 = os.path.splitext(root)
168
+ extension = ext2 + ext1 # Preserve the order in the file name.
169
+
170
+ if len(extension) == 0:
171
+ extension = '(no extension found)'
172
+
173
+ if ext1 in ['.pdb', '.dll', '.exe']:
174
+ extension = 'PEFile (linking)'
175
+ # Make sure that .dll and .exe are grouped together and that the
176
+ # .dll.lib files don't cause these to be listed as libraries
177
+ break
178
+ if ext1 in ['.so', '.TOC']:
179
+ extension = '.so (linking)'
180
+ # Attempt to identify linking, avoid identifying as '.TOC'
181
+ break
182
+ # Make sure .obj files don't get categorized as mojo files
183
+ if ext1 in ['.obj', '.o']:
184
+ break
185
+ # Jars are the canonical output of java targets.
186
+ if ext1 == '.jar':
187
+ break
188
+ # Normalize all mojo related outputs to 'mojo'.
189
+ if output.count('.mojom') > 0:
190
+ extension = 'mojo'
191
+ break
192
+ return extension
193
+
194
+
195
+def SummarizeEntries(entries, extra_step_types):
196
+ """Print a summary of the passed in list of Target objects."""
197
+
198
+ # Create a list that is in order by time stamp and has entries for the
199
+ # beginning and ending of each build step (one time stamp may have multiple
200
+ # entries due to multiple steps starting/stopping at exactly the same time).
201
+ # Iterate through this list, keeping track of which tasks are running at all
202
+ # times. At each time step calculate a running total for weighted time so
203
+ # that when each task ends its own weighted time can easily be calculated.
204
+ task_start_stop_times = []
205
+
206
+ earliest = -1
207
+ latest = 0
208
+ total_cpu_time = 0
209
+ for target in entries:
210
+ if earliest < 0 or target.start < earliest:
211
+ earliest = target.start
212
+ if target.end > latest:
213
+ latest = target.end
214
+ total_cpu_time += target.Duration()
215
+ task_start_stop_times.append((target.start, 'start', target))
216
+ task_start_stop_times.append((target.end, 'stop', target))
217
+ length = latest - earliest
218
+ weighted_total = 0.0
219
+
220
+ # Sort by the time/type records and ignore |target|
221
+ task_start_stop_times.sort(key=lambda times: times[:2])
222
+ # Now we have all task start/stop times sorted by when they happen. If a
223
+ # task starts and stops on the same time stamp then the start will come
224
+ # first because of the alphabet, which is important for making this work
225
+ # correctly.
226
+ # Track the tasks which are currently running.
227
+ running_tasks = {}
228
+ # Record the time we have processed up to so we know how to calculate time
229
+ # deltas.
230
+ last_time = task_start_stop_times[0][0]
231
+ # Track the accumulated weighted time so that it can efficiently be added
232
+ # to individual tasks.
233
+ last_weighted_time = 0.0
234
+ # Scan all start/stop events.
235
+ for event in task_start_stop_times:
236
+ time, action_name, target = event
237
+ # Accumulate weighted time up to now.
238
+ num_running = len(running_tasks)
239
+ if num_running > 0:
240
+ # Update the total weighted time up to this moment.
241
+ last_weighted_time += (time - last_time) / float(num_running)
242
+ if action_name == 'start':
243
+ # Record the total weighted task time when this task starts.
244
+ running_tasks[target] = last_weighted_time
245
+ if action_name == 'stop':
246
+ # Record the change in the total weighted task time while this task
247
+ # ran.
248
+ weighted_duration = last_weighted_time - running_tasks[target]
249
+ target.SetWeightedDuration(weighted_duration)
250
+ weighted_total += weighted_duration
251
+ del running_tasks[target]
252
+ last_time = time
253
+ assert (len(running_tasks) == 0)
254
+
255
+ # Warn if the sum of weighted times is off by more than half a second.
256
+ if abs(length - weighted_total) > 500:
257
+ print('Warning: Possible corrupt ninja log, results may be '
258
+ 'untrustworthy. Length = %.3f, weighted total = %.3f' %
259
+ (length, weighted_total))
260
+
261
+ entries_by_ext = defaultdict(list)
262
+ for target in entries:
263
+ extension = GetExtension(target, extra_step_types)
264
+ entries_by_ext[extension].append(target)
265
+
266
+ for key, values in entries_by_ext.items():
267
+ print(' Longest build steps for %s:' % key)
268
+ values.sort(key=lambda x: x.WeightedDuration())
269
+ for target in values[-long_count:]:
270
+ print(' %8.1f weighted s to build %s (%.1f s elapsed time)' %
271
+ (target.WeightedDuration(), target.DescribeTargets(),
272
+ target.Duration()))
273
+
274
+ print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx '
275
+ 'parallelism)' %
276
+ (length, total_cpu_time, total_cpu_time * 1.0 / length))
277
+ print(' %d build steps completed, average of %1.2f/s' %
278
+ (len(entries), len(entries) / (length)))
279
+
280
+
281
+def main():
282
+ log_file = '.ninja_log'
283
+ parser = argparse.ArgumentParser()
284
+ parser.add_argument('-C', dest='build_directory', help='Build directory.')
285
+ parser.add_argument(
286
+ '-s',
287
+ '--step-types',
288
+ help='semicolon separated fnmatch patterns for build-step grouping')
289
+ parser.add_argument('--log-file',
290
+ help="specific ninja log file to analyze.")
291
+ args, _extra_args = parser.parse_known_args()
292
+ if args.build_directory:
293
+ log_file = os.path.join(args.build_directory, log_file)
294
+ if args.log_file:
295
+ log_file = args.log_file
296
+ if args.step_types:
297
+ # Make room for the extra build types.
298
+ global long_ext_count
299
+ long_ext_count += len(args.step_types.split(';'))
300
+
301
+ try:
302
+ with open(log_file, 'r') as log:
303
+ entries = ReadTargets(log, False)
304
+ SummarizeEntries(entries, args.step_types)
305
+ except IOError:
306
+ print('Log file %r not found, no build summary created.' % log_file)
307
+ return errno.ENOENT
308
+
309
+
310
+if __name__ == '__main__':
311
+ sys.exit(main())
vllm/_custom_ops.py
+9
@@ -32,6 +32,15 @@ def hint_on_error(fn):
32
def wrapper(*args, **kwargs):
33
try:
34
return fn(*args, **kwargs)
35
+
36
+ except NotImplementedError as e:
37
+ msg = (
38
+ "Error in calling custom op %s: %s\n"
39
+ "Not implemented or built, mostly likely because the current current device "
40
+ "does not support this kernel (less likely TORCH_CUDA_ARCH_LIST was set "
41
+ "incorrectly while building)")
42
+ logger.error(msg, fn.__name__, e)
43
+ raise NotImplementedError(msg % (fn.__name__, e)) from e
44
except AttributeError as e:
45
msg = (
46
"Error in calling custom op %s: %s\n"