@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aeb37c2a72
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[CI/Build] Per file CUDA Archs (improve wheel size and dev build times) (#8845)

Lucas Wilkinson committed Oct 3, 2024 at 22:55 UTC aeb37c2a725554791ff6f258b1e18830867a3ab9
22 files changed +828 -370
CMakeLists.txt
+169 -55
@@ -143,6 +143,19 @@ else()
143 message(FATAL_ERROR "Can't find CUDA or HIP installation.")
144 endif()
145
146 +
147 +#
148 +# For cuda we want to be able to control which architectures we compile for on
149 +# a per-file basis in order to cut down on compile time. So here we extract
150 +# the set of architectures we want to compile for and remove the from the
151 +# CMAKE_CUDA_FLAGS so that they are not applied globally.
152 +#
153 +if(VLLM_GPU_LANG STREQUAL "CUDA")
154 + clear_cuda_arches(CUDA_ARCH_FLAGS)
155 + extract_unique_cuda_archs_ascending(CUDA_ARCHS "${CUDA_ARCH_FLAGS}")
156 + message(STATUS "CUDA target architectures: ${CUDA_ARCHS}")
157 +endif()
158 +
159 #
160 # Override the GPU architectures detected by cmake/torch and filter them by
161 # the supported versions for the current language.
@@ -223,30 +236,89 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
236 "csrc/mamba/causal_conv1d/causal_conv1d.cu"
237 "csrc/quantization/aqlm/gemm_kernels.cu"
238 "csrc/quantization/awq/gemm_kernels.cu"
226 - "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
227 - "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
228 - "csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu"
229 - "csrc/quantization/gptq_marlin/gptq_marlin.cu"
230 - "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
231 - "csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
239 "csrc/quantization/gguf/gguf_kernel.cu"
233 - "csrc/quantization/fp8/fp8_marlin.cu"
240 "csrc/custom_all_reduce.cu"
241 "csrc/permute_cols.cu"
236 - "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
237 - "csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu"
238 - "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
242 + "csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu")
243 +
244 + set_gencode_flags_for_srcs(
245 + SRCS "${VLLM_EXT_SRC}"
246 + CUDA_ARCHS "${CUDA_ARCHS}")
247 +
248 + # Only build Marlin kernels if we are building for at least some compatible archs.
249 + # Keep building Marlin for 9.0 as there are some group sizes and shapes that
250 + # are not supported by Machete yet.
251 + cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.9;9.0" ${CUDA_ARCHS})
252 + if (MARLIN_ARCHS)
253 + set(MARLIN_SRCS
254 + "csrc/quantization/fp8/fp8_marlin.cu"
255 + "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
256 + "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
257 + "csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu"
258 + "csrc/quantization/gptq_marlin/gptq_marlin.cu"
259 + "csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
260 + "csrc/quantization/gptq_marlin/awq_marlin_repack.cu")
261 + set_gencode_flags_for_srcs(
262 + SRCS "${MARLIN_SRCS}"
263 + CUDA_ARCHS "${MARLIN_ARCHS}")
264 + list(APPEND VLLM_EXT_SRC "${MARLIN_SRCS}")
265 + message(STATUS "Building Marlin kernels for archs: ${MARLIN_ARCHS}")
266 + else()
267 + message(STATUS "Not building Marlin kernels as no compatible archs found"
268 + "in CUDA target architectures")
269 + endif()
270 +
271 + #
272 + # The cutlass_scaled_mm kernels for Hopper (c3x, i.e. CUTLASS 3.x) require
273 + # CUDA 12.0 or later (and only work on Hopper, 9.0/9.0a for now).
274 + cuda_archs_loose_intersection(SCALED_MM_3X_ARCHS "9.0;9.0a" "${CUDA_ARCHS}")
275 + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
276 + set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
277 + set_gencode_flags_for_srcs(
278 + SRCS "${SRCS}"
279 + CUDA_ARCHS "${SCALED_MM_3X_ARCHS}")
280 + list(APPEND VLLM_EXT_SRC "${SRCS}")
281 + list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_C3X=1")
282 + message(STATUS "Building scaled_mm_c3x for archs: ${SCALED_MM_3X_ARCHS}")
283 + else()
284 + # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
285 + # build any 3x kernels
286 + set(SCALED_MM_3X_ARCHS)
287 +
288 + if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND SCALED_MM_3X_ARCHS)
289 + message(STATUS "Not building scaled_mm_c3x as CUDA Compiler version is "
290 + "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
291 + "later if you intend on running FP8 quantized models on "
292 + "Hopper.")
293 + else()
294 + message(STATUS "Not building scaled_mm_c3x as no compatible archs found "
295 + "in CUDA target architectures")
296 + endif()
297 + endif()
298
299 #
241 - # The CUTLASS kernels for Hopper require sm90a to be enabled.
242 - # This is done via the below gencode option, BUT that creates kernels for both sm90 and sm90a.
243 - # That adds an extra 17MB to compiled binary, so instead we selectively enable it.
244 - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
245 - set_source_files_properties(
246 - "csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
247 - PROPERTIES
248 - COMPILE_FLAGS
249 - "-gencode arch=compute_90a,code=sm_90a")
300 + # For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
301 + # kernels for the remaining archs that are not already built for 3x.
302 + cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
303 + "7.5;8.0;8.6;8.9;9.0;9.0a" "${CUDA_ARCHS}")
304 + # subtract out the archs that are already built for 3x
305 + list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
306 + if (SCALED_MM_2X_ARCHS)
307 + set(SRCS "csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu")
308 + set_gencode_flags_for_srcs(
309 + SRCS "${SRCS}"
310 + CUDA_ARCHS "${SCALED_MM_2X_ARCHS}")
311 + list(APPEND VLLM_EXT_SRC "${SRCS}")
312 + list(APPEND VLLM_GPU_FLAGS "-DENABLE_SCALED_MM_C2X=1")
313 + message(STATUS "Building scaled_mm_c2x for archs: ${SCALED_MM_2X_ARCHS}")
314 + else()
315 + if (SCALED_MM_3X_ARCHS)
316 + message(STATUS "Not building scaled_mm_c2x as all archs are already built"
317 + " for and covered by scaled_mm_c3x")
318 + else()
319 + message(STATUS "Not building scaled_mm_c2x as no compatible archs found "
320 + "in CUDA target architectures")
321 + endif()
322 endif()
323
324
@@ -254,47 +326,72 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
326 # Machete kernels
327
328 # The machete kernels only work on hopper and require CUDA 12.0 or later.
257 - if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
329 + # Only build Machete kernels if we are building for something compatible with sm90a
330 + cuda_archs_loose_intersection(MACHETE_ARCHS "9.0a" "${CUDA_ARCHS}")
331 + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND MACHETE_ARCHS)
332 #
333 # For the Machete kernels we automatically generate sources for various
334 # preselected input type pairs and schedules.
335 # Generate sources:
262 - execute_process(
263 - COMMAND ${CMAKE_COMMAND} -E env
264 - PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
265 - ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py
266 - RESULT_VARIABLE machete_generation_result
267 - OUTPUT_VARIABLE machete_generation_output
268 - OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
269 - ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
270 - )
271 -
272 - if (NOT machete_generation_result EQUAL 0)
273 - message(FATAL_ERROR "Machete generation failed."
274 - " Result: \"${machete_generation_result}\""
275 - "\nCheck the log for details: "
276 - "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
336 + set(MACHETE_GEN_SCRIPT
337 + ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py)
338 + file(MD5 ${MACHETE_GEN_SCRIPT} MACHETE_GEN_SCRIPT_HASH)
339 +
340 + message(STATUS "Machete generation script hash: ${MACHETE_GEN_SCRIPT_HASH}")
341 + message(STATUS "Last run machete generate script hash: $CACHE{MACHETE_GEN_SCRIPT_HASH}")
342 +
343 + if (NOT DEFINED CACHE{MACHETE_GEN_SCRIPT_HASH}
344 + OR NOT $CACHE{MACHETE_GEN_SCRIPT_HASH} STREQUAL ${MACHETE_GEN_SCRIPT_HASH})
345 + execute_process(
346 + COMMAND ${CMAKE_COMMAND} -E env
347 + PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
348 + ${Python_EXECUTABLE} ${MACHETE_GEN_SCRIPT}
349 + RESULT_VARIABLE machete_generation_result
350 + OUTPUT_VARIABLE machete_generation_output
351 + OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
352 + ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
353 + )
354 +
355 + if (NOT machete_generation_result EQUAL 0)
356 + message(FATAL_ERROR "Machete generation failed."
357 + " Result: \"${machete_generation_result}\""
358 + "\nCheck the log for details: "
359 + "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
360 + else()
361 + set(MACHETE_GEN_SCRIPT_HASH ${MACHETE_GEN_SCRIPT_HASH}
362 + CACHE STRING "Last run machete generate script hash" FORCE)
363 + message(STATUS "Machete generation completed successfully.")
364 + endif()
365 else()
278 - message(STATUS "Machete generation completed successfully.")
366 + message(STATUS "Machete generation script has not changed, skipping generation.")
367 endif()
368
369 # Add machete generated sources
370 file(GLOB MACHETE_GEN_SOURCES "csrc/quantization/machete/generated/*.cu")
371 list(APPEND VLLM_EXT_SRC ${MACHETE_GEN_SOURCES})
284 - message(STATUS "Machete generated sources: ${MACHETE_GEN_SOURCES}")
372
286 - set_source_files_properties(
287 - ${MACHETE_GEN_SOURCES}
288 - PROPERTIES
289 - COMPILE_FLAGS
290 - "-gencode arch=compute_90a,code=sm_90a")
373 + # forward compatible
374 + set_gencode_flags_for_srcs(
375 + SRCS "${MACHETE_GEN_SOURCES}"
376 + CUDA_ARCHS "${MACHETE_ARCHS}")
377 +
378 + list(APPEND VLLM_EXT_SRC
379 + csrc/quantization/machete/machete_pytorch.cu)
380 +
381 + message(STATUS "Building Machete kernels for archs: ${MACHETE_ARCHS}")
382 + else()
383 + if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
384 + AND MACHETE_ARCHS)
385 + message(STATUS "Not building Machete kernels as CUDA Compiler version is "
386 + "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
387 + "later if you intend on running w4a16 quantized models on "
388 + "Hopper.")
389 + else()
390 + message(STATUS "Not building Machete kernels as no compatible archs "
391 + "found in CUDA target architectures")
392 + endif()
393 endif()
292 -
293 - # Add pytorch binding for machete (add on even CUDA < 12.0 so that we can
294 - # raise an error if the user that this was built with an incompatible
295 - # CUDA version)
296 - list(APPEND VLLM_EXT_SRC
297 - csrc/quantization/machete/machete_pytorch.cu)
394 +# if CUDA endif
395 endif()
396
397 message(STATUS "Enabling C extension.")
@@ -323,14 +420,31 @@ set(VLLM_MOE_EXT_SRC
420 "csrc/moe/torch_bindings.cpp"
421 "csrc/moe/topk_softmax_kernels.cu")
422
423 +set_gencode_flags_for_srcs(
424 + SRCS "${VLLM_MOE_EXT_SRC}"
425 + CUDA_ARCHS "${CUDA_ARCHS}")
426 +
427 if(VLLM_GPU_LANG STREQUAL "CUDA")
327 - list(APPEND VLLM_MOE_EXT_SRC
328 - "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
329 - "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h"
330 - "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
331 - "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
332 - "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
333 - "csrc/moe/marlin_moe_ops.cu")
428 + cuda_archs_loose_intersection(MARLIN_MOE_ARCHS "8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
429 + if (MARLIN_MOE_ARCHS)
430 + set(MARLIN_MOE_SRC
431 + "csrc/moe/marlin_kernels/marlin_moe_kernel.h"
432 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.h"
433 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku4b8.cu"
434 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.h"
435 + "csrc/moe/marlin_kernels/marlin_moe_kernel_ku8b128.cu"
436 + "csrc/moe/marlin_moe_ops.cu")
437 +
438 + set_gencode_flags_for_srcs(
439 + SRCS "${MARLIN_MOE_SRC}"
440 + CUDA_ARCHS "${MARLIN_MOE_ARCHS}")
441 +
442 + list(APPEND VLLM_MOE_EXT_SRC "${MARLIN_MOE_SRC}")
443 + message(STATUS "Building Marlin MOE kernels for archs: ${MARLIN_MOE_ARCHS}")
444 + else()
445 + message(STATUS "Not building Marlin MOE kernels as no compatible archs found"
446 + "in CUDA target architectures")
447 + endif()
448 endif()
449
450 message(STATUS "Enabling moe extension.")
cmake/utils.cmake
+172 -103
@@ -133,10 +133,181 @@ macro(string_to_ver OUT_VER IN_STR)
133 string(REGEX REPLACE "\([0-9]+\)\([0-9]\)" "\\1.\\2" ${OUT_VER} ${IN_STR})
134 endmacro()
135
136 +#
137 +# Clear all `-gencode` flags from `CMAKE_CUDA_FLAGS` and store them in
138 +# `CUDA_ARCH_FLAGS`.
139 +#
140 +# Example:
141 +# CMAKE_CUDA_FLAGS="-Wall -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75"
142 +# clear_cuda_arches(CUDA_ARCH_FLAGS)
143 +# CUDA_ARCH_FLAGS="-gencode arch=compute_70,code=sm_70;-gencode arch=compute_75,code=sm_75"
144 +# CMAKE_CUDA_FLAGS="-Wall"
145 +#
146 +macro(clear_cuda_arches CUDA_ARCH_FLAGS)
147 + # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS`
148 + string(REGEX MATCHALL "-gencode arch=[^ ]+" CUDA_ARCH_FLAGS
149 + ${CMAKE_CUDA_FLAGS})
150 +
151 + # Remove all `-gencode` flags from `CMAKE_CUDA_FLAGS` since they will be modified
152 + # and passed back via the `CUDA_ARCHITECTURES` property.
153 + string(REGEX REPLACE "-gencode arch=[^ ]+ *" "" CMAKE_CUDA_FLAGS
154 + ${CMAKE_CUDA_FLAGS})
155 +endmacro()
156 +
157 +#
158 +# Extract unique CUDA architectures from a list of compute capabilities codes in
159 +# the form `<major><minor>[<letter>]`, convert them to the form sort
160 +# `<major>.<minor>`, dedupes them and then sorts them in ascending order and
161 +# stores them in `OUT_ARCHES`.
162 +#
163 +# Example:
164 +# CUDA_ARCH_FLAGS="-gencode arch=compute_75,code=sm_75;...;-gencode arch=compute_90a,code=sm_90a"
165 +# extract_unique_cuda_archs_ascending(OUT_ARCHES CUDA_ARCH_FLAGS)
166 +# OUT_ARCHES="7.5;...;9.0"
167 +function(extract_unique_cuda_archs_ascending OUT_ARCHES CUDA_ARCH_FLAGS)
168 + set(_CUDA_ARCHES)
169 + foreach(_ARCH ${CUDA_ARCH_FLAGS})
170 + string(REGEX MATCH "arch=compute_\([0-9]+a?\)" _COMPUTE ${_ARCH})
171 + if (_COMPUTE)
172 + set(_COMPUTE ${CMAKE_MATCH_1})
173 + endif()
174 +
175 + string_to_ver(_COMPUTE_VER ${_COMPUTE})
176 + list(APPEND _CUDA_ARCHES ${_COMPUTE_VER})
177 + endforeach()
178 +
179 + list(REMOVE_DUPLICATES _CUDA_ARCHES)
180 + list(SORT _CUDA_ARCHES COMPARE NATURAL ORDER ASCENDING)
181 + set(${OUT_ARCHES} ${_CUDA_ARCHES} PARENT_SCOPE)
182 +endfunction()
183 +
184 +#
185 +# For a specific file set the `-gencode` flag in compile options conditionally
186 +# for the CUDA language.
187 +#
188 +# Example:
189 +# set_gencode_flag_for_srcs(
190 +# SRCS "foo.cu"
191 +# ARCH "compute_75"
192 +# CODE "sm_75")
193 +# adds: "-gencode arch=compute_75,code=sm_75" to the compile options for
194 +# `foo.cu` (only for the CUDA language).
195 +#
196 +macro(set_gencode_flag_for_srcs)
197 + set(options)
198 + set(oneValueArgs ARCH CODE)
199 + set(multiValueArgs SRCS)
200 + cmake_parse_arguments(arg "${options}" "${oneValueArgs}"
201 + "${multiValueArgs}" ${ARGN} )
202 + set(_FLAG -gencode arch=${arg_ARCH},code=${arg_CODE})
203 + set_property(
204 + SOURCE ${arg_SRCS}
205 + APPEND PROPERTY
206 + COMPILE_OPTIONS "$<$<COMPILE_LANGUAGE:CUDA>:${_FLAG}>"
207 + )
208 +
209 + message(DEBUG "Setting gencode flag for ${arg_SRCS}: ${_FLAG}")
210 +endmacro(set_gencode_flag_for_srcs)
211 +
212 +#
213 +# For a list of source files set the `-gencode` flags in the files specific
214 +# compile options (specifically for the CUDA language).
215 +#
216 +# arguments are:
217 +# SRCS: list of source files
218 +# CUDA_ARCHS: list of CUDA architectures in the form `<major>.<minor>[letter]`
219 +# BUILD_PTX_FOR_ARCH: if set to true, then the PTX code will be built
220 +# for architecture `BUILD_PTX_FOR_ARCH` if there is a CUDA_ARCH in CUDA_ARCHS
221 +# that is larger than BUILD_PTX_FOR_ARCH.
222 +#
223 +macro(set_gencode_flags_for_srcs)
224 + set(options)
225 + set(oneValueArgs BUILD_PTX_FOR_ARCH)
226 + set(multiValueArgs SRCS CUDA_ARCHS)
227 + cmake_parse_arguments(arg "${options}" "${oneValueArgs}"
228 + "${multiValueArgs}" ${ARGN} )
229 +
230 + foreach(_ARCH ${arg_CUDA_ARCHS})
231 + string(REPLACE "." "" _ARCH "${_ARCH}")
232 + set_gencode_flag_for_srcs(
233 + SRCS ${arg_SRCS}
234 + ARCH "compute_${_ARCH}"
235 + CODE "sm_${_ARCH}")
236 + endforeach()
237 +
238 + if (${arg_BUILD_PTX_FOR_ARCH})
239 + list(SORT arg_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
240 + list(GET arg_CUDA_ARCHS -1 _HIGHEST_ARCH)
241 + if (_HIGHEST_ARCH VERSION_GREATER_EQUAL ${arg_BUILD_PTX_FOR_ARCH})
242 + string(REPLACE "." "" _PTX_ARCH "${arg_BUILD_PTX_FOR_ARCH}")
243 + set_gencode_flag_for_srcs(
244 + SRCS ${arg_SRCS}
245 + ARCH "compute_${_PTX_ARCH}"
246 + CODE "compute_${_PTX_ARCH}")
247 + endif()
248 + endif()
249 +endmacro()
250 +
251 +#
252 +# For the given `SRC_CUDA_ARCHS` list of gencode versions in the form
253 +# `<major>.<minor>[letter]` compute the "loose intersection" with the
254 +# `TGT_CUDA_ARCHS` list of gencodes.
255 +# The loose intersection is defined as:
256 +# { max{ x \in tgt | x <= y } | y \in src, { x \in tgt | x <= y } != {} }
257 +# where `<=` is the version comparison operator.
258 +# In other words, for each version in `TGT_CUDA_ARCHS` find the highest version
259 +# in `SRC_CUDA_ARCHS` that is less or equal to the version in `TGT_CUDA_ARCHS`.
260 +# We have special handling for 9.0a, if 9.0a is in `SRC_CUDA_ARCHS` and 9.0 is
261 +# in `TGT_CUDA_ARCHS` then we should remove 9.0a from `SRC_CUDA_ARCHS` and add
262 +# 9.0a to the result.
263 +# The result is stored in `OUT_CUDA_ARCHS`.
264 +#
265 +# Example:
266 +# SRC_CUDA_ARCHS="7.5;8.0;8.6;9.0;9.0a"
267 +# TGT_CUDA_ARCHS="8.0;8.9;9.0"
268 +# cuda_archs_loose_intersection(OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_ARCHS)
269 +# OUT_CUDA_ARCHS="8.0;8.6;9.0;9.0a"
270 +#
271 +function(cuda_archs_loose_intersection OUT_CUDA_ARCHS SRC_CUDA_ARCHS TGT_CUDA_ARCHS)
272 + list(REMOVE_DUPLICATES SRC_CUDA_ARCHS)
273 +
274 + # if 9.0a is in SRC_CUDA_ARCHS and 9.0 is in CUDA_ARCHS then we should
275 + # remove 9.0a from SRC_CUDA_ARCHS and add 9.0a to _CUDA_ARCHS
276 + set(_CUDA_ARCHS)
277 + if ("9.0a" IN_LIST SRC_CUDA_ARCHS)
278 + list(REMOVE_ITEM SRC_CUDA_ARCHS "9.0a")
279 + if ("9.0" IN_LIST TGT_CUDA_ARCHS)
280 + set(_CUDA_ARCHS "9.0a")
281 + endif()
282 + endif()
283 +
284 + list(SORT SRC_CUDA_ARCHS COMPARE NATURAL ORDER ASCENDING)
285 +
286 + # for each ARCH in CUDA_ARCHS find the highest arch in SRC_CUDA_ARCHS that is
287 + # less or eqault to ARCH
288 + foreach(_ARCH ${CUDA_ARCHS})
289 + set(_TMP_ARCH)
290 + foreach(_SRC_ARCH ${SRC_CUDA_ARCHS})
291 + if (_SRC_ARCH VERSION_LESS_EQUAL _ARCH)
292 + set(_TMP_ARCH ${_SRC_ARCH})
293 + else()
294 + break()
295 + endif()
296 + endforeach()
297 + if (_TMP_ARCH)
298 + list(APPEND _CUDA_ARCHS ${_TMP_ARCH})
299 + endif()
300 + endforeach()
301 +
302 + list(REMOVE_DUPLICATES _CUDA_ARCHS)
303 + set(${OUT_CUDA_ARCHS} ${_CUDA_ARCHS} PARENT_SCOPE)
304 +endfunction()
305 +
306 #
307 # Override the GPU architectures detected by cmake/torch and filter them by
308 # `GPU_SUPPORTED_ARCHES`. Sets the final set of architectures in
139 -# `GPU_ARCHES`.
309 +# `GPU_ARCHES`. This only applies to the HIP language since for CUDA we set
310 +# the architectures on a per file basis.
311 #
312 # Note: this is defined as a macro since it updates `CMAKE_CUDA_FLAGS`.
313 #
@@ -174,109 +345,7 @@ macro(override_gpu_arches GPU_ARCHES GPU_LANG GPU_SUPPORTED_ARCHES)
345 "None of the detected ROCm architectures: ${HIP_ARCHITECTURES} is"
346 " supported. Supported ROCm architectures are: ${_GPU_SUPPORTED_ARCHES_LIST}.")
347 endif()
177 -
178 - elseif(${GPU_LANG} STREQUAL "CUDA")
179 - #
180 - # Setup/process CUDA arch flags.
181 - #
182 - # The torch cmake setup hardcodes the detected architecture flags in
183 - # `CMAKE_CUDA_FLAGS`. Since `CMAKE_CUDA_FLAGS` is a "global" variable, it
184 - # can't modified on a per-target basis.
185 - # So, all the `-gencode` flags need to be extracted and removed from
186 - # `CMAKE_CUDA_FLAGS` for processing so they can be passed by another method.
187 - # Since it's not possible to use `target_compiler_options` for adding target
188 - # specific `-gencode` arguments, the target's `CUDA_ARCHITECTURES` property
189 - # must be used instead. This requires repackaging the architecture flags
190 - # into a format that cmake expects for `CUDA_ARCHITECTURES`.
191 - #
192 - # This is a bit fragile in that it depends on torch using `-gencode` as opposed
193 - # to one of the other nvcc options to specify architectures.
194 - #
195 - # Note: torch uses the `TORCH_CUDA_ARCH_LIST` environment variable to override
196 - # detected architectures.
197 - #
198 - message(DEBUG "initial CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
199 -
200 - # Extract all `-gencode` flags from `CMAKE_CUDA_FLAGS`
201 - string(REGEX MATCHALL "-gencode arch=[^ ]+" _CUDA_ARCH_FLAGS
202 - ${CMAKE_CUDA_FLAGS})
203 -
204 - # Remove all `-gencode` flags from `CMAKE_CUDA_FLAGS` since they will be modified
205 - # and passed back via the `CUDA_ARCHITECTURES` property.
206 - string(REGEX REPLACE "-gencode arch=[^ ]+ *" "" CMAKE_CUDA_FLAGS
207 - ${CMAKE_CUDA_FLAGS})
208 -
209 - # If this error is triggered, it might mean that torch has changed how it sets
210 - # up nvcc architecture code generation flags.
211 - if (NOT _CUDA_ARCH_FLAGS)
212 - message(FATAL_ERROR
213 - "Could not find any architecture related code generation flags in "
214 - "CMAKE_CUDA_FLAGS. (${CMAKE_CUDA_FLAGS})")
215 - endif()
216 -
217 - message(DEBUG "final CMAKE_CUDA_FLAGS: ${CMAKE_CUDA_FLAGS}")
218 - message(DEBUG "arch flags: ${_CUDA_ARCH_FLAGS}")
219 -
220 - # Initialize the architecture lists to empty.
221 - set(${GPU_ARCHES})
222 -
223 - # Process each `gencode` flag.
224 - foreach(_ARCH ${_CUDA_ARCH_FLAGS})
225 - # For each flag, extract the version number and whether it refers to PTX
226 - # or native code.
227 - # Note: if a regex matches then `CMAKE_MATCH_1` holds the binding
228 - # for that match.
229 -
230 - string(REGEX MATCH "arch=compute_\([0-9]+a?\)" _COMPUTE ${_ARCH})
231 - if (_COMPUTE)
232 - set(_COMPUTE ${CMAKE_MATCH_1})
233 - endif()
234 -
235 - string(REGEX MATCH "code=sm_\([0-9]+a?\)" _SM ${_ARCH})
236 - if (_SM)
237 - set(_SM ${CMAKE_MATCH_1})
238 - endif()
239 -
240 - string(REGEX MATCH "code=compute_\([0-9]+a?\)" _CODE ${_ARCH})
241 - if (_CODE)
242 - set(_CODE ${CMAKE_MATCH_1})
243 - endif()
244 -
245 - # Make sure the virtual architecture can be matched.
246 - if (NOT _COMPUTE)
247 - message(FATAL_ERROR
248 - "Could not determine virtual architecture from: ${_ARCH}.")
249 - endif()
250 -
251 - # One of sm_ or compute_ must exist.
252 - if ((NOT _SM) AND (NOT _CODE))
253 - message(FATAL_ERROR
254 - "Could not determine a codegen architecture from: ${_ARCH}.")
255 - endif()
256 -
257 - if (_SM)
258 - # -real suffix let CMake to only generate elf code for the kernels.
259 - # we want this, otherwise the added ptx (default) will increase binary size.
260 - set(_VIRT "-real")
261 - set(_CODE_ARCH ${_SM})
262 - else()
263 - # -virtual suffix let CMake to generate ptx code for the kernels.
264 - set(_VIRT "-virtual")
265 - set(_CODE_ARCH ${_CODE})
266 - endif()
267 -
268 - # Check if the current version is in the supported arch list.
269 - string_to_ver(_CODE_VER ${_CODE_ARCH})
270 - if (NOT _CODE_VER IN_LIST _GPU_SUPPORTED_ARCHES_LIST)
271 - message(STATUS "discarding unsupported CUDA arch ${_VER}.")
272 - continue()
273 - endif()
274 -
275 - # Add it to the arch list.
276 - list(APPEND ${GPU_ARCHES} "${_CODE_ARCH}${_VIRT}")
277 - endforeach()
348 endif()
279 - message(STATUS "${GPU_LANG} target arches: ${${GPU_ARCHES}}")
349 endmacro()
350
351 #
csrc/core/registration.h
+5
@@ -12,6 +12,11 @@
12 // could be a macro instead of a literal token.
13 #define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE)
14
15 +// A version of the TORCH_LIBRARY_IMPL macro that expands the NAME, i.e. so NAME
16 +// could be a macro instead of a literal token.
17 +#define TORCH_LIBRARY_IMPL_EXPAND(NAME, DEVICE, MODULE) \
18 + TORCH_LIBRARY_IMPL(NAME, DEVICE, MODULE)
19 +
20 // REGISTER_EXTENSION allows the shared library to be loaded and initialized
21 // via python's import statement.
22 #define REGISTER_EXTENSION(NAME) \
csrc/moe/marlin_moe_ops.cu
+5
@@ -27,6 +27,7 @@
27
28 #include "core/exception.hpp"
29 #include "core/scalar_type.hpp"
30 +#include "core/registration.h"
31 #include "marlin_kernels/marlin_moe_kernel_ku4b8.h"
32 #include "marlin_kernels/marlin_moe_kernel_ku8b128.h"
33
@@ -552,3 +553,7 @@ torch::Tensor marlin_gemm_moe(
553 thread_n, sms, max_par, replicate_input, apply_weights);
554 return c;
555 }
556 +
557 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
558 + m.impl("marlin_gemm_moe", &marlin_gemm_moe);
559 +}
csrc/moe/marlin_moe_ops.h deleted
-15
@@ -1,15 +0,0 @@
1 -#pragma once
2 -
3 -#include <torch/all.h>
4 -
5 -#include "core/scalar_type.hpp"
6 -
7 -torch::Tensor marlin_gemm_moe(
8 - const torch::Tensor& a, const torch::Tensor& b_q_weights,
9 - const torch::Tensor& sorted_ids, const torch::Tensor& topk_weights,
10 - const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
11 - const torch::Tensor& g_idx, const torch::Tensor& perm,
12 - torch::Tensor& workspace, vllm::ScalarTypeTorchPtr const& b_q_type,
13 - int64_t size_m, int64_t size_n, int64_t size_k, bool is_k_full,
14 - int64_t num_experts, int64_t topk, int64_t moe_block_size,
15 - bool replicate_input, bool apply_weights);
csrc/moe/torch_bindings.cpp
+1 -2
@@ -1,6 +1,5 @@
1 #include "core/registration.h"
2 #include "moe_ops.h"
3 -#include "marlin_moe_ops.h"
3
4 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
5 // Apply topk softmax to the gating outputs.
@@ -18,7 +17,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
17 "int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
18 "int moe_block_size, bool replicate_input, bool apply_weights)"
19 " -> Tensor");
21 - m.impl("marlin_gemm_moe", torch::kCUDA, &marlin_gemm_moe);
20 + // conditionally compiled so impl registration is in source file
21 #endif
22 }
23
csrc/ops.h
-68
@@ -90,63 +90,8 @@ torch::Tensor awq_dequantize(torch::Tensor _kernel,
90 torch::Tensor _zeros, int64_t split_k_iters,
91 int64_t thx, int64_t thy);
92
93 -torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
94 - torch::Tensor& b_scales, torch::Tensor& workspace,
95 - int64_t size_m, int64_t size_n, int64_t size_k);
96 -
97 -namespace machete {
98 -
99 -std::vector<std::string> supported_schedules(
100 - vllm::ScalarTypeTorchPtr const& btype);
101 -
102 -torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
103 - vllm::ScalarTypeTorchPtr const& btype,
104 - c10::optional<torch::Tensor> const& scales,
105 - c10::optional<torch::Tensor> const& zeros,
106 - c10::optional<int64_t> group_size,
107 - c10::optional<torch::Tensor> const& C,
108 - c10::optional<double> alpha, c10::optional<double> beta,
109 - c10::optional<std::string> schedule);
110 -
111 -torch::Tensor prepack_B(torch::Tensor const& B,
112 - vllm::ScalarTypeTorchPtr const& btype);
113 -
114 -}; // namespace machete
115 -
93 torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
94
118 -torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
119 - torch::Tensor& b_meta,
120 - torch::Tensor& b_scales,
121 - torch::Tensor& workspace,
122 - vllm::ScalarTypeTorchPtr const& b_q_type,
123 - int64_t size_m, int64_t size_n,
124 - int64_t size_k);
125 -
126 -torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
127 - torch::Tensor& b_scales, torch::Tensor& b_zeros,
128 - torch::Tensor& g_idx, torch::Tensor& perm,
129 - torch::Tensor& workspace,
130 - vllm::ScalarTypeTorchPtr const& b_q_type,
131 - int64_t size_m, int64_t size_n, int64_t size_k,
132 - bool is_k_full, bool has_zp,
133 - bool use_fp32_reduce);
134 -
135 -torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
136 - int64_t size_k, int64_t size_n,
137 - int64_t num_bits);
138 -
139 -torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
140 - torch::Tensor& perm, c10::SymInt size_k,
141 - c10::SymInt size_n, int64_t num_bits);
142 -
143 -torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
144 - int64_t size_n, int64_t num_bits);
145 -
146 -torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
147 - c10::SymInt size_k, c10::SymInt size_n,
148 - int64_t num_bits);
149 -
95 torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
96 int64_t n);
97
@@ -156,11 +101,6 @@ torch::Tensor ggml_mul_mat_vec_a8(torch::Tensor W, torch::Tensor X,
101 torch::Tensor ggml_mul_mat_a8(torch::Tensor W, torch::Tensor X, int64_t type,
102 int64_t row);
103
159 -torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
160 - torch::Tensor& b_scales, torch::Tensor& workspace,
161 - int64_t num_bits, int64_t size_m, int64_t size_n,
162 - int64_t size_k);
163 -
104 bool cutlass_scaled_mm_supports_fp8(int64_t cuda_device_capability);
105
106 void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
@@ -175,14 +115,6 @@ void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
115 torch::Tensor const& azp_adj,
116 c10::optional<torch::Tensor> const& azp,
117 c10::optional<torch::Tensor> const& bias);
178 -
179 -torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
180 - torch::Tensor const& b_q_weight,
181 - torch::Tensor const& s_tok,
182 - torch::Tensor const& s_ch,
183 - torch::Tensor const& s_group,
184 - torch::Tensor& workspace, int64_t size_m,
185 - int64_t size_n, int64_t size_k);
118 #endif
119
120 void static_scaled_int8_quant(torch::Tensor& out, torch::Tensor const& input,
csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu
+51 -25
@@ -21,7 +21,7 @@ void cutlass_scaled_mm_sm89(torch::Tensor& c, torch::Tensor const& a,
21 torch::Tensor const& b_scales,
22 c10::optional<torch::Tensor> const& bias);
23
24 -#if defined CUDA_VERSION && CUDA_VERSION >= 12000
24 +#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
25 void cutlass_scaled_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
26 torch::Tensor const& b,
27 torch::Tensor const& a_scales,
@@ -114,26 +114,39 @@ void cutlass_scaled_mm(torch::Tensor& c, torch::Tensor const& a,
114
115 at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
116 int32_t version_num = get_sm_version_num();
117 - if (version_num >= 90) {
118 - // Hopper
117 + // Hopper
118
120 - // Guard against compilation issues for sm90 kernels
121 -#if defined CUDA_VERSION && CUDA_VERSION >= 12000
119 + // Guard against compilation issues for sm90 kernels
120 +#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
121 + if (version_num >= 90) {
122 cutlass_scaled_mm_sm90(c, a, b, a_scales, b_scales, bias);
123 -#else
124 - cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
123 + return;
124 + }
125 #endif
126 - } else if (version_num == 89) {
126 +
127 +#if defined ENABLE_SCALED_MM_C2X && ENABLE_SCALED_MM_C2X
128 + if (version_num == 89) {
129 // Ada Lovelace
130 cutlass_scaled_mm_sm89(c, a, b, a_scales, b_scales, bias);
129 - } else if (version_num >= 80) {
131 + return;
132 + }
133 +
134 + if (version_num >= 80) {
135 // Ampere
136 cutlass_scaled_mm_sm80(c, a, b, a_scales, b_scales, bias);
132 - } else {
133 - // Turing
134 - TORCH_CHECK(version_num >= 75);
135 - cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
137 + return;
138 }
139 +
140 + // Turing
141 + TORCH_CHECK(version_num >= 75);
142 + cutlass_scaled_mm_sm75(c, a, b, a_scales, b_scales, bias);
143 +#endif
144 +
145 + TORCH_CHECK_NOT_IMPLEMENTED(
146 + false,
147 + "No compiled cutlass_scaled_mm for a compute capability less than "
148 + "CUDA device capability: ",
149 + version_num);
150 }
151
152 void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
@@ -174,25 +187,38 @@ void cutlass_scaled_mm_azp(torch::Tensor& c, torch::Tensor const& a,
187 "currently bias dtype must match output dtype ", c.dtype());
188
189 at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
190 +
191 int32_t version_num = get_sm_version_num();
178 - if (version_num >= 90) {
179 - // Hopper
192
181 - // Guard against compilation issues for sm90 kernels
182 -#if defined CUDA_VERSION && CUDA_VERSION >= 12000
193 +#if defined ENABLE_SCALED_MM_C3X && ENABLE_SCALED_MM_C3X
194 + if (version_num >= 90) {
195 cutlass_scaled_mm_azp_sm90(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
184 -#else
185 - cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
196 + return;
197 + }
198 #endif
187 - } else if (version_num == 89) {
199 +
200 +#if defined ENABLE_SCALED_MM_C2X && ENABLE_SCALED_MM_C2X
201 + if (version_num == 89) {
202 // Ada Lovelace
203 cutlass_scaled_mm_azp_sm89(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
190 - } else if (version_num >= 80) {
204 + return;
205 + }
206 +
207 + if (version_num >= 80) {
208 // Ampere
209 cutlass_scaled_mm_azp_sm80(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
193 - } else {
194 - // Turing
195 - TORCH_CHECK(version_num >= 75);
196 - cutlass_scaled_mm_azp_sm75(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
210 + return;
211 }
212 +
213 + // Turing
214 + TORCH_CHECK(version_num >= 75);
215 + cutlass_scaled_mm_azp_sm75(c, a, b, a_scales, b_scales, azp_adj, azp, bias);
216 + return;
217 +#endif
218 +
219 + TORCH_CHECK_NOT_IMPLEMENTED(
220 + false,
221 + "No compiled cutlass_scaled_mm_azp for a compute capability less than "
222 + "CUDA device capability: ",
223 + version_num);
224 }
\ No newline at end of file
csrc/quantization/fp8/fp8_marlin.cu
+6
@@ -22,6 +22,8 @@
22 #include "../gptq_marlin/marlin.cuh"
23 #include "../gptq_marlin/marlin_dtypes.cuh"
24
25 +#include "core/registration.h"
26 +
27 using namespace marlin;
28
29 #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
@@ -1303,3 +1305,7 @@ torch::Tensor fp8_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1305 }
1306
1307 #endif
1308 +
1309 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1310 + m.impl("fp8_marlin_gemm", &fp8_marlin_gemm);
1311 +}
\ No newline at end of file
csrc/quantization/gptq_marlin/awq_marlin_repack.cu
+24 -37
@@ -1,25 +1,6 @@
1 #include "marlin.cuh"
2
3 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4 -
5 -namespace marlin {
6 -
7 -template <int const num_threads, int const num_bits, bool const has_perm>
8 -__global__ void awq_marlin_repack_kernel(
9 - uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr,
10 - int size_k, int size_n) {}
11 -
12 -} // namespace marlin
13 -
14 -torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
15 - int64_t size_k, int64_t size_n,
16 - int64_t num_bits) {
17 - TORCH_CHECK_NOT_IMPLEMENTED(
18 - false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
19 - return torch::empty({1, 1});
20 -}
21 -
22 -#else
3 +#include "core/registration.h"
4
5 namespace marlin {
6
@@ -122,7 +103,7 @@ __global__ void awq_marlin_repack_kernel(
103 }
104
105 uint32_t vals[8];
125 - #pragma unroll
106 +#pragma unroll
107 for (int i = 0; i < 4; i++) {
108 int cur_elem = tc_row + tc_offsets[i];
109
@@ -143,7 +124,7 @@ __global__ void awq_marlin_repack_kernel(
124 constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
125
126 uint32_t res = 0;
146 - #pragma unroll
127 +#pragma unroll
128 for (int i = 0; i < 8; i++) {
129 res |= vals[pack_idx[i]] << (i * 4);
130 }
@@ -155,7 +136,7 @@ __global__ void awq_marlin_repack_kernel(
136
137 uint32_t res1 = 0;
138 uint32_t res2 = 0;
158 - #pragma unroll
139 +#pragma unroll
140 for (int i = 0; i < 4; i++) {
141 res1 |= vals[pack_idx[i]] << (i * 8);
142 res2 |= vals[4 + pack_idx[i]] << (i * 8);
@@ -167,21 +148,21 @@ __global__ void awq_marlin_repack_kernel(
148 };
149
150 auto start_pipes = [&](int k_tile_id, int n_tile_id) {
170 - #pragma unroll
151 +#pragma unroll
152 for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
153 fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
154 }
155
156 wait_for_stage();
157 };
177 - #pragma unroll
158 +#pragma unroll
159 for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
160 int n_tile_id = 0;
161
162 start_pipes(k_tile_id, n_tile_id);
163
164 while (n_tile_id < n_tiles) {
184 - #pragma unroll
165 +#pragma unroll
166 for (int pipe = 0; pipe < repack_stages; pipe++) {
167 fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
168 n_tile_id + pipe + repack_stages - 1);
@@ -195,15 +176,15 @@ __global__ void awq_marlin_repack_kernel(
176
177 } // namespace marlin
178
198 - #define CALL_IF(NUM_BITS) \
199 - else if (num_bits == NUM_BITS) { \
200 - cudaFuncSetAttribute( \
201 - marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS>, \
202 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
203 - marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS> \
204 - <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
205 - b_q_weight_ptr, out_ptr, size_k, size_n); \
206 - }
179 +#define CALL_IF(NUM_BITS) \
180 + else if (num_bits == NUM_BITS) { \
181 + cudaFuncSetAttribute( \
182 + marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS>, \
183 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
184 + marlin::awq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS> \
185 + <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
186 + b_q_weight_ptr, out_ptr, size_k, size_n); \
187 + }
188
189 torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
190 int64_t size_n, int64_t num_bits) {
@@ -266,8 +247,6 @@ torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k,
247 return out;
248 }
249
269 -#endif
270 -
250 torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
251 c10::SymInt size_k, c10::SymInt size_n,
252 int64_t num_bits) {
@@ -279,3 +258,11 @@ torch::Tensor awq_marlin_repack_meta(torch::Tensor& b_q_weight,
258 {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
259 options);
260 }
261 +
262 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
263 + m.impl("awq_marlin_repack", &awq_marlin_repack);
264 +}
265 +
266 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, Meta, m) {
267 + m.impl("awq_marlin_repack", &awq_marlin_repack_meta);
268 +}
\ No newline at end of file
csrc/quantization/gptq_marlin/gptq_marlin.cu
+6
@@ -23,6 +23,8 @@
23 #include "marlin_dtypes.cuh"
24 #include "core/scalar_type.hpp"
25
26 +#include "core/registration.h"
27 +
28 #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
29 static_assert(std::is_same<scalar_t, half>::value || \
30 std::is_same<scalar_t, nv_bfloat16>::value, \
@@ -2297,3 +2299,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2299 }
2300
2301 #endif
2302 +
2303 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
2304 + m.impl("gptq_marlin_gemm", &gptq_marlin_gemm);
2305 +}
\ No newline at end of file
csrc/quantization/gptq_marlin/gptq_marlin_repack.cu
+27 -41
@@ -1,26 +1,6 @@
1 #include "marlin.cuh"
2
3 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
4 -
5 -namespace marlin {
6 -
7 -template <int const num_threads, int const num_bits, bool const has_perm>
8 -__global__ void gptq_marlin_repack_kernel(
9 - uint32_t const* __restrict__ b_q_weight_ptr,
10 - uint32_t const* __restrict__ perm_ptr, uint32_t* __restrict__ out_ptr,
11 - int size_k, int size_n) {}
12 -
13 -} // namespace marlin
14 -
15 -torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
16 - int64_t size_k, int64_t size_n,
17 - int64_t num_bits) {
18 - TORCH_CHECK_NOT_IMPLEMENTED(
19 - false, "marlin_repack_from_gptq(..) requires CUDA_ARCH >= 8.0");
20 - return torch::empty({1, 1});
21 -}
22 -
23 -#else
3 +#include "core/registration.h"
4
5 namespace marlin {
6
@@ -174,13 +154,13 @@ __global__ void gptq_marlin_repack_kernel(
154 uint32_t b1_vals[tile_ints];
155 uint32_t b2_vals[tile_ints];
156
177 - #pragma unroll
157 +#pragma unroll
158 for (int i = 0; i < tile_ints; i++) {
159 b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i];
160 b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i];
161 }
162
183 - #pragma unroll
163 +#pragma unroll
164 for (int i = 0; i < 4; i++) {
165 int cur_elem = tc_row + tc_offsets[i];
166 int cur_int = cur_elem / pack_factor;
@@ -200,7 +180,7 @@ __global__ void gptq_marlin_repack_kernel(
180 constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7};
181
182 uint32_t res = 0;
203 - #pragma unroll
183 +#pragma unroll
184 for (int i = 0; i < 8; i++) {
185 res |= vals[pack_idx[i]] << (i * 4);
186 }
@@ -212,7 +192,7 @@ __global__ void gptq_marlin_repack_kernel(
192
193 uint32_t res1 = 0;
194 uint32_t res2 = 0;
215 - #pragma unroll
195 +#pragma unroll
196 for (int i = 0; i < 4; i++) {
197 res1 |= vals[pack_idx[i]] << (i * 8);
198 res2 |= vals[4 + pack_idx[i]] << (i * 8);
@@ -224,14 +204,14 @@ __global__ void gptq_marlin_repack_kernel(
204 };
205
206 auto start_pipes = [&](int k_tile_id, int n_tile_id) {
227 - #pragma unroll
207 +#pragma unroll
208 for (int pipe = 0; pipe < repack_stages - 1; pipe++) {
209 fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe);
210 }
211
212 wait_for_stage();
213 };
234 - #pragma unroll
214 +#pragma unroll
215 for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) {
216 int n_tile_id = 0;
217
@@ -242,7 +222,7 @@ __global__ void gptq_marlin_repack_kernel(
222 start_pipes(k_tile_id, n_tile_id);
223
224 while (n_tile_id < n_tiles) {
245 - #pragma unroll
225 +#pragma unroll
226 for (int pipe = 0; pipe < repack_stages; pipe++) {
227 fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id,
228 n_tile_id + pipe + repack_stages - 1);
@@ -256,17 +236,17 @@ __global__ void gptq_marlin_repack_kernel(
236
237 } // namespace marlin
238
259 - #define CALL_IF(NUM_BITS, HAS_PERM) \
260 - else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
261 - cudaFuncSetAttribute( \
262 - marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
263 - HAS_PERM>, \
264 - cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
265 - marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
266 - HAS_PERM> \
267 - <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
268 - b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
269 - }
239 +#define CALL_IF(NUM_BITS, HAS_PERM) \
240 + else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \
241 + cudaFuncSetAttribute( \
242 + marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
243 + HAS_PERM>, \
244 + cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); \
245 + marlin::gptq_marlin_repack_kernel<marlin::repack_threads, NUM_BITS, \
246 + HAS_PERM> \
247 + <<<blocks, marlin::repack_threads, max_shared_mem, stream>>>( \
248 + b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \
249 + }
250
251 torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
252 int64_t size_k, int64_t size_n,
@@ -341,8 +321,6 @@ torch::Tensor gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm,
321 return out;
322 }
323
344 -#endif
345 -
324 torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
325 torch::Tensor& perm, c10::SymInt size_k,
326 c10::SymInt size_n, int64_t num_bits) {
@@ -354,3 +332,11 @@ torch::Tensor gptq_marlin_repack_meta(torch::Tensor& b_q_weight,
332 {size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor},
333 options);
334 }
335 +
336 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
337 + m.impl("gptq_marlin_repack", &gptq_marlin_repack);
338 +}
339 +
340 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, Meta, m) {
341 + m.impl("gptq_marlin_repack", &gptq_marlin_repack_meta);
342 +}
\ No newline at end of file
csrc/quantization/machete/generate.py
+1 -1
@@ -284,7 +284,7 @@ mm_impl_template = create_template(IMPL_TEMPLATE)
284 prepack_dispatch_template = create_template(PREPACK_TEMPLATE)
285
286
287 -def create_sources(impl_config: ImplConfig, num_impl_files=2):
287 +def create_sources(impl_config: ImplConfig, num_impl_files=1):
288 sources = []
289
290 type_name = generate_type_signature(impl_config.type_config)
csrc/quantization/machete/machete_prepack_kernel.cuh
+3 -4
@@ -34,10 +34,9 @@ static __global__ void prepack_B_kernel(BInTensor B_in,
34 }
35
36 template <typename PrepackedLayoutB, typename InLayout>
37 -static void prepack_B(cudaStream_t stream,
38 - typename PrepackedLayoutB::ElementB const* B_in_ptr,
39 - InLayout B_layout,
40 - typename PrepackedLayoutB::ElementB* B_out_ptr) {
37 +static void prepack_B_template(
38 + cudaStream_t stream, typename PrepackedLayoutB::ElementB const* B_in_ptr,
39 + InLayout B_layout, typename PrepackedLayoutB::ElementB* B_out_ptr) {
40 using TileShapeNKL =
41 decltype(append(typename PrepackedLayoutB::PPBlockShape_NK{}, _1{}));
42 auto ilvd_NKbNbKL_to_offset =
csrc/quantization/machete/machete_prepack_launcher.cuh
+2 -2
@@ -55,8 +55,8 @@ torch::Tensor prepack_impl(torch::Tensor const B) {
55 // Allocate output
56 torch::Tensor D = torch::empty_like(B, {}, at::MemoryFormat::Contiguous);
57
58 - prepack_B<PrepackedLayoutB>(stream, B_ptr, layout_Bt,
59 - static_cast<ElementB*>(D.mutable_data_ptr()));
58 + prepack_B_template<PrepackedLayoutB>(
59 + stream, B_ptr, layout_Bt, static_cast<ElementB*>(D.mutable_data_ptr()));
60
61 return D;
62 };
csrc/quantization/machete/machete_pytorch.cu
+9 -5
@@ -2,6 +2,8 @@
2 #include "machete_prepack_launcher.cuh"
3 #include "core/scalar_type.hpp"
4
5 +#include "core/registration.h"
6 +
7 namespace machete {
8
9 using namespace vllm;
@@ -78,14 +80,16 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
80 }
81
82 torch::Tensor prepack_B(torch::Tensor const& B,
81 - ScalarTypeTorchPtr const& btype) {
82 -#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
83 + vllm::ScalarTypeTorchPtr const& btype) {
84 return scalar_type_dispatch(*btype, [&](auto BType) {
85 return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
86 });
86 -#else
87 - TORCH_CHECK(false, "Machete requires CUDA 12.0 or later");
88 -#endif
87 +}
88 +
89 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
90 + m.impl("machete_prepack_B", &prepack_B);
91 + m.impl("machete_gemm", &gemm);
92 + m.impl("machete_supported_schedules", &supported_schedules);
93 }
94
95 }; // namespace machete
csrc/quantization/marlin/dense/marlin_cuda_kernel.cu
+5
@@ -26,6 +26,7 @@
26 #include <iostream>
27
28 #include "common/base.h"
29 +#include "core/registration.h"
30
31 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
32 #include "common/mem.h"
@@ -1066,3 +1067,7 @@ torch::Tensor marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1067
1068 return c;
1069 }
1070 +
1071 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1072 + m.impl("marlin_gemm", &marlin_gemm);
1073 +}
csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu
+5
@@ -30,6 +30,7 @@
30 #include <iostream>
31
32 #include "../dense/common/base.h"
33 +#include "core/registration.h"
34
35 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
36 #include "../dense/common/mem.h"
@@ -1241,3 +1242,7 @@ torch::Tensor marlin_qqq_gemm(torch::Tensor const& a,
1242
1243 return d;
1244 }
1245 +
1246 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1247 + m.impl("marlin_qqq_gemm", &marlin_qqq_gemm);
1248 +}
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+5
@@ -28,6 +28,7 @@
28
29 #include "common/base.h"
30 #include "core/scalar_type.hpp"
31 +#include "core/registration.h"
32
33 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800
34
@@ -1134,3 +1135,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1135
1136 return c;
1137 }
1138 +
1139 +TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
1140 + m.impl("gptq_marlin_24_gemm", &gptq_marlin_24_gemm);
1141 +}
csrc/torch_bindings.cpp
+12 -12
@@ -167,7 +167,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
167 ops.def(
168 "marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
169 "Tensor! workspace, int size_m, int size_n, int size_k) -> Tensor");
170 - ops.impl("marlin_gemm", torch::kCUDA, &marlin_gemm);
170 + // conditionally compiled so impl in source file
171
172 // Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
173 ops.def(
@@ -175,22 +175,24 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
175 "Tensor b_scales, Tensor workspace, "
176 "__torch__.torch.classes._core_C.ScalarType b_q_type, "
177 "int size_m, int size_n, int size_k) -> Tensor");
178 - ops.impl("gptq_marlin_24_gemm", torch::kCUDA, &gptq_marlin_24_gemm);
178 + // conditionally compiled so impl in source file
179
180 // Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
181 - ops.def("machete_supported_schedules", &machete::supported_schedules);
181 + ops.def(
182 + "machete_supported_schedules("
183 + " __torch__.torch.classes._core_C.ScalarType btype"
184 + ") -> str[]");
185 ops.def(
186 "machete_gemm(Tensor A, Tensor B,"
187 " __torch__.torch.classes._core_C.ScalarType btype,"
188 " Tensor? scales, Tensor? zeros, int? group_size,"
189 " Tensor? C, float? alpha, float? beta, str? schedule)"
190 "-> Tensor");
188 - ops.impl("machete_gemm", torch::kCUDA, &machete::gemm);
191 ops.def(
192 "machete_prepack_B(Tensor B,"
193 " __torch__.torch.classes._core_C.ScalarType btype)"
194 "-> Tensor");
193 - ops.impl("machete_prepack_B", torch::kCUDA, &machete::prepack_B);
195 + // conditionally compiled so impl registration is in source file
196
197 ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
198 ops.impl("permute_cols", torch::kCUDA, &permute_cols);
@@ -202,21 +204,19 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
204 "__torch__.torch.classes._core_C.ScalarType b_q_type, "
205 "int size_m, int size_n, int size_k, bool is_k_full, "
206 "bool has_zp, bool use_fp32_reduce) -> Tensor");
205 - ops.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm);
207 + // conditionally compiled so impl registration is in source file
208
209 // gptq_marlin repack from GPTQ.
210 ops.def(
211 "gptq_marlin_repack(Tensor b_q_weight, Tensor perm, "
212 "SymInt size_k, SymInt size_n, int num_bits) -> Tensor");
211 - ops.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack);
212 - ops.impl("gptq_marlin_repack", torch::kMeta, &gptq_marlin_repack_meta);
213 + // conditionally compiled so impl registrations are in source file
214
215 // awq_marlin repack from AWQ.
216 ops.def(
217 "awq_marlin_repack(Tensor b_q_weight, SymInt size_k, "
218 "SymInt size_n, int num_bits) -> Tensor");
218 - ops.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack);
219 - ops.impl("awq_marlin_repack", torch::kMeta, &awq_marlin_repack_meta);
219 + // conditionally compiled so impl registrations are in source file
220
221 // Dequantization for GGML.
222 ops.def("ggml_dequantize(Tensor W, int type, int m, int n) -> Tensor");
@@ -237,7 +237,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
237 "fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
238 "Tensor! workspace, int num_bits, int size_m, int size_n, "
239 "int size_k) -> Tensor");
240 - ops.impl("fp8_marlin_gemm", torch::kCUDA, &fp8_marlin_gemm);
240 + // conditionally compiled so impl registration is in source file
241
242 // marlin_qqq_gemm for QQQ.
243 ops.def(
@@ -245,7 +245,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
245 "Tensor s_tok, Tensor s_ch, Tensor s_group, "
246 "Tensor! workspace, int size_m, int size_n, "
247 "int size_k) -> Tensor");
248 - ops.impl("marlin_qqq_gemm", torch::kCUDA, &marlin_qqq_gemm);
248 + // conditionally compiled so impl registration is in source file
249
250 // CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
251 // quantization, as well as bias
tools/report_build_time_ninja.py new
+311
@@ -0,0 +1,311 @@
1 +#!/usr/bin/env python3
2 +# Copyright (c) 2018 The Chromium Authors. All rights reserved.
3 +# Use of this source code is governed by a BSD-style license that can be
4 +# found in the LICENSE file.
5 +
6 +# Modified version of: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/refs/heads/main/post_build_ninja_summary.py
7 +"""Summarize the last ninja build, invoked with ninja's -C syntax.
8 +
9 +> python3 tools/report_build_time_ninja.py -C build/..
10 +
11 +Typical output looks like this:
12 +```
13 + Longest build steps for .cpp.o:
14 + 1.0 weighted s to build ...torch_bindings.cpp.o (12.4 s elapsed time)
15 + 2.0 weighted s to build ..._attn_c.dir/csrc... (23.5 s elapsed time)
16 + 2.6 weighted s to build ...torch_bindings.cpp.o (31.5 s elapsed time)
17 + 3.2 weighted s to build ...torch_bindings.cpp.o (38.5 s elapsed time)
18 + Longest build steps for .so (linking):
19 + 0.1 weighted s to build _core_C.abi3.so (0.7 s elapsed time)
20 + 0.1 weighted s to build _moe_C.abi3.so (1.0 s elapsed time)
21 + 0.5 weighted s to build ...flash_attn_c.abi3.so (1.1 s elapsed time)
22 + 6.2 weighted s to build _C.abi3.so (6.2 s elapsed time)
23 + Longest build steps for .cu.o:
24 + 15.3 weighted s to build ...machete_mm_... (183.5 s elapsed time)
25 + 15.3 weighted s to build ...machete_mm_... (183.5 s elapsed time)
26 + 15.3 weighted s to build ...machete_mm_... (183.6 s elapsed time)
27 + 15.3 weighted s to build ...machete_mm_... (183.7 s elapsed time)
28 + 15.5 weighted s to build ...machete_mm_... (185.6 s elapsed time)
29 + 15.5 weighted s to build ...machete_mm_... (185.9 s elapsed time)
30 + 15.5 weighted s to build ...machete_mm_... (186.2 s elapsed time)
31 + 37.4 weighted s to build ...scaled_mm_c3x.cu... (449.0 s elapsed time)
32 + 43.9 weighted s to build ...scaled_mm_c2x.cu... (527.4 s elapsed time)
33 + 344.8 weighted s to build ...attention_...cu.o (1087.2 s elapsed time)
34 + 1110.0 s weighted time (10120.4 s elapsed time sum, 9.1x parallelism)
35 + 134 build steps completed, average of 0.12/s
36 +```
37 +"""
38 +
39 +import argparse
40 +import errno
41 +import fnmatch
42 +import os
43 +import sys
44 +from collections import defaultdict
45 +
46 +# The number of long build times to report:
47 +long_count = 10
48 +# The number of long times by extension to report
49 +long_ext_count = 10
50 +
51 +
52 +class Target:
53 + """Represents a single line read for a .ninja_log file."""
54 +
55 + def __init__(self, start, end):
56 + """Creates a target object by passing in the start/end times in seconds
57 + as a float."""
58 + self.start = start
59 + self.end = end
60 + # A list of targets, appended to by the owner of this object.
61 + self.targets = []
62 + self.weighted_duration = 0.0
63 +
64 + def Duration(self):
65 + """Returns the task duration in seconds as a float."""
66 + return self.end - self.start
67 +
68 + def SetWeightedDuration(self, weighted_duration):
69 + """Sets the duration, in seconds, passed in as a float."""
70 + self.weighted_duration = weighted_duration
71 +
72 + def WeightedDuration(self):
73 + """Returns the task's weighted duration in seconds as a float.
74 +
75 + Weighted_duration takes the elapsed time of the task and divides it
76 + by how many other tasks were running at the same time. Thus, it
77 + represents the approximate impact of this task on the total build time,
78 + with serialized or serializing steps typically ending up with much
79 + longer weighted durations.
80 + weighted_duration should always be the same or shorter than duration.
81 + """
82 + # Allow for modest floating-point errors
83 + epsilon = 0.000002
84 + if (self.weighted_duration > self.Duration() + epsilon):
85 + print('%s > %s?' % (self.weighted_duration, self.Duration()))
86 + assert (self.weighted_duration <= self.Duration() + epsilon)
87 + return self.weighted_duration
88 +
89 + def DescribeTargets(self):
90 + """Returns a printable string that summarizes the targets."""
91 + # Some build steps generate dozens of outputs - handle them sanely.
92 + # The max_length was chosen so that it can fit most of the long
93 + # single-target names, while minimizing word wrapping.
94 + result = ', '.join(self.targets)
95 + max_length = 65
96 + if len(result) > max_length:
97 + result = result[:max_length] + '...'
98 + return result
99 +
100 +
101 +# Copied with some modifications from ninjatracing
102 +def ReadTargets(log, show_all):
103 + """Reads all targets from .ninja_log file |log_file|, sorted by duration.
104 +
105 + The result is a list of Target objects."""
106 + header = log.readline()
107 + assert header == '# ninja log v5\n', \
108 + 'unrecognized ninja log version %r' % header
109 + targets_dict = {}
110 + last_end_seen = 0.0
111 + for line in log:
112 + parts = line.strip().split('\t')
113 + if len(parts) != 5:
114 + # If ninja.exe is rudely halted then the .ninja_log file may be
115 + # corrupt. Silently continue.
116 + continue
117 + start, end, _, name, cmdhash = parts # Ignore restat.
118 + # Convert from integral milliseconds to float seconds.
119 + start = int(start) / 1000.0
120 + end = int(end) / 1000.0
121 + if not show_all and end < last_end_seen:
122 + # An earlier time stamp means that this step is the first in a new
123 + # build, possibly an incremental build. Throw away the previous
124 + # data so that this new build will be displayed independently.
125 + # This has to be done by comparing end times because records are
126 + # written to the .ninja_log file when commands complete, so end
127 + # times are guaranteed to be in order, but start times are not.
128 + targets_dict = {}
129 + target = None
130 + if cmdhash in targets_dict:
131 + target = targets_dict[cmdhash]
132 + if not show_all and (target.start != start or target.end != end):
133 + # If several builds in a row just run one or two build steps
134 + # then the end times may not go backwards so the last build may
135 + # not be detected as such. However in many cases there will be a
136 + # build step repeated in the two builds and the changed
137 + # start/stop points for that command, identified by the hash,
138 + # can be used to detect and reset the target dictionary.
139 + targets_dict = {}
140 + target = None
141 + if not target:
142 + targets_dict[cmdhash] = target = Target(start, end)
143 + last_end_seen = end
144 + target.targets.append(name)
145 + return list(targets_dict.values())
146 +
147 +
148 +def GetExtension(target, extra_patterns):
149 + """Return the file extension that best represents a target.
150 +
151 + For targets that generate multiple outputs it is important to return a
152 + consistent 'canonical' extension. Ultimately the goal is to group build steps
153 + by type."""
154 + for output in target.targets:
155 + if extra_patterns:
156 + for fn_pattern in extra_patterns.split(';'):
157 + if fnmatch.fnmatch(output, '*' + fn_pattern + '*'):
158 + return fn_pattern
159 + # Not a true extension, but a good grouping.
160 + if output.endswith('type_mappings'):
161 + extension = 'type_mappings'
162 + break
163 +
164 + # Capture two extensions if present. For example: file.javac.jar should
165 + # be distinguished from file.interface.jar.
166 + root, ext1 = os.path.splitext(output)
167 + _, ext2 = os.path.splitext(root)
168 + extension = ext2 + ext1 # Preserve the order in the file name.
169 +
170 + if len(extension) == 0:
171 + extension = '(no extension found)'
172 +
173 + if ext1 in ['.pdb', '.dll', '.exe']:
174 + extension = 'PEFile (linking)'
175 + # Make sure that .dll and .exe are grouped together and that the
176 + # .dll.lib files don't cause these to be listed as libraries
177 + break
178 + if ext1 in ['.so', '.TOC']:
179 + extension = '.so (linking)'
180 + # Attempt to identify linking, avoid identifying as '.TOC'
181 + break
182 + # Make sure .obj files don't get categorized as mojo files
183 + if ext1 in ['.obj', '.o']:
184 + break
185 + # Jars are the canonical output of java targets.
186 + if ext1 == '.jar':
187 + break
188 + # Normalize all mojo related outputs to 'mojo'.
189 + if output.count('.mojom') > 0:
190 + extension = 'mojo'
191 + break
192 + return extension
193 +
194 +
195 +def SummarizeEntries(entries, extra_step_types):
196 + """Print a summary of the passed in list of Target objects."""
197 +
198 + # Create a list that is in order by time stamp and has entries for the
199 + # beginning and ending of each build step (one time stamp may have multiple
200 + # entries due to multiple steps starting/stopping at exactly the same time).
201 + # Iterate through this list, keeping track of which tasks are running at all
202 + # times. At each time step calculate a running total for weighted time so
203 + # that when each task ends its own weighted time can easily be calculated.
204 + task_start_stop_times = []
205 +
206 + earliest = -1
207 + latest = 0
208 + total_cpu_time = 0
209 + for target in entries:
210 + if earliest < 0 or target.start < earliest:
211 + earliest = target.start
212 + if target.end > latest:
213 + latest = target.end
214 + total_cpu_time += target.Duration()
215 + task_start_stop_times.append((target.start, 'start', target))
216 + task_start_stop_times.append((target.end, 'stop', target))
217 + length = latest - earliest
218 + weighted_total = 0.0
219 +
220 + # Sort by the time/type records and ignore |target|
221 + task_start_stop_times.sort(key=lambda times: times[:2])
222 + # Now we have all task start/stop times sorted by when they happen. If a
223 + # task starts and stops on the same time stamp then the start will come
224 + # first because of the alphabet, which is important for making this work
225 + # correctly.
226 + # Track the tasks which are currently running.
227 + running_tasks = {}
228 + # Record the time we have processed up to so we know how to calculate time
229 + # deltas.
230 + last_time = task_start_stop_times[0][0]
231 + # Track the accumulated weighted time so that it can efficiently be added
232 + # to individual tasks.
233 + last_weighted_time = 0.0
234 + # Scan all start/stop events.
235 + for event in task_start_stop_times:
236 + time, action_name, target = event
237 + # Accumulate weighted time up to now.
238 + num_running = len(running_tasks)
239 + if num_running > 0:
240 + # Update the total weighted time up to this moment.
241 + last_weighted_time += (time - last_time) / float(num_running)
242 + if action_name == 'start':
243 + # Record the total weighted task time when this task starts.
244 + running_tasks[target] = last_weighted_time
245 + if action_name == 'stop':
246 + # Record the change in the total weighted task time while this task
247 + # ran.
248 + weighted_duration = last_weighted_time - running_tasks[target]
249 + target.SetWeightedDuration(weighted_duration)
250 + weighted_total += weighted_duration
251 + del running_tasks[target]
252 + last_time = time
253 + assert (len(running_tasks) == 0)
254 +
255 + # Warn if the sum of weighted times is off by more than half a second.
256 + if abs(length - weighted_total) > 500:
257 + print('Warning: Possible corrupt ninja log, results may be '
258 + 'untrustworthy. Length = %.3f, weighted total = %.3f' %
259 + (length, weighted_total))
260 +
261 + entries_by_ext = defaultdict(list)
262 + for target in entries:
263 + extension = GetExtension(target, extra_step_types)
264 + entries_by_ext[extension].append(target)
265 +
266 + for key, values in entries_by_ext.items():
267 + print(' Longest build steps for %s:' % key)
268 + values.sort(key=lambda x: x.WeightedDuration())
269 + for target in values[-long_count:]:
270 + print(' %8.1f weighted s to build %s (%.1f s elapsed time)' %
271 + (target.WeightedDuration(), target.DescribeTargets(),
272 + target.Duration()))
273 +
274 + print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx '
275 + 'parallelism)' %
276 + (length, total_cpu_time, total_cpu_time * 1.0 / length))
277 + print(' %d build steps completed, average of %1.2f/s' %
278 + (len(entries), len(entries) / (length)))
279 +
280 +
281 +def main():
282 + log_file = '.ninja_log'
283 + parser = argparse.ArgumentParser()
284 + parser.add_argument('-C', dest='build_directory', help='Build directory.')
285 + parser.add_argument(
286 + '-s',
287 + '--step-types',
288 + help='semicolon separated fnmatch patterns for build-step grouping')
289 + parser.add_argument('--log-file',
290 + help="specific ninja log file to analyze.")
291 + args, _extra_args = parser.parse_known_args()
292 + if args.build_directory:
293 + log_file = os.path.join(args.build_directory, log_file)
294 + if args.log_file:
295 + log_file = args.log_file
296 + if args.step_types:
297 + # Make room for the extra build types.
298 + global long_ext_count
299 + long_ext_count += len(args.step_types.split(';'))
300 +
301 + try:
302 + with open(log_file, 'r') as log:
303 + entries = ReadTargets(log, False)
304 + SummarizeEntries(entries, args.step_types)
305 + except IOError:
306 + print('Log file %r not found, no build summary created.' % log_file)
307 + return errno.ENOENT
308 +
309 +
310 +if __name__ == '__main__':
311 + sys.exit(main())
vllm/_custom_ops.py
+9
@@ -32,6 +32,15 @@ def hint_on_error(fn):
32 def wrapper(*args, **kwargs):
33 try:
34 return fn(*args, **kwargs)
35 +
36 + except NotImplementedError as e:
37 + msg = (
38 + "Error in calling custom op %s: %s\n"
39 + "Not implemented or built, mostly likely because the current current device "
40 + "does not support this kernel (less likely TORCH_CUDA_ARCH_LIST was set "
41 + "incorrectly while building)")
42 + logger.error(msg, fn.__name__, e)
43 + raise NotImplementedError(msg % (fn.__name__, e)) from e
44 except AttributeError as e:
45 msg = (
46 "Error in calling custom op %s: %s\n"