| 1 | include(FetchContent) |
| 2 | include(CheckCXXCompilerFlag) |
| 3 | |
| 4 | set(CMAKE_CXX_STANDARD_REQUIRED ON) |
| 5 | set(CMAKE_CXX_STANDARD 20) |
| 6 | set(CMAKE_CXX_EXTENSIONS ON) |
| 7 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) |
| 8 | |
| 9 | if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") |
| 10 | set(MACOSX_FOUND TRUE) |
| 11 | endif() |
| 12 | |
| 13 | |
| 14 | # |
| 15 | # Define environment variables for special configurations |
| 16 | # |
| 17 | set(ENABLE_X86_ISA $ENV{VLLM_CPU_X86}) |
| 18 | set(ENABLE_ARM_BF16 $ENV{VLLM_CPU_ARM_BF16}) |
| 19 | set(ENABLE_ARM_I8MM $ENV{VLLM_CPU_ARM_I8MM}) |
| 20 | set(ENABLE_RVV_BF16 $ENV{VLLM_CPU_RVV_BF16}) |
| 21 | |
| 22 | include_directories("${CMAKE_SOURCE_DIR}/csrc") |
| 23 | |
| 24 | set (ENABLE_NUMA TRUE) |
| 25 | |
| 26 | # |
| 27 | # Check the compile flags |
| 28 | # |
| 29 | if(MACOSX_FOUND) |
| 30 | # Apple clang needs -Xpreprocessor to enable OpenMP. No runtime link is |
| 31 | # needed: _C is a dynamic_lookup bundle and resolves libomp from torch. |
| 32 | list(APPEND CXX_COMPILE_FLAGS |
| 33 | "-Xpreprocessor" "-fopenmp" |
| 34 | "-DVLLM_CPU_EXTENSION") |
| 35 | else() |
| 36 | list(APPEND CXX_COMPILE_FLAGS |
| 37 | "-fopenmp" |
| 38 | "-DVLLM_CPU_EXTENSION") |
| 39 | |
| 40 | # locate PyTorch's libgomp (e.g. site-packages/torch.libs/libgomp-947d5fa1.so.1.0.0) |
| 41 | # and create a local shim dir with it. When PyTorch is built from source or packaged |
| 42 | # by a distro (common on RISC-V, s390x, Fedora/RHEL aarch64), no vendored libgomp |
| 43 | # exists and the shim dir is empty; fall back to the system libgomp in that case. |
| 44 | vllm_prepare_torch_gomp_shim(VLLM_TORCH_GOMP_SHIM_DIR) |
| 45 | |
| 46 | if(VLLM_TORCH_GOMP_SHIM_DIR) |
| 47 | find_library(OPEN_MP |
| 48 | NAMES gomp |
| 49 | PATHS "${VLLM_TORCH_GOMP_SHIM_DIR}" |
| 50 | NO_DEFAULT_PATH |
| 51 | REQUIRED |
| 52 | ) |
| 53 | # Use the same libgomp as PyTorch at runtime |
| 54 | set(ENV{LD_LIBRARY_PATH} "${VLLM_TORCH_GOMP_SHIM_DIR}:$ENV{LD_LIBRARY_PATH}") |
| 55 | else() |
| 56 | # Fall back to system / toolchain libgomp |
| 57 | find_library(OPEN_MP NAMES gomp REQUIRED) |
| 58 | endif() |
| 59 | endif() |
| 60 | |
| 61 | if (NOT MACOSX_FOUND) |
| 62 | execute_process(COMMAND cat /proc/cpuinfo |
| 63 | RESULT_VARIABLE CPUINFO_RET |
| 64 | OUTPUT_VARIABLE CPUINFO) |
| 65 | if (NOT CPUINFO_RET EQUAL 0) |
| 66 | message(FATAL_ERROR "Failed to check CPU features via /proc/cpuinfo") |
| 67 | endif() |
| 68 | endif() |
| 69 | |
| 70 | |
| 71 | function (find_isa CPUINFO TARGET OUT) |
| 72 | string(FIND ${CPUINFO} ${TARGET} ISA_FOUND) |
| 73 | if(NOT ISA_FOUND EQUAL -1) |
| 74 | set(${OUT} ON PARENT_SCOPE) |
| 75 | else() |
| 76 | set(${OUT} OFF PARENT_SCOPE) |
| 77 | endif() |
| 78 | endfunction() |
| 79 | |
| 80 | |
| 81 | function(check_sysctl TARGET OUT) |
| 82 | execute_process(COMMAND sysctl -n "${TARGET}" |
| 83 | RESULT_VARIABLE SYSCTL_RET |
| 84 | OUTPUT_VARIABLE SYSCTL_INFO |
| 85 | ERROR_QUIET |
| 86 | OUTPUT_STRIP_TRAILING_WHITESPACE) |
| 87 | if(SYSCTL_RET EQUAL 0 AND |
| 88 | (SYSCTL_INFO STREQUAL "1" OR SYSCTL_INFO GREATER 0)) |
| 89 | set(${OUT} ON PARENT_SCOPE) |
| 90 | else() |
| 91 | set(${OUT} OFF PARENT_SCOPE) |
| 92 | endif() |
| 93 | endfunction() |
| 94 | |
| 95 | if (MACOSX_FOUND AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") |
| 96 | message(STATUS "Apple Silicon Detected") |
| 97 | set(APPLE_SILICON_FOUND TRUE) |
| 98 | set(ENABLE_NUMA OFF) |
| 99 | check_sysctl(hw.optional.neon ASIMD_FOUND) |
| 100 | check_sysctl(hw.optional.arm.FEAT_BF16 ARM_BF16_FOUND) |
| 101 | check_sysctl(hw.optional.arm.FEAT_I8MM ARM_I8MM_FOUND) |
| 102 | else() |
| 103 | find_isa(${CPUINFO} "Power11" POWER11_FOUND) |
| 104 | find_isa(${CPUINFO} "POWER10" POWER10_FOUND) |
| 105 | find_isa(${CPUINFO} "POWER9" POWER9_FOUND) |
| 106 | find_isa(${CPUINFO} "asimd" ASIMD_FOUND) # Check for ARM NEON support |
| 107 | find_isa(${CPUINFO} "bf16" ARM_BF16_FOUND) # Check for ARM BF16 support |
| 108 | find_isa(${CPUINFO} "i8mm" ARM_I8MM_FOUND) # Check for ARM I8MM support |
| 109 | find_isa(${CPUINFO} "S390" S390_FOUND) |
| 110 | find_isa(${CPUINFO} "zvfhmin" RVV_FP16_FOUND) # Check for RISC-V Vector FP16 support |
| 111 | find_isa(${CPUINFO} "zvfbfmin" RVV_BF16_FOUND) # Check for RISC-V Vector BF16 support |
| 112 | |
| 113 | # Support cross-compilation by allowing override via environment variables |
| 114 | if (ENABLE_ARM_BF16) |
| 115 | set(ARM_BF16_FOUND ON) |
| 116 | message(STATUS "ARM BF16 support enabled via VLLM_CPU_ARM_BF16 environment variable") |
| 117 | endif() |
| 118 | if (ENABLE_ARM_I8MM) |
| 119 | set(ARM_I8MM_FOUND ON) |
| 120 | message(STATUS |
| 121 | "ARM I8MM support enabled via VLLM_CPU_ARM_I8MM environment variable") |
| 122 | endif() |
| 123 | # Some kernels (e.g. Bianbu on Spacemit X100) do not report zvfbfmin |
| 124 | # in /proc/cpuinfo despite hardware support. VLLM_CPU_RVV_BF16=1 |
| 125 | # overrides the detection result. |
| 126 | if (ENABLE_RVV_BF16) |
| 127 | set(RVV_BF16_FOUND ON) |
| 128 | message(STATUS "RVV BF16 support enabled via VLLM_CPU_RVV_BF16 environment variable") |
| 129 | endif() |
| 130 | endif() |
| 131 | |
| 132 | if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64" OR ENABLE_X86_ISA) |
| 133 | set(ENABLE_X86_ISA ON) |
| 134 | if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND |
| 135 | CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3)) |
| 136 | message(FATAL_ERROR "X86 backend requires gcc/g++ >= 12.3") |
| 137 | endif() |
| 138 | # Work around a GCC 15 optimizer crash in oneDNN (tree-ssa-pre pass). |
| 139 | # Keep this scoped to GCC 15+ so older toolchains are unaffected. |
| 140 | if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND |
| 141 | CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15) |
| 142 | list(APPEND CXX_COMPILE_FLAGS "-fno-tree-pre") |
| 143 | endif() |
| 144 | list(APPEND CXX_COMPILE_FLAGS "-mf16c") |
| 145 | list(APPEND CXX_COMPILE_FLAGS_AVX512 ${CXX_COMPILE_FLAGS}) |
| 146 | list(APPEND CXX_COMPILE_FLAGS_AVX2 ${CXX_COMPILE_FLAGS}) |
| 147 | list(APPEND CXX_COMPILE_FLAGS_AVX512 |
| 148 | "-mavx512f" |
| 149 | "-mavx512vl" |
| 150 | "-mavx512bw" |
| 151 | "-mavx512dq") |
| 152 | list(APPEND CXX_COMPILE_FLAGS_AVX512_AMX |
| 153 | ${CXX_COMPILE_FLAGS_AVX512} |
| 154 | "-mamx-bf16" |
| 155 | "-mamx-tile" |
| 156 | "-mavx512bf16" |
| 157 | "-mavx512vnni") |
| 158 | # Some packaged GCC 14 builds still do not accept -mamx-fp8, so gate on |
| 159 | # actual flag support rather than compiler version alone. |
| 160 | # Option to enable/disable AMX-FP8 support (default: ON) |
| 161 | option(ENABLE_AMX_FP8 "Enable AMX-FP8 support" ON) |
| 162 | |
| 163 | if (ENABLE_AMX_FP8) |
| 164 | check_cxx_compiler_flag("-mamx-fp8" COMPILER_SUPPORTS_AMX_FP8_FLAG) |
| 165 | if (COMPILER_SUPPORTS_AMX_FP8_FLAG) |
| 166 | list(APPEND CXX_COMPILE_FLAGS_AVX512_AMX "-mamx-fp8") |
| 167 | set(AMX_FP8_SUPPORTED TRUE) |
| 168 | else() |
| 169 | set(AMX_FP8_SUPPORTED FALSE) |
| 170 | message(STATUS "AMX-FP8 disabled: compiler does not support -mamx-fp8 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})") |
| 171 | endif() |
| 172 | else() |
| 173 | set(AMX_FP8_SUPPORTED FALSE) |
| 174 | message(STATUS "AMX-FP8 disabled by ENABLE_AMX_FP8=OFF") |
| 175 | endif() |
| 176 | |
| 177 | # AVX10.2 arrived in GCC 15, so the target attribute, the fp8 convert |
| 178 | # intrinsics and __builtin_cpu_supports("avx10.2") are all hard errors on |
| 179 | # the 12.3+ the backend otherwise supports. Probe instead of assuming. |
| 180 | check_cxx_compiler_flag("-mavx10.2" COMPILER_SUPPORTS_AVX10_2_FLAG) |
| 181 | if (COMPILER_SUPPORTS_AVX10_2_FLAG) |
| 182 | set(AVX10_2_SUPPORTED TRUE) |
| 183 | else() |
| 184 | set(AVX10_2_SUPPORTED FALSE) |
| 185 | message(STATUS "AVX10.2 disabled: compiler does not support -mavx10.2 (compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION})") |
| 186 | endif() |
| 187 | |
| 188 | list(APPEND CXX_COMPILE_FLAGS_AVX2 |
| 189 | "-mavx2") |
| 190 | elseif (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) |
| 191 | message(STATUS "PowerPC detected") |
| 192 | if (POWER9_FOUND) |
| 193 | list(APPEND CXX_COMPILE_FLAGS |
| 194 | "-mvsx" |
| 195 | "-mcpu=power9" |
| 196 | "-mtune=power9") |
| 197 | elseif (POWER10_FOUND OR POWER11_FOUND) |
| 198 | list(APPEND CXX_COMPILE_FLAGS |
| 199 | "-mvsx" |
| 200 | "-mcpu=power10" |
| 201 | "-mtune=power10") |
| 202 | endif() |
| 203 | |
| 204 | elseif (ASIMD_FOUND) |
| 205 | message(STATUS "ARMv8 or later architecture detected") |
| 206 | if(ARM_BF16_FOUND) |
| 207 | message(STATUS "BF16 extension detected") |
| 208 | set(MARCH_FLAGS "-march=armv8.2-a+bf16+dotprod+fp16") |
| 209 | add_compile_definitions(ARM_BF16_SUPPORT) |
| 210 | else() |
| 211 | message(WARNING "BF16 functionality is not available") |
| 212 | set(MARCH_FLAGS "-march=armv8.2-a+dotprod+fp16") |
| 213 | endif() |
| 214 | if(ARM_I8MM_FOUND) |
| 215 | message(STATUS "I8MM extension detected") |
| 216 | string(APPEND MARCH_FLAGS "+i8mm") |
| 217 | add_compile_definitions(ARM_I8MM_SUPPORT) |
| 218 | endif() |
| 219 | list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS}) |
| 220 | elseif (S390_FOUND) |
| 221 | message(STATUS "S390 detected") |
| 222 | # Check for S390 VXE support |
| 223 | list(APPEND CXX_COMPILE_FLAGS |
| 224 | "-mvx" |
| 225 | "-mzvector" |
| 226 | "-march=native" |
| 227 | "-mtune=native") |
| 228 | elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") |
| 229 | message(STATUS "RISC-V detected") |
| 230 | if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0) |
| 231 | message(FATAL_ERROR |
| 232 | "VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'") |
| 233 | endif() |
| 234 | # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo |
| 235 | # by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build. |
| 236 | # Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV. |
| 237 | if(NOT DEFINED VLLM_RVV_VLEN) |
| 238 | # Auto-detect: find the largest zvl<N>b in /proc/cpuinfo isa line. |
| 239 | # Skip when cross-compiling — /proc/cpuinfo describes the build host. |
| 240 | if(CMAKE_CROSSCOMPILING) |
| 241 | message(STATUS "Cross-compiling: skipping VLEN auto-detection from /proc/cpuinfo") |
| 242 | elseif(EXISTS /proc/cpuinfo) |
| 243 | file(READ /proc/cpuinfo _cpuinfo) |
| 244 | set(_best 0) |
| 245 | foreach(_n IN ITEMS 128 256 512 1024) |
| 246 | if(_cpuinfo MATCHES "zvl${_n}b") |
| 247 | set(_best ${_n}) |
| 248 | endif() |
| 249 | endforeach() |
| 250 | # Only VLEN=128 and VLEN=256 are supported by the RVV kernels. |
| 251 | if(_best GREATER 256) |
| 252 | message(WARNING |
| 253 | "Detected VLEN=${_best} but only 128/256 are supported; " |
| 254 | "clamping to 256") |
| 255 | set(_best 256) |
| 256 | endif() |
| 257 | if(_best GREATER 0) |
| 258 | set(VLLM_RVV_VLEN ${_best}) |
| 259 | endif() |
| 260 | endif() |
| 261 | # If auto-detect failed (no /proc/cpuinfo or no zvl<N>b reported) |
| 262 | # but the compiler supports RVV, require explicit specification. |
| 263 | if(NOT DEFINED VLLM_RVV_VLEN AND (RVV_FP16_FOUND OR RVV_BF16_FOUND)) |
| 264 | message(FATAL_ERROR |
| 265 | "RISC-V RVV is available but VLEN could not be auto-detected. " |
| 266 | "Please specify VLEN explicitly via CMAKE_ARGS:\n" |
| 267 | " CMAKE_ARGS='-DVLLM_RVV_VLEN=128' (for VLEN=128 hardware)\n" |
| 268 | " CMAKE_ARGS='-DVLLM_RVV_VLEN=256' (for VLEN=256 hardware, e.g. Spacemit X100)") |
| 269 | endif() |
| 270 | endif() |
| 271 | if(VLLM_RVV_VLEN AND VLLM_RVV_VLEN GREATER 0) |
| 272 | message(STATUS "RISC-V RVV VLEN=${VLLM_RVV_VLEN}") |
| 273 | # Sources gate FP16/BF16 paths on the compiler-provided |
| 274 | # __riscv_zvfh / __riscv_zvfbfmin macros, which GCC and clang |
| 275 | # define automatically when those extensions appear in -march. |
| 276 | if(RVV_BF16_FOUND) |
| 277 | message(STATUS "BF16 extension detected") |
| 278 | set(MARCH_FLAGS -march=rv64gcv_zvfh_zfbfmin_zvfbfmin_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d) |
| 279 | elseif(RVV_FP16_FOUND) |
| 280 | message(WARNING "BF16 functionality is not available.") |
| 281 | set(MARCH_FLAGS -march=rv64gcv_zvfh_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d) |
| 282 | else() |
| 283 | message(STATUS "compile riscv with scalar (no FP16/BF16)") |
| 284 | set(MARCH_FLAGS -march=rv64gc) |
| 285 | endif() |
| 286 | else() |
| 287 | message(STATUS "compile riscv with scalar") |
| 288 | set(MARCH_FLAGS -march=rv64gc) |
| 289 | endif() |
| 290 | list(APPEND CXX_COMPILE_FLAGS ${MARCH_FLAGS}) |
| 291 | else() |
| 292 | message(FATAL_ERROR "vLLM CPU backend requires X86, Power9+ ISA, S390X ISA, ARMv8 or RISC-V support.") |
| 293 | endif() |
| 294 | |
| 295 | |
| 296 | # Build oneDNN for GEMM kernels |
| 297 | if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND OR RVV_FP16_FOUND OR RVV_BF16_FOUND OR S390_FOUND) |
| 298 | # Fetch and build Arm Compute Library (ACL) as oneDNN's backend for AArch64 |
| 299 | # TODO [fadara01]: remove this once ACL can be fetched and built automatically as a dependency of oneDNN |
| 300 | set(ONEDNN_AARCH64_USE_ACL OFF CACHE BOOL "") |
| 301 | if(ASIMD_FOUND) |
| 302 | # Set number of parallel build processes |
| 303 | include(ProcessorCount) |
| 304 | ProcessorCount(NPROC) |
| 305 | if(NOT NPROC) |
| 306 | set(NPROC 4) |
| 307 | endif() |
| 308 | |
| 309 | # Fetch and populate ACL |
| 310 | if(DEFINED ENV{ACL_ROOT_DIR} AND IS_DIRECTORY "$ENV{ACL_ROOT_DIR}") |
| 311 | message(STATUS "Using ACL from specified source directory: $ENV{ACL_ROOT_DIR}") |
| 312 | else() |
| 313 | message(STATUS "Downloading Arm Compute Library (ACL) from GitHub") |
| 314 | FetchContent_Populate(arm_compute |
| 315 | SUBBUILD_DIR "${FETCHCONTENT_BASE_DIR}/arm_compute-subbuild" |
| 316 | SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/arm_compute-src" |
| 317 | GIT_REPOSITORY https://github.com/ARM-software/ComputeLibrary.git |
| 318 | GIT_TAG v52.6.0 |
| 319 | GIT_SHALLOW TRUE |
| 320 | GIT_PROGRESS TRUE |
| 321 | ) |
| 322 | set(ENV{ACL_ROOT_DIR} "${arm_compute_SOURCE_DIR}") |
| 323 | set(ACL_LIB_DIR "$ENV{ACL_ROOT_DIR}/build") |
| 324 | endif() |
| 325 | |
| 326 | # Build ACL with CMake |
| 327 | set(_cmake_config_cmd |
| 328 | ${CMAKE_COMMAND} -G Ninja -B build |
| 329 | -DARM_COMPUTE_BUILD_SHARED_LIB=OFF |
| 330 | -DCMAKE_BUILD_TYPE=Release |
| 331 | -DARM_COMPUTE_ARCH=armv8.2-a |
| 332 | -DARM_COMPUTE_ENABLE_ASSERTS=OFF |
| 333 | -DARM_COMPUTE_ENABLE_CPPTHREADS=OFF |
| 334 | -DARM_COMPUTE_ENABLE_OPENMP=ON |
| 335 | -DARM_COMPUTE_ENABLE_WERROR=OFF |
| 336 | -DARM_COMPUTE_BUILD_EXAMPLES=OFF |
| 337 | -DARM_COMPUTE_BUILD_TESTING=OFF) |
| 338 | set(_cmake_build_cmd |
| 339 | ${CMAKE_COMMAND} --build build -- -j${NPROC} |
| 340 | ) |
| 341 | |
| 342 | execute_process( |
| 343 | COMMAND ${_cmake_config_cmd} |
| 344 | WORKING_DIRECTORY "$ENV{ACL_ROOT_DIR}" |
| 345 | ) |
| 346 | execute_process( |
| 347 | COMMAND ${_cmake_build_cmd} |
| 348 | WORKING_DIRECTORY "$ENV{ACL_ROOT_DIR}" |
| 349 | RESULT_VARIABLE _acl_rc |
| 350 | ) |
| 351 | |
| 352 | if(NOT _acl_rc EQUAL 0) |
| 353 | message(FATAL_ERROR "ACL SCons build failed (exit ${_acl_rc}).") |
| 354 | endif() |
| 355 | message(STATUS "Arm Compute Library (ACL) built successfully.") |
| 356 | |
| 357 | # VLLM/oneDNN settings for ACL |
| 358 | set(ONEDNN_AARCH64_USE_ACL ON CACHE BOOL "" FORCE) |
| 359 | add_compile_definitions(VLLM_USE_ACL) |
| 360 | endif() |
| 361 | |
| 362 | set(FETCHCONTENT_SOURCE_DIR_ONEDNN "$ENV{FETCHCONTENT_SOURCE_DIR_ONEDNN}" CACHE PATH "Path to a local oneDNN source directory.") |
| 363 | |
| 364 | if(FETCHCONTENT_SOURCE_DIR_ONEDNN) |
| 365 | message(STATUS "Using oneDNN from specified source directory: ${FETCHCONTENT_SOURCE_DIR_ONEDNN}") |
| 366 | FetchContent_Declare( |
| 367 | oneDNN |
| 368 | SOURCE_DIR ${FETCHCONTENT_SOURCE_DIR_ONEDNN} |
| 369 | ) |
| 370 | else() |
| 371 | message(STATUS "Downloading oneDNN from GitHub") |
| 372 | if(ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) |
| 373 | message(STATUS "aarch64 detected: using pinned oneDNN commit 9c5be1cc59e368aebf0909e6cf20f981ea61462a") |
| 374 | FetchContent_Declare( |
| 375 | oneDNN |
| 376 | GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git |
| 377 | GIT_TAG 9c5be1cc59e368aebf0909e6cf20f981ea61462a |
| 378 | GIT_PROGRESS TRUE |
| 379 | GIT_SHALLOW FALSE |
| 380 | ) |
| 381 | else() |
| 382 | FetchContent_Declare( |
| 383 | oneDNN |
| 384 | GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git |
| 385 | GIT_TAG v3.13 |
| 386 | GIT_PROGRESS TRUE |
| 387 | GIT_SHALLOW TRUE |
| 388 | ) |
| 389 | endif() |
| 390 | endif() |
| 391 | |
| 392 | set(ONEDNN_LIBRARY_TYPE "STATIC") |
| 393 | set(ONEDNN_BUILD_DOC "OFF") |
| 394 | set(ONEDNN_BUILD_EXAMPLES "OFF") |
| 395 | set(ONEDNN_BUILD_TESTS "OFF") |
| 396 | set(ONEDNN_ENABLE_WORKLOAD "INFERENCE") |
| 397 | set(ONEDNN_ENABLE_PRIMITIVE "MATMUL;REORDER") |
| 398 | set(ONEDNN_BUILD_GRAPH "OFF") |
| 399 | set(ONEDNN_ENABLE_JIT_PROFILING "ON") |
| 400 | set(ONEDNN_ENABLE_ITT_TASKS "ON") |
| 401 | set(ONEDNN_ENABLE_MAX_CPU_ISA "ON") |
| 402 | set(ONEDNN_ENABLE_CPU_ISA_HINTS "ON") |
| 403 | set(ONEDNN_VERBOSE "ON") |
| 404 | set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) |
| 405 | |
| 406 | set(VLLM_BUILD_TYPE ${CMAKE_BUILD_TYPE}) |
| 407 | set(CMAKE_BUILD_TYPE "Release") # remove oneDNN debug symbols to reduce size |
| 408 | |
| 409 | if(S390_FOUND) |
| 410 | FetchContent_GetProperties(oneDNN) |
| 411 | if(NOT onednn_POPULATED) |
| 412 | FetchContent_Populate(oneDNN) |
| 413 | # Patch s390x helpers.h: ALWAYS_INLINE on operator+= breaks C++20/GCC14 |
| 414 | file(READ "${onednn_SOURCE_DIR}/src/cpu/s390x/helpers.h" _helpers_content) |
| 415 | string(REPLACE |
| 416 | "vec_type_t<T> &ALWAYS_INLINE operator+=" |
| 417 | "ALWAYS_INLINE vec_type_t<T> &operator+=" |
| 418 | _helpers_content "${_helpers_content}") |
| 419 | file(WRITE "${onednn_SOURCE_DIR}/src/cpu/s390x/helpers.h" "${_helpers_content}") |
| 420 | add_subdirectory("${onednn_SOURCE_DIR}" "${onednn_BINARY_DIR}") |
| 421 | endif() |
| 422 | else() |
| 423 | FetchContent_MakeAvailable(oneDNN) |
| 424 | endif() |
| 425 | set(CMAKE_BUILD_TYPE ${VLLM_BUILD_TYPE}) |
| 426 | add_library(dnnl_ext OBJECT "csrc/cpu/dnnl_helper.cpp") |
| 427 | target_include_directories( |
| 428 | dnnl_ext |
| 429 | PUBLIC ${oneDNN_SOURCE_DIR}/include |
| 430 | PUBLIC ${oneDNN_BINARY_DIR}/include |
| 431 | PRIVATE ${oneDNN_SOURCE_DIR}/src |
| 432 | ) |
| 433 | target_link_libraries(dnnl_ext dnnl torch) |
| 434 | if (ENABLE_X86_ISA) |
| 435 | target_compile_options(dnnl_ext PRIVATE ${CXX_COMPILE_FLAGS_AVX2} -fPIC) |
| 436 | else() |
| 437 | target_compile_options(dnnl_ext PRIVATE ${CXX_COMPILE_FLAGS} -fPIC) |
| 438 | endif() |
| 439 | list(APPEND LIBS dnnl_ext) |
| 440 | |
| 441 | |
| 442 | set(USE_ONEDNN ON) |
| 443 | else() |
| 444 | set(USE_ONEDNN OFF) |
| 445 | endif() |
| 446 | |
| 447 | # TODO: Refactor this |
| 448 | if (ENABLE_X86_ISA) |
| 449 | message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) compile flags: ${CXX_COMPILE_FLAGS_AVX512_AMX}") |
| 450 | message(STATUS "CPU extension (AVX512F) compile flags: ${CXX_COMPILE_FLAGS_AVX512}") |
| 451 | message(STATUS "CPU extension (AVX2) compile flags: ${CXX_COMPILE_FLAGS_AVX2}") |
| 452 | else() |
| 453 | message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}") |
| 454 | endif() |
| 455 | |
| 456 | if(ENABLE_NUMA) |
| 457 | list(APPEND LIBS numa) |
| 458 | else() |
| 459 | message(STATUS "NUMA is disabled") |
| 460 | add_compile_definitions(-DVLLM_NUMA_DISABLED) |
| 461 | endif() |
| 462 | |
| 463 | # check if the pytorch wheel ships libopenblas.so. |
| 464 | set(VLLM_OPENBLAS_LIB "") |
| 465 | if (NOT ENABLE_X86_ISA) |
| 466 | file(GLOB _VLLM_TORCH_OPENBLAS_LIBS |
| 467 | "${TORCH_INSTALL_PREFIX}/lib/libopenblas*.so*") |
| 468 | # Note: we don't link openblas directly to _C extension, as it's available through libtorch.so |
| 469 | if (_VLLM_TORCH_OPENBLAS_LIBS) |
| 470 | list(GET _VLLM_TORCH_OPENBLAS_LIBS 0 VLLM_OPENBLAS_LIB) |
| 471 | message(STATUS "CPU OpenBLAS library: ${VLLM_OPENBLAS_LIB}") |
| 472 | endif() |
| 473 | endif() |
| 474 | |
| 475 | # |
| 476 | # Generate CPU attention dispatch header |
| 477 | # |
| 478 | message(STATUS "Generating CPU attention dispatch header") |
| 479 | execute_process( |
| 480 | COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/csrc/cpu/generate_cpu_attn_dispatch.py |
| 481 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csrc/cpu |
| 482 | RESULT_VARIABLE GEN_RESULT |
| 483 | ) |
| 484 | if(NOT GEN_RESULT EQUAL 0) |
| 485 | message(FATAL_ERROR "Failed to generate CPU attention dispatch header") |
| 486 | endif() |
| 487 | |
| 488 | # |
| 489 | # _C extension |
| 490 | # |
| 491 | set(VLLM_EXT_SRC |
| 492 | "csrc/cpu/activation.cpp" |
| 493 | "csrc/cpu/sgl-kernels/fla.cpp" |
| 494 | "csrc/cpu/utils.cpp" |
| 495 | "csrc/cpu/spec_decode_utils.cpp" |
| 496 | "csrc/cpu/layernorm.cpp" |
| 497 | "csrc/cpu/mla_decode.cpp" |
| 498 | "csrc/cpu/pos_encoding.cpp" |
| 499 | "csrc/cpu/mamba_cpu.cpp" |
| 500 | "csrc/moe/dynamic_4bit_int_moe_cpu.cpp" |
| 501 | "csrc/cpu/cpu_fused_moe.cpp" |
| 502 | "csrc/cpu/cpu_attn.cpp" |
| 503 | "csrc/cpu/cpu_isa.cpp" |
| 504 | "csrc/cpu/sampling_kernels.cpp" |
| 505 | "csrc/cpu/torch_bindings.cpp") |
| 506 | |
| 507 | if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND |
| 508 | VLLM_RVV_VLEN GREATER 0 AND (RVV_FP16_FOUND OR RVV_BF16_FOUND)) |
| 509 | set(VLLM_EXT_SRC |
| 510 | "csrc/cpu/cpu_wna16.cpp" |
| 511 | ${VLLM_EXT_SRC}) |
| 512 | endif() |
| 513 | |
| 514 | if (S390_FOUND) |
| 515 | set(VLLM_EXT_SRC |
| 516 | "csrc/cpu/cpu_wna16.cpp" |
| 517 | ${VLLM_EXT_SRC}) |
| 518 | endif() |
| 519 | |
| 520 | if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) |
| 521 | set(VLLM_EXT_SRC |
| 522 | "csrc/cpu/shm.cpp" |
| 523 | "csrc/cpu/activation_lut_bf16.cpp" |
| 524 | "csrc/cpu/cpu_tanhf_neon.hpp" |
| 525 | ${VLLM_EXT_SRC}) |
| 526 | if (ARM_BF16_FOUND) |
| 527 | set(VLLM_EXT_SRC "csrc/cpu/sgl-kernels/conv.cpp" ${VLLM_EXT_SRC}) |
| 528 | if (ARM_I8MM_FOUND) |
| 529 | set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe_int8.cpp" ${VLLM_EXT_SRC}) |
| 530 | endif() |
| 531 | endif() |
| 532 | endif() |
| 533 | |
| 534 | if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) |
| 535 | set(VLLM_EXT_SRC |
| 536 | "csrc/cpu/cpu_wna16.cpp" |
| 537 | "csrc/cpu/shm.cpp" |
| 538 | ${VLLM_EXT_SRC}) |
| 539 | endif() |
| 540 | |
| 541 | if (POWER10_FOUND OR POWER11_FOUND) |
| 542 | set(VLLM_EXT_SRC |
| 543 | "csrc/cpu/cpu_fused_moe_int8.cpp" |
| 544 | ${VLLM_EXT_SRC}) |
| 545 | endif() |
| 546 | |
| 547 | if(USE_ONEDNN) |
| 548 | set(VLLM_EXT_SRC |
| 549 | "csrc/cpu/dnnl_kernels.cpp" |
| 550 | ${VLLM_EXT_SRC}) |
| 551 | endif() |
| 552 | |
| 553 | if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64") |
| 554 | set(VLLM_EXT_SRC |
| 555 | "csrc/cpu/sgl-kernels/gemm_int4.cpp" |
| 556 | ${VLLM_EXT_SRC}) |
| 557 | endif() |
| 558 | |
| 559 | if (ENABLE_X86_ISA) |
| 560 | set(VLLM_EXT_SRC_SGL |
| 561 | "csrc/cpu/sgl-kernels/conv.cpp" |
| 562 | "csrc/cpu/sgl-kernels/gemm.cpp" |
| 563 | "csrc/cpu/sgl-kernels/gemm_int8.cpp" |
| 564 | "csrc/cpu/sgl-kernels/gemm_fp8.cpp" |
| 565 | "csrc/cpu/sgl-kernels/gemm_fp8_w8a8.cpp" |
| 566 | "csrc/cpu/sgl-kernels/gemm_int4.cpp" |
| 567 | "csrc/cpu/sgl-kernels/moe.cpp" |
| 568 | "csrc/cpu/sgl-kernels/moe_int8.cpp" |
| 569 | "csrc/cpu/sgl-kernels/moe_int4.cpp" |
| 570 | "csrc/cpu/sgl-kernels/moe_fp8.cpp" |
| 571 | "csrc/cpu/sgl-kernels/moe_fp8_w8a8.cpp" |
| 572 | "csrc/cpu/sgl-kernels/bmm.cpp" |
| 573 | "csrc/cpu/sgl-kernels/decode.cpp" |
| 574 | "csrc/cpu/sgl-kernels/extend.cpp" |
| 575 | "csrc/cpu/sgl-kernels/mla_cache.cpp" |
| 576 | "csrc/cpu/sgl-kernels/mhc.cpp" |
| 577 | "csrc/cpu/sgl-kernels/store_cache.cpp" |
| 578 | "csrc/cpu/sgl-kernels/flash_mla.cpp" |
| 579 | "csrc/cpu/sgl-kernels/compressor.cpp" |
| 580 | "csrc/cpu/sgl-kernels/paged_mqa_logits.cpp" |
| 581 | "csrc/cpu/sgl-kernels/topk.cpp" |
| 582 | "csrc/cpu/sgl-kernels/indexer.cpp") |
| 583 | |
| 584 | set(VLLM_EXT_SRC_AVX512 |
| 585 | "csrc/cpu/sgl-kernels/fla.cpp" |
| 586 | "csrc/cpu/shm.cpp" |
| 587 | "csrc/cpu/cpu_wna16.cpp" |
| 588 | "csrc/cpu/cpu_fused_moe.cpp" |
| 589 | "csrc/cpu/utils.cpp" |
| 590 | "csrc/cpu/spec_decode_utils.cpp" |
| 591 | "csrc/cpu/cpu_attn.cpp" |
| 592 | "csrc/cpu/cpu_isa.cpp" |
| 593 | "csrc/cpu/dnnl_kernels.cpp" |
| 594 | "csrc/cpu/mamba_cpu.cpp" |
| 595 | "csrc/cpu/sampling_kernels.cpp" |
| 596 | "csrc/cpu/torch_bindings.cpp" |
| 597 | # TODO: Remove these files |
| 598 | "csrc/cpu/activation.cpp" |
| 599 | "csrc/cpu/layernorm.cpp" |
| 600 | "csrc/cpu/mla_decode.cpp" |
| 601 | "csrc/cpu/pos_encoding.cpp" |
| 602 | "csrc/moe/dynamic_4bit_int_moe_cpu.cpp") |
| 603 | |
| 604 | set(VLLM_EXT_SRC_AVX2 |
| 605 | "csrc/cpu/sgl-kernels/fla.cpp" |
| 606 | "csrc/cpu/cpu_fused_moe.cpp" |
| 607 | "csrc/cpu/utils.cpp" |
| 608 | "csrc/cpu/spec_decode_utils.cpp" |
| 609 | "csrc/cpu/cpu_attn.cpp" |
| 610 | "csrc/cpu/cpu_isa.cpp" |
| 611 | "csrc/cpu/mamba_cpu.cpp" |
| 612 | "csrc/cpu/dnnl_kernels.cpp" |
| 613 | "csrc/cpu/sampling_kernels.cpp" |
| 614 | "csrc/cpu/torch_bindings.cpp" |
| 615 | # TODO: Remove these files |
| 616 | "csrc/cpu/activation.cpp" |
| 617 | "csrc/cpu/layernorm.cpp" |
| 618 | "csrc/cpu/mla_decode.cpp" |
| 619 | "csrc/cpu/pos_encoding.cpp" |
| 620 | "csrc/moe/dynamic_4bit_int_moe_cpu.cpp") |
| 621 | |
| 622 | message(STATUS "CPU extension (AVX512F + BF16 + VNNI + AMX) source files: ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL}") |
| 623 | message(STATUS "CPU extension (AVX512F) source files: ${VLLM_EXT_SRC_AVX512}") |
| 624 | message(STATUS "CPU extension (AVX2) source files: ${VLLM_EXT_SRC_AVX2}") |
| 625 | |
| 626 | set(_C_LIBS numa dnnl_ext) |
| 627 | set(_C_AVX512_LIBS numa dnnl_ext) |
| 628 | set(_C_AVX2_LIBS numa dnnl_ext) |
| 629 | |
| 630 | # AMX + AVX512F + AVX512BF16 + AVX512VNNI |
| 631 | define_extension_target( |
| 632 | _C |
| 633 | DESTINATION vllm |
| 634 | LANGUAGE CXX |
| 635 | SOURCES ${VLLM_EXT_SRC_AVX512} ${VLLM_EXT_SRC_SGL} |
| 636 | LIBRARIES ${_C_LIBS} |
| 637 | COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512_AMX} |
| 638 | USE_SABI 3 |
| 639 | WITH_SOABI |
| 640 | ) |
| 641 | |
| 642 | # For AMX kernels |
| 643 | target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AMXBF16") |
| 644 | if (AMX_FP8_SUPPORTED) |
| 645 | target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AMXFP8") |
| 646 | message(STATUS "AMX-FP8 (Diamond Rapids) enabled") |
| 647 | endif() |
| 648 | |
| 649 | # For the sgl-kernels AVX10.2 fp8 quantize paths |
| 650 | if (AVX10_2_SUPPORTED) |
| 651 | target_compile_definitions(_C PRIVATE "-DCPU_CAPABILITY_AVX10_2") |
| 652 | message(STATUS "AVX10.2 enabled") |
| 653 | endif() |
| 654 | |
| 655 | # AVX512F |
| 656 | define_extension_target( |
| 657 | _C_AVX512 |
| 658 | DESTINATION vllm |
| 659 | LANGUAGE CXX |
| 660 | SOURCES ${VLLM_EXT_SRC_AVX512} |
| 661 | LIBRARIES ${_C_AVX512_LIBS} |
| 662 | COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX512} |
| 663 | USE_SABI 3 |
| 664 | WITH_SOABI |
| 665 | ) |
| 666 | |
| 667 | # AVX2 |
| 668 | define_extension_target( |
| 669 | _C_AVX2 |
| 670 | DESTINATION vllm |
| 671 | LANGUAGE CXX |
| 672 | SOURCES ${VLLM_EXT_SRC_AVX2} |
| 673 | LIBRARIES ${_C_AVX2_LIBS} |
| 674 | COMPILE_FLAGS ${CXX_COMPILE_FLAGS_AVX2} |
| 675 | USE_SABI 3 |
| 676 | WITH_SOABI |
| 677 | ) |
| 678 | else() |
| 679 | message(STATUS "CPU extension source files: ${VLLM_EXT_SRC}") |
| 680 | # |
| 681 | # Define extension targets |
| 682 | # |
| 683 | define_extension_target( |
| 684 | _C |
| 685 | DESTINATION vllm |
| 686 | LANGUAGE CXX |
| 687 | SOURCES ${VLLM_EXT_SRC} |
| 688 | LIBRARIES ${LIBS} |
| 689 | COMPILE_FLAGS ${CXX_COMPILE_FLAGS} |
| 690 | USE_SABI 3 |
| 691 | WITH_SOABI |
| 692 | ) |
| 693 | if (VLLM_OPENBLAS_LIB) |
| 694 | target_compile_definitions(_C PRIVATE VLLM_HAS_OPENBLAS) |
| 695 | endif() |
| 696 | endif() |
| 697 | |
| 698 | message(STATUS "Enabling C extension.") |