| 1 | #pragma once |
| 2 | |
| 3 | #include <torch/csrc/inductor/aoti_torch/c/shim.h> |
| 4 | #include <torch/csrc/stable/accelerator.h> |
| 5 | #include <torch/csrc/stable/ops.h> |
| 6 | #include <torch/csrc/stable/tensor.h> |
| 7 | #include <torch/headeronly/util/shim_utils.h> |
| 8 | |
| 9 | #include <cuda_runtime.h> |
| 10 | #include <cublas_v2.h> |
| 11 | |
| 12 | #include <deque> |
| 13 | #include <mutex> |
| 14 | #include <string> |
| 15 | #include <vector> |
| 16 | |
| 17 | // Stable ABI equivalent of TORCH_CHECK_NOT_IMPLEMENTED. |
| 18 | #define STD_TORCH_CHECK_NOT_IMPLEMENTED(cond, ...) \ |
| 19 | STD_TORCH_CHECK(cond, "NotImplementedError: ", __VA_ARGS__) |
| 20 | |
| 21 | // Device properties cache for stable ABI compatibility. |
| 22 | // Uses raw CUDA/HIP APIs instead of ATen functions. |
| 23 | // Using inline ensures a single instance across all translation units. |
| 24 | inline std::deque<std::once_flag> device_flags; |
| 25 | inline std::vector<cudaDeviceProp> device_properties; |
| 26 | inline std::once_flag vectors_init_flag; |
| 27 | |
| 28 | inline void do_init_device_vectors() { |
| 29 | int device_count; |
| 30 | cudaError_t err = cudaGetDeviceCount(&device_count); |
| 31 | if (err != cudaSuccess) { |
| 32 | STD_TORCH_CHECK(false, "cudaGetDeviceCount failed: " + |
| 33 | std::string(cudaGetErrorString(err))); |
| 34 | } |
| 35 | device_flags.resize(device_count); |
| 36 | device_properties.resize(device_count); |
| 37 | } |
| 38 | |
| 39 | inline void initDeviceVectors() { |
| 40 | std::call_once(vectors_init_flag, do_init_device_vectors); |
| 41 | } |
| 42 | |
| 43 | inline void initDeviceProperty(int device_index) { |
| 44 | cudaDeviceProp device_prop{}; |
| 45 | cudaError_t err = cudaGetDeviceProperties(&device_prop, device_index); |
| 46 | if (err != cudaSuccess) { |
| 47 | STD_TORCH_CHECK(false, "cudaGetDeviceProperties failed: " + |
| 48 | std::string(cudaGetErrorString(err))); |
| 49 | } |
| 50 | device_properties[device_index] = device_prop; |
| 51 | } |
| 52 | |
| 53 | // Get device properties using raw CUDA/HIP APIs (stable ABI compatible). |
| 54 | // Caches results per device so cudaGetDeviceProperties is called at most once |
| 55 | // per device. |
| 56 | inline cudaDeviceProp* get_device_prop() { |
| 57 | initDeviceVectors(); |
| 58 | int device_index; |
| 59 | cudaError_t err = cudaGetDevice(&device_index); |
| 60 | if (err != cudaSuccess) { |
| 61 | STD_TORCH_CHECK( |
| 62 | false, "cudaGetDevice failed: " + std::string(cudaGetErrorString(err))); |
| 63 | } |
| 64 | STD_TORCH_CHECK(device_index >= 0 && static_cast<size_t>(device_index) < |
| 65 | device_properties.size(), |
| 66 | "CUDA device index " + std::to_string(device_index) + |
| 67 | " out of range [0, " + |
| 68 | std::to_string(device_properties.size()) + ")"); |
| 69 | |
| 70 | std::call_once(device_flags[device_index], initDeviceProperty, device_index); |
| 71 | return &device_properties[device_index]; |
| 72 | } |
| 73 | |
| 74 | // Utility to get the current CUDA stream for a given device using stable APIs. |
| 75 | // Returns a cudaStream_t for use in kernel launches. |
| 76 | inline cudaStream_t get_current_cuda_stream(int32_t device_index = -1) { |
| 77 | void* stream_ptr = nullptr; |
| 78 | TORCH_ERROR_CODE_CHECK( |
| 79 | aoti_torch_get_current_cuda_stream(device_index, &stream_ptr)); |
| 80 | return reinterpret_cast<cudaStream_t>(stream_ptr); |
| 81 | } |
| 82 | |
| 83 | // Utility to get the current cuBLAS handle using stable APIs. |
| 84 | inline cublasHandle_t get_current_cuda_blas_handle() { |
| 85 | void* blas_handle_ptr = nullptr; |
| 86 | TORCH_ERROR_CODE_CHECK(torch_get_current_cuda_blas_handle(&blas_handle_ptr)); |
| 87 | return reinterpret_cast<cublasHandle_t>(blas_handle_ptr); |
| 88 | } |