A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Bugfix] Make moe_align_block_size AMD-compatible (#3470)
Woosuk Kwon committed
Mar 18, 2024 at 11:26 UTC
9101d832e6fe3811db8faa739f4a7e6e2f32a240
1 file changed
+2
-1
csrc/moe_align_block_size_kernels.cu
+2
-1
@@ -111,7 +111,8 @@ void moe_align_block_size(
111
112
// set dynamic shared mem
113
auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
114
- AT_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem));
114
+ AT_CUDA_CHECK(
115
+ VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize((void *)kernel, shared_mem));
116
kernel<<<1, num_experts, shared_mem, stream>>>(
117
topk_ids.data_ptr<scalar_t>(),
118
sorted_token_ids.data_ptr<int32_t>(),