@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9101d832e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Bugfix] Make moe_align_block_size AMD-compatible (#3470)

Woosuk Kwon committed Mar 18, 2024 at 11:26 UTC 9101d832e6fe3811db8faa739f4a7e6e2f32a240
1 file changed +2 -1
csrc/moe_align_block_size_kernels.cu
+2 -1
@@ -111,7 +111,8 @@ void moe_align_block_size(
111
112 // set dynamic shared mem
113 auto kernel = vllm::moe_align_block_size_kernel<scalar_t>;
114 - AT_CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, shared_mem));
114 + AT_CUDA_CHECK(
115 + VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize((void *)kernel, shared_mem));
116 kernel<<<1, num_experts, shared_mem, stream>>>(
117 topk_ids.data_ptr<scalar_t>(),
118 sorted_token_ids.data_ptr<int32_t>(),