@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ef00b03a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Enable CUDA graph for GPTQ & SqueezeLLM (#2318)

Woosuk Kwon committed Jan 3, 2024 at 09:52 UTC 6ef00b03a2b6679d494530e9a98932c5a6cc8418
3 files changed +15 -13
csrc/quantization/gptq/q_gemm.cu
+12 -6
@@ -287,7 +287,8 @@ void gemm_half_q_half_cuda_part
287
288 fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);
289
290 - kernel<<<gridDim, blockDim>>>
290 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
291 + kernel<<<gridDim, blockDim, 0, stream>>>
292 (
293 a,
294 b_q_weight,
@@ -434,7 +435,8 @@ void reconstruct_exllama
435 gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
436 gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
437
437 - reconstruct_exllama_kernel<<<gridDim, blockDim>>>
438 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
439 + reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>
440 (
441 b_q_weight,
442 b_q_perm,
@@ -567,7 +569,8 @@ void gemm_half_q_half_alt
569 gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
570 gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
571
570 - gemm_half_q_half_alt_kernel<<<gridDim, blockDim>>>
572 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
573 + gemm_half_q_half_alt_kernel<<<gridDim, blockDim, 0, stream>>>
574 (
575 (const half2*) a,
576 b_q_weight,
@@ -639,7 +642,8 @@ void reconstruct_gptq
642 blockDim.y = 1;
643 gridDim.y = DIVIDE(height, 8);
644 gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
642 - reconstruct_gptq_kernel<<<gridDim, blockDim>>>
645 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
646 + reconstruct_gptq_kernel<<<gridDim, blockDim, 0, stream>>>
647 (
648 b_q_weight,
649 b_gptq_scales,
@@ -794,7 +798,8 @@ void shuffle_exllama_weight
798 gridDim.x = DIVIDE(width, THREADS_X);
799 gridDim.y = height / 8;
800
797 - make_sequential_kernel<<<gridDim, blockDim>>>
801 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
802 + make_sequential_kernel<<<gridDim, blockDim, 0, stream>>>
803 (
804 q_weight,
805 new_qweight,
@@ -813,7 +818,8 @@ void shuffle_exllama_weight
818 blockDim.y = 1;
819 gridDim.x = DIVIDE(width, THREADS_X);
820 gridDim.y = 1;
816 - shuffle_kernel<<<gridDim, blockDim>>>(q_weight, height, width);
821 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
822 + shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
823 }
824
825 } // namespace gptq
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+3 -1
@@ -200,8 +200,10 @@ void squeezellm_gemm(
200 (width + BLOCKWIDTH - 1) / BLOCKWIDTH
201 );
202 dim3 threads(BLOCKWIDTH);
203 +
204 const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
204 - vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
205 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
206 + vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads, 0, stream>>>(
207 #ifndef USE_ROCM
208 (half2*) vec.data<at::Half>(),
209 #else
vllm/config.py
-6
@@ -181,12 +181,6 @@ class ModelConfig:
181 self.max_context_len_to_capture = self.max_model_len
182 self.max_context_len_to_capture = min(self.max_context_len_to_capture,
183 self.max_model_len)
184 - if (self.quantization in ["gptq", "squeezellm"]
185 - and not self.enforce_eager):
186 - # Related issue: https://github.com/vllm-project/vllm/issues/2147
187 - logger.warning(f"{self.quantization} does not support CUDA graph "
188 - "yet. Disabling CUDA graph.")
189 - self.enforce_eager = True
184
185 def verify_with_parallel_config(
186 self,