A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Enable CUDA graph for GPTQ & SqueezeLLM (#2318)
Woosuk Kwon committed
Jan 3, 2024 at 09:52 UTC
6ef00b03a2b6679d494530e9a98932c5a6cc8418
3 files changed
+15
-13
csrc/quantization/gptq/q_gemm.cu
+12
-6
@@ -287,7 +287,8 @@ void gemm_half_q_half_cuda_part
287
288
fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);
289
290
- kernel<<<gridDim, blockDim>>>
290
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
291
+ kernel<<<gridDim, blockDim, 0, stream>>>
292
(
293
a,
294
b_q_weight,
@@ -434,7 +435,8 @@ void reconstruct_exllama
435
gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
436
gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
437
437
- reconstruct_exllama_kernel<<<gridDim, blockDim>>>
438
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
439
+ reconstruct_exllama_kernel<<<gridDim, blockDim, 0, stream>>>
440
(
441
b_q_weight,
442
b_q_perm,
@@ -567,7 +569,8 @@ void gemm_half_q_half_alt
569
gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX);
570
gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
571
570
- gemm_half_q_half_alt_kernel<<<gridDim, blockDim>>>
572
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
573
+ gemm_half_q_half_alt_kernel<<<gridDim, blockDim, 0, stream>>>
574
(
575
(const half2*) a,
576
b_q_weight,
@@ -639,7 +642,8 @@ void reconstruct_gptq
642
blockDim.y = 1;
643
gridDim.y = DIVIDE(height, 8);
644
gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
642
- reconstruct_gptq_kernel<<<gridDim, blockDim>>>
645
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
646
+ reconstruct_gptq_kernel<<<gridDim, blockDim, 0, stream>>>
647
(
648
b_q_weight,
649
b_gptq_scales,
@@ -794,7 +798,8 @@ void shuffle_exllama_weight
798
gridDim.x = DIVIDE(width, THREADS_X);
799
gridDim.y = height / 8;
800
797
- make_sequential_kernel<<<gridDim, blockDim>>>
801
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
802
+ make_sequential_kernel<<<gridDim, blockDim, 0, stream>>>
803
(
804
q_weight,
805
new_qweight,
@@ -813,7 +818,8 @@ void shuffle_exllama_weight
818
blockDim.y = 1;
819
gridDim.x = DIVIDE(width, THREADS_X);
820
gridDim.y = 1;
816
- shuffle_kernel<<<gridDim, blockDim>>>(q_weight, height, width);
821
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
822
+ shuffle_kernel<<<gridDim, blockDim, 0, stream>>>(q_weight, height, width);
823
}
824
825
} // namespace gptq
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+3
-1
@@ -200,8 +200,10 @@ void squeezellm_gemm(
200
(width + BLOCKWIDTH - 1) / BLOCKWIDTH
201
);
202
dim3 threads(BLOCKWIDTH);
203
+
204
const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
204
- vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
205
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
206
+ vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads, 0, stream>>>(
207
#ifndef USE_ROCM
208
(half2*) vec.data<at::Half>(),
209
#else
vllm/config.py
-6
@@ -181,12 +181,6 @@ class ModelConfig:
181
self.max_context_len_to_capture = self.max_model_len
182
self.max_context_len_to_capture = min(self.max_context_len_to_capture,
183
self.max_model_len)
184
- if (self.quantization in ["gptq", "squeezellm"]
185
- and not self.enforce_eager):
186
- # Related issue: https://github.com/vllm-project/vllm/issues/2147
187
- logger.warning(f"{self.quantization} does not support CUDA graph "
188
- "yet. Disabling CUDA graph.")
189
- self.enforce_eager = True
184
185
def verify_with_parallel_config(
186
self,