@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2acd76f346
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[ROCm] Temporarily remove GPTQ ROCm support (#2138)

Woosuk Kwon committed Dec 15, 2023 at 17:13 UTC 2acd76f346efcdff4f6ca1d92fe1575c448e4b70
2 files changed +2 -2
setup.py
+1 -1
@@ -219,13 +219,13 @@ vllm_extension_sources = [
219 "csrc/activation_kernels.cu",
220 "csrc/layernorm_kernels.cu",
221 "csrc/quantization/squeezellm/quant_cuda_kernel.cu",
222 - "csrc/quantization/gptq/q_gemm.cu",
222 "csrc/cuda_utils_kernels.cu",
223 "csrc/pybind.cpp",
224 ]
225
226 if _is_cuda():
227 vllm_extension_sources.append("csrc/quantization/awq/gemm_kernels.cu")
228 + vllm_extension_sources.append("csrc/quantization/gptq/q_gemm.cu")
229
230 vllm_extension = CUDAExtension(
231 name="vllm._C",
vllm/config.py
+1 -1
@@ -143,7 +143,7 @@ class ModelConfig:
143
144 def _verify_quantization(self) -> None:
145 supported_quantization = ["awq", "gptq", "squeezellm"]
146 - rocm_not_supported_quantization = ["awq"]
146 + rocm_not_supported_quantization = ["awq", "gptq"]
147 if self.quantization is not None:
148 self.quantization = self.quantization.lower()
149