A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Make name of `compressed-tensors` quant method consistent across vLLM (#17255)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 28, 2025 at 17:28 UTC
b6dd32aa07c1484a90895257e7b7ec2fbbc426d3
5 files changed
+10
-14
tests/compile/test_full_graph.py
+1
-5
@@ -20,15 +20,11 @@ def models_list(*, all: bool = True, keywords: Optional[list[str]] = None):
20
("facebook/opt-125m", {}),
21
("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
22
"dtype": torch.float16,
23
- "quantization": "compressed-tensors"
23
}),
24
("neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic", {
25
"dtype": torch.float16,
27
- "quantization": "compressed-tensors"
28
- }),
29
- ("neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8", {
30
- "quantization": "compressed-tensors"
26
}),
27
+ ("neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8", {}),
28
("meta-llama/Llama-3.2-1B-Instruct", {}),
29
]
30
vllm/config.py
+5
-3
@@ -752,9 +752,8 @@ class ModelConfig:
752
supported_quantization = QUANTIZATION_METHODS
753
optimized_quantization_methods = [
754
"fp8", "marlin", "modelopt", "gptq_marlin_24", "gptq_marlin",
755
- "awq_marlin", "fbgemm_fp8", "compressed_tensors",
756
- "compressed-tensors", "experts_int8", "quark", "nvfp4", "bitblas",
757
- "gptq_bitblas"
755
+ "awq_marlin", "fbgemm_fp8", "compressed-tensors", "experts_int8",
756
+ "quark", "nvfp4", "bitblas", "gptq_bitblas"
757
]
758
if self.quantization is not None:
759
self.quantization = self.quantization.lower()
@@ -764,6 +763,9 @@ class ModelConfig:
763
764
if quant_cfg is not None:
765
quant_method = quant_cfg.get("quant_method", "").lower()
766
+ quant_method = quant_method.replace("compressed_tensors",
767
+ "compressed-tensors")
768
+ quant_cfg["quant_method"] = quant_method
769
770
# Detect which checkpoint is it
771
for name in QUANTIZATION_METHODS:
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+1
-1
@@ -72,7 +72,7 @@ class CompressedTensorsConfig(QuantizationConfig):
72
return 70
73
74
def get_name(self) -> str:
75
- return "compressed_tensors"
75
+ return "compressed-tensors"
76
77
def get_quant_method(
78
self,
vllm/platforms/rocm.py
+2
-2
@@ -130,8 +130,8 @@ class RocmPlatform(Platform):
130
device_control_env_var: str = "CUDA_VISIBLE_DEVICES"
131
132
supported_quantization: list[str] = [
133
- "awq", "gptq", "fp8", "compressed_tensors", "compressed-tensors",
134
- "fbgemm_fp8", "gguf", "quark", "ptpc_fp8"
133
+ "awq", "gptq", "fp8", "compressed-tensors", "fbgemm_fp8", "gguf",
134
+ "quark", "ptpc_fp8"
135
]
136
137
@classmethod
vllm/platforms/tpu.py
+1
-3
@@ -30,9 +30,7 @@ class TpuPlatform(Platform):
30
ray_device_key: str = "TPU"
31
device_control_env_var: str = "TPU_VISIBLE_CHIPS"
32
33
- supported_quantization: list[str] = [
34
- "tpu_int8", "compressed-tensors", "compressed_tensors"
35
- ]
33
+ supported_quantization: list[str] = ["tpu_int8", "compressed-tensors"]
34
35
additional_env_vars: list[str] = [
36
"TPU_CHIPS_PER_HOST_BOUNDS", "TPU_HOST_BOUNDS"