A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Get available quantization methods from quantization registry (#4098)
Michael Goin committed
Apr 18, 2024 at 03:21 UTC
53b018edcbc601f0eea9f65f13a9a9620c4be8dc
6 files changed
+18
-13
benchmarks/benchmark_latency.py
+2
-1
@@ -9,6 +9,7 @@ import torch
9
from tqdm import tqdm
10
11
from vllm import LLM, SamplingParams
12
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
14
15
def main(args: argparse.Namespace):
@@ -101,7 +102,7 @@ if __name__ == '__main__':
102
parser.add_argument('--tokenizer', type=str, default=None)
103
parser.add_argument('--quantization',
104
'-q',
104
- choices=['awq', 'gptq', 'squeezellm', None],
105
+ choices=[*QUANTIZATION_METHODS, None],
106
default=None)
107
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
108
parser.add_argument('--input-len', type=int, default=32)
benchmarks/benchmark_throughput.py
+3
-1
@@ -10,6 +10,8 @@ from tqdm import tqdm
10
from transformers import (AutoModelForCausalLM, AutoTokenizer,
11
PreTrainedTokenizerBase)
12
13
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14
+
15
16
def sample_requests(
17
dataset_path: str,
@@ -267,7 +269,7 @@ if __name__ == "__main__":
269
parser.add_argument("--tokenizer", type=str, default=None)
270
parser.add_argument('--quantization',
271
'-q',
270
- choices=['awq', 'gptq', 'squeezellm', None],
272
+ choices=[*QUANTIZATION_METHODS, None],
273
default=None)
274
parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
275
parser.add_argument("--n",
tests/models/test_marlin.py
+3
-4
@@ -16,13 +16,12 @@ from dataclasses import dataclass
16
import pytest
17
import torch
18
19
-from vllm.model_executor.layers.quantization import (
20
- _QUANTIZATION_CONFIG_REGISTRY)
19
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
20
21
capability = torch.cuda.get_device_capability()
22
capability = capability[0] * 10 + capability[1]
24
-marlin_not_supported = (
25
- capability < _QUANTIZATION_CONFIG_REGISTRY["marlin"].get_min_capability())
23
+marlin_not_supported = (capability <
24
+ QUANTIZATION_METHODS["marlin"].get_min_capability())
25
26
27
@dataclass
vllm/config.py
+4
-3
@@ -9,6 +9,7 @@ from packaging.version import Version
9
from transformers import PretrainedConfig
10
11
from vllm.logger import init_logger
12
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
from vllm.transformers_utils.config import get_config, get_hf_text_config
14
from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
15
is_neuron)
@@ -118,8 +119,8 @@ class ModelConfig:
119
self.tokenizer_mode = tokenizer_mode
120
121
def _verify_quantization(self) -> None:
121
- supported_quantization = ["awq", "gptq", "squeezellm", "marlin"]
122
- rocm_not_supported_quantization = ["awq", "marlin"]
122
+ supported_quantization = [*QUANTIZATION_METHODS]
123
+ rocm_supported_quantization = ["gptq", "squeezellm"]
124
if self.quantization is not None:
125
self.quantization = self.quantization.lower()
126
@@ -155,7 +156,7 @@ class ModelConfig:
156
f"Unknown quantization method: {self.quantization}. Must "
157
f"be one of {supported_quantization}.")
158
if is_hip(
158
- ) and self.quantization in rocm_not_supported_quantization:
159
+ ) and self.quantization not in rocm_supported_quantization:
160
raise ValueError(
161
f"{self.quantization} quantization is currently not "
162
f"supported in ROCm.")
vllm/engine/arg_utils.py
+2
-1
@@ -7,6 +7,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
7
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
8
ParallelConfig, SchedulerConfig, SpeculativeConfig,
9
TokenizerPoolConfig, VisionLanguageConfig)
10
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11
from vllm.utils import str_to_int_tuple
12
13
@@ -286,7 +287,7 @@ class EngineArgs:
287
parser.add_argument('--quantization',
288
'-q',
289
type=str,
289
- choices=['awq', 'gptq', 'squeezellm', None],
290
+ choices=[*QUANTIZATION_METHODS, None],
291
default=EngineArgs.quantization,
292
help='Method used to quantize the weights. If '
293
'None, we first check the `quantization_config` '
vllm/model_executor/layers/quantization/__init__.py
+4
-3
@@ -7,7 +7,7 @@ from vllm.model_executor.layers.quantization.gptq import GPTQConfig
7
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
8
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
9
10
-_QUANTIZATION_CONFIG_REGISTRY = {
10
+QUANTIZATION_METHODS = {
11
"awq": AWQConfig,
12
"gptq": GPTQConfig,
13
"squeezellm": SqueezeLLMConfig,
@@ -16,12 +16,13 @@ _QUANTIZATION_CONFIG_REGISTRY = {
16
17
18
def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
19
- if quantization not in _QUANTIZATION_CONFIG_REGISTRY:
19
+ if quantization not in QUANTIZATION_METHODS:
20
raise ValueError(f"Invalid quantization method: {quantization}")
21
- return _QUANTIZATION_CONFIG_REGISTRY[quantization]
21
+ return QUANTIZATION_METHODS[quantization]
22
23
24
__all__ = [
25
"QuantizationConfig",
26
"get_quantization_config",
27
+ "QUANTIZATION_METHODS",
28
]