@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 53b018edcb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Get available quantization methods from quantization registry (#4098)

Michael Goin committed Apr 18, 2024 at 03:21 UTC 53b018edcbc601f0eea9f65f13a9a9620c4be8dc
6 files changed +18 -13
benchmarks/benchmark_latency.py
+2 -1
@@ -9,6 +9,7 @@ import torch
9 from tqdm import tqdm
10
11 from vllm import LLM, SamplingParams
12 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
14
15 def main(args: argparse.Namespace):
@@ -101,7 +102,7 @@ if __name__ == '__main__':
102 parser.add_argument('--tokenizer', type=str, default=None)
103 parser.add_argument('--quantization',
104 '-q',
104 - choices=['awq', 'gptq', 'squeezellm', None],
105 + choices=[*QUANTIZATION_METHODS, None],
106 default=None)
107 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1)
108 parser.add_argument('--input-len', type=int, default=32)
benchmarks/benchmark_throughput.py
+3 -1
@@ -10,6 +10,8 @@ from tqdm import tqdm
10 from transformers import (AutoModelForCausalLM, AutoTokenizer,
11 PreTrainedTokenizerBase)
12
13 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14 +
15
16 def sample_requests(
17 dataset_path: str,
@@ -267,7 +269,7 @@ if __name__ == "__main__":
269 parser.add_argument("--tokenizer", type=str, default=None)
270 parser.add_argument('--quantization',
271 '-q',
270 - choices=['awq', 'gptq', 'squeezellm', None],
272 + choices=[*QUANTIZATION_METHODS, None],
273 default=None)
274 parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
275 parser.add_argument("--n",
tests/models/test_marlin.py
+3 -4
@@ -16,13 +16,12 @@ from dataclasses import dataclass
16 import pytest
17 import torch
18
19 -from vllm.model_executor.layers.quantization import (
20 - _QUANTIZATION_CONFIG_REGISTRY)
19 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
20
21 capability = torch.cuda.get_device_capability()
22 capability = capability[0] * 10 + capability[1]
24 -marlin_not_supported = (
25 - capability < _QUANTIZATION_CONFIG_REGISTRY["marlin"].get_min_capability())
23 +marlin_not_supported = (capability <
24 + QUANTIZATION_METHODS["marlin"].get_min_capability())
25
26
27 @dataclass
vllm/config.py
+4 -3
@@ -9,6 +9,7 @@ from packaging.version import Version
9 from transformers import PretrainedConfig
10
11 from vllm.logger import init_logger
12 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13 from vllm.transformers_utils.config import get_config, get_hf_text_config
14 from vllm.utils import (get_cpu_memory, get_nvcc_cuda_version, is_cpu, is_hip,
15 is_neuron)
@@ -118,8 +119,8 @@ class ModelConfig:
119 self.tokenizer_mode = tokenizer_mode
120
121 def _verify_quantization(self) -> None:
121 - supported_quantization = ["awq", "gptq", "squeezellm", "marlin"]
122 - rocm_not_supported_quantization = ["awq", "marlin"]
122 + supported_quantization = [*QUANTIZATION_METHODS]
123 + rocm_supported_quantization = ["gptq", "squeezellm"]
124 if self.quantization is not None:
125 self.quantization = self.quantization.lower()
126
@@ -155,7 +156,7 @@ class ModelConfig:
156 f"Unknown quantization method: {self.quantization}. Must "
157 f"be one of {supported_quantization}.")
158 if is_hip(
158 - ) and self.quantization in rocm_not_supported_quantization:
159 + ) and self.quantization not in rocm_supported_quantization:
160 raise ValueError(
161 f"{self.quantization} quantization is currently not "
162 f"supported in ROCm.")
vllm/engine/arg_utils.py
+2 -1
@@ -7,6 +7,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
7 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
8 ParallelConfig, SchedulerConfig, SpeculativeConfig,
9 TokenizerPoolConfig, VisionLanguageConfig)
10 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11 from vllm.utils import str_to_int_tuple
12
13
@@ -286,7 +287,7 @@ class EngineArgs:
287 parser.add_argument('--quantization',
288 '-q',
289 type=str,
289 - choices=['awq', 'gptq', 'squeezellm', None],
290 + choices=[*QUANTIZATION_METHODS, None],
291 default=EngineArgs.quantization,
292 help='Method used to quantize the weights. If '
293 'None, we first check the `quantization_config` '
vllm/model_executor/layers/quantization/__init__.py
+4 -3
@@ -7,7 +7,7 @@ from vllm.model_executor.layers.quantization.gptq import GPTQConfig
7 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
8 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
9
10 -_QUANTIZATION_CONFIG_REGISTRY = {
10 +QUANTIZATION_METHODS = {
11 "awq": AWQConfig,
12 "gptq": GPTQConfig,
13 "squeezellm": SqueezeLLMConfig,
@@ -16,12 +16,13 @@ _QUANTIZATION_CONFIG_REGISTRY = {
16
17
18 def get_quantization_config(quantization: str) -> Type[QuantizationConfig]:
19 - if quantization not in _QUANTIZATION_CONFIG_REGISTRY:
19 + if quantization not in QUANTIZATION_METHODS:
20 raise ValueError(f"Invalid quantization method: {quantization}")
21 - return _QUANTIZATION_CONFIG_REGISTRY[quantization]
21 + return QUANTIZATION_METHODS[quantization]
22
23
24 __all__ = [
25 "QuantizationConfig",
26 "get_quantization_config",
27 + "QUANTIZATION_METHODS",
28 ]