A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Use `quantization_config` in hf config (#1695)
Woosuk Kwon committed
Nov 17, 2023 at 16:23 UTC
bb00f66e19acdf6cb614683ab74f777ed3932eee
3 files changed
+34
-10
vllm/config.py
+24
-8
@@ -104,14 +104,30 @@ class ModelConfig:
104
105
def _verify_quantization(self) -> None:
106
supported_quantization = ["awq", "squeezellm"]
107
- if self.quantization is None:
108
- return
109
- quantization = self.quantization.lower()
110
- if quantization not in supported_quantization:
111
- raise ValueError(
112
- f"Unknown quantization: {self.quantization}. Must be one of "
113
- f"{supported_quantization}.")
114
- self.quantization = quantization
107
+ if self.quantization is not None:
108
+ self.quantization = self.quantization.lower()
109
+
110
+ # Parse quantization method from the HF model config, if available.
111
+ hf_quant_config = getattr(self.hf_config, "quantization_config", None)
112
+ if hf_quant_config is not None:
113
+ hf_quant_method = str(hf_quant_config["quant_method"]).lower()
114
+ if self.quantization is None:
115
+ self.quantization = hf_quant_method
116
+ elif self.quantization != hf_quant_method:
117
+ raise ValueError(
118
+ "Quantization method specified in the model config "
119
+ f"({hf_quant_method}) does not match the quantization "
120
+ f"method specified in the `quantization` argument "
121
+ f"({self.quantization}).")
122
+
123
+ if self.quantization is not None:
124
+ if self.quantization not in supported_quantization:
125
+ raise ValueError(
126
+ f"Unknown quantization method: {self.quantization}. Must "
127
+ f"be one of {supported_quantization}.")
128
+ logger.warning(f"{self.quantization} quantization is not fully "
129
+ "optimized yet. The speed can be slower than "
130
+ "non-quantized models.")
131
132
def verify_with_parallel_config(
133
self,
vllm/model_executor/model_loader.py
+1
@@ -66,6 +66,7 @@ def get_model(model_config: ModelConfig) -> nn.Module:
66
if model_config.quantization is not None:
67
quant_config = get_quant_config(model_config.quantization,
68
model_config.model,
69
+ model_config.hf_config,
70
model_config.download_dir)
71
capability = torch.cuda.get_device_capability()
72
capability = capability[0] * 10 + capability[1]
vllm/model_executor/weight_utils.py
+9
-2
@@ -7,9 +7,10 @@ from collections import defaultdict
7
from typing import Any, Iterator, List, Optional, Tuple
8
9
from huggingface_hub import snapshot_download
10
-from safetensors.torch import load_file, save_file, safe_open
10
import numpy as np
11
+from safetensors.torch import load_file, save_file, safe_open
12
import torch
13
+from transformers import PretrainedConfig
14
from tqdm.auto import tqdm
15
16
from vllm.logger import init_logger
@@ -84,8 +85,15 @@ def convert_bin_to_safetensor_file(
85
def get_quant_config(
86
quantization: str,
87
model_name_or_path: str,
88
+ hf_config: PretrainedConfig,
89
cache_dir: Optional[str] = None,
90
) -> QuantizationConfig:
91
+ quant_cls = get_quantization_config(quantization)
92
+ # Read the quantization config from the HF model config, if available.
93
+ hf_quant_config = getattr(hf_config, "quantization_config", None)
94
+ if hf_quant_config is not None:
95
+ return quant_cls.from_config(hf_quant_config)
96
+
97
is_local = os.path.isdir(model_name_or_path)
98
if not is_local:
99
# Download the config files.
@@ -98,7 +106,6 @@ def get_quant_config(
106
hf_folder = model_name_or_path
107
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
108
101
- quant_cls = get_quantization_config(quantization)
109
quant_config_files = [
110
f for f in config_files if any(
111
f.endswith(x) for x in quant_cls.get_config_filenames())