@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bb00f66e19
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Use `quantization_config` in hf config (#1695)

Woosuk Kwon committed Nov 17, 2023 at 16:23 UTC bb00f66e19acdf6cb614683ab74f777ed3932eee
3 files changed +34 -10
vllm/config.py
+24 -8
@@ -104,14 +104,30 @@ class ModelConfig:
104
105 def _verify_quantization(self) -> None:
106 supported_quantization = ["awq", "squeezellm"]
107 - if self.quantization is None:
108 - return
109 - quantization = self.quantization.lower()
110 - if quantization not in supported_quantization:
111 - raise ValueError(
112 - f"Unknown quantization: {self.quantization}. Must be one of "
113 - f"{supported_quantization}.")
114 - self.quantization = quantization
107 + if self.quantization is not None:
108 + self.quantization = self.quantization.lower()
109 +
110 + # Parse quantization method from the HF model config, if available.
111 + hf_quant_config = getattr(self.hf_config, "quantization_config", None)
112 + if hf_quant_config is not None:
113 + hf_quant_method = str(hf_quant_config["quant_method"]).lower()
114 + if self.quantization is None:
115 + self.quantization = hf_quant_method
116 + elif self.quantization != hf_quant_method:
117 + raise ValueError(
118 + "Quantization method specified in the model config "
119 + f"({hf_quant_method}) does not match the quantization "
120 + f"method specified in the `quantization` argument "
121 + f"({self.quantization}).")
122 +
123 + if self.quantization is not None:
124 + if self.quantization not in supported_quantization:
125 + raise ValueError(
126 + f"Unknown quantization method: {self.quantization}. Must "
127 + f"be one of {supported_quantization}.")
128 + logger.warning(f"{self.quantization} quantization is not fully "
129 + "optimized yet. The speed can be slower than "
130 + "non-quantized models.")
131
132 def verify_with_parallel_config(
133 self,
vllm/model_executor/model_loader.py
+1
@@ -66,6 +66,7 @@ def get_model(model_config: ModelConfig) -> nn.Module:
66 if model_config.quantization is not None:
67 quant_config = get_quant_config(model_config.quantization,
68 model_config.model,
69 + model_config.hf_config,
70 model_config.download_dir)
71 capability = torch.cuda.get_device_capability()
72 capability = capability[0] * 10 + capability[1]
vllm/model_executor/weight_utils.py
+9 -2
@@ -7,9 +7,10 @@ from collections import defaultdict
7 from typing import Any, Iterator, List, Optional, Tuple
8
9 from huggingface_hub import snapshot_download
10 -from safetensors.torch import load_file, save_file, safe_open
10 import numpy as np
11 +from safetensors.torch import load_file, save_file, safe_open
12 import torch
13 +from transformers import PretrainedConfig
14 from tqdm.auto import tqdm
15
16 from vllm.logger import init_logger
@@ -84,8 +85,15 @@ def convert_bin_to_safetensor_file(
85 def get_quant_config(
86 quantization: str,
87 model_name_or_path: str,
88 + hf_config: PretrainedConfig,
89 cache_dir: Optional[str] = None,
90 ) -> QuantizationConfig:
91 + quant_cls = get_quantization_config(quantization)
92 + # Read the quantization config from the HF model config, if available.
93 + hf_quant_config = getattr(hf_config, "quantization_config", None)
94 + if hf_quant_config is not None:
95 + return quant_cls.from_config(hf_quant_config)
96 +
97 is_local = os.path.isdir(model_name_or_path)
98 if not is_local:
99 # Download the config files.
@@ -98,7 +106,6 @@ def get_quant_config(
106 hf_folder = model_name_or_path
107 config_files = glob.glob(os.path.join(hf_folder, "*.json"))
108
101 - quant_cls = get_quantization_config(quantization)
109 quant_config_files = [
110 f for f in config_files if any(
111 f.endswith(x) for x in quant_cls.get_config_filenames())