A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Use revision when downloading the quantization config file (#2697)
Co-authored-by: Pernekhan Utemuratov <pernekhan@deepinfra.com>
Pernekhan Utemuratov committed
Feb 1, 2024 at 15:41 UTC
c410f5d020216df2dfedde52bcae24ae7f0ac7ec
2 files changed
+15
-19
vllm/model_executor/model_loader.py
+1
-4
@@ -44,10 +44,7 @@ def get_model(model_config: ModelConfig,
44
# Get the (maybe quantized) linear method.
45
linear_method = None
46
if model_config.quantization is not None:
47
- quant_config = get_quant_config(model_config.quantization,
48
- model_config.model,
49
- model_config.hf_config,
50
- model_config.download_dir)
47
+ quant_config = get_quant_config(model_config)
48
capability = torch.cuda.get_device_capability()
49
capability = capability[0] * 10 + capability[1]
50
if capability < quant_config.get_min_capability():
vllm/model_executor/weight_utils.py
+14
-15
@@ -11,9 +11,9 @@ from huggingface_hub import snapshot_download, HfFileSystem
11
import numpy as np
12
from safetensors.torch import load_file, save_file, safe_open
13
import torch
14
-from transformers import PretrainedConfig
14
from tqdm.auto import tqdm
15
16
+from vllm.config import ModelConfig
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.quantization import (get_quantization_config,
19
QuantizationConfig)
@@ -83,25 +83,22 @@ def convert_bin_to_safetensor_file(
83
84
85
# TODO(woosuk): Move this to other place.
86
-def get_quant_config(
87
- quantization: str,
88
- model_name_or_path: str,
89
- hf_config: PretrainedConfig,
90
- cache_dir: Optional[str] = None,
91
-) -> QuantizationConfig:
92
- quant_cls = get_quantization_config(quantization)
86
+def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
87
+ quant_cls = get_quantization_config(model_config.quantization)
88
# Read the quantization config from the HF model config, if available.
94
- hf_quant_config = getattr(hf_config, "quantization_config", None)
89
+ hf_quant_config = getattr(model_config.hf_config, "quantization_config",
90
+ None)
91
if hf_quant_config is not None:
92
return quant_cls.from_config(hf_quant_config)
97
-
93
+ model_name_or_path = model_config.model
94
is_local = os.path.isdir(model_name_or_path)
95
if not is_local:
96
# Download the config files.
101
- with get_lock(model_name_or_path, cache_dir):
97
+ with get_lock(model_name_or_path, model_config.download_dir):
98
hf_folder = snapshot_download(model_name_or_path,
99
+ revision=model_config.revision,
100
allow_patterns="*.json",
104
- cache_dir=cache_dir,
101
+ cache_dir=model_config.download_dir,
102
tqdm_class=Disabledtqdm)
103
else:
104
hf_folder = model_name_or_path
@@ -112,10 +109,12 @@ def get_quant_config(
109
f.endswith(x) for x in quant_cls.get_config_filenames())
110
]
111
if len(quant_config_files) == 0:
115
- raise ValueError(f"Cannot find the config file for {quantization}")
112
+ raise ValueError(
113
+ f"Cannot find the config file for {model_config.quantization}")
114
if len(quant_config_files) > 1:
117
- raise ValueError(f"Found multiple config files for {quantization}: "
118
- f"{quant_config_files}")
115
+ raise ValueError(
116
+ f"Found multiple config files for {model_config.quantization}: "
117
+ f"{quant_config_files}")
118
119
quant_config_file = quant_config_files[0]
120
with open(quant_config_file, "r") as f: