@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c410f5d020
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Use revision when downloading the quantization config file (#2697)

Co-authored-by: Pernekhan Utemuratov <pernekhan@deepinfra.com>

Pernekhan Utemuratov committed Feb 1, 2024 at 15:41 UTC c410f5d020216df2dfedde52bcae24ae7f0ac7ec
2 files changed +15 -19
vllm/model_executor/model_loader.py
+1 -4
@@ -44,10 +44,7 @@ def get_model(model_config: ModelConfig,
44 # Get the (maybe quantized) linear method.
45 linear_method = None
46 if model_config.quantization is not None:
47 - quant_config = get_quant_config(model_config.quantization,
48 - model_config.model,
49 - model_config.hf_config,
50 - model_config.download_dir)
47 + quant_config = get_quant_config(model_config)
48 capability = torch.cuda.get_device_capability()
49 capability = capability[0] * 10 + capability[1]
50 if capability < quant_config.get_min_capability():
vllm/model_executor/weight_utils.py
+14 -15
@@ -11,9 +11,9 @@ from huggingface_hub import snapshot_download, HfFileSystem
11 import numpy as np
12 from safetensors.torch import load_file, save_file, safe_open
13 import torch
14 -from transformers import PretrainedConfig
14 from tqdm.auto import tqdm
15
16 +from vllm.config import ModelConfig
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.quantization import (get_quantization_config,
19 QuantizationConfig)
@@ -83,25 +83,22 @@ def convert_bin_to_safetensor_file(
83
84
85 # TODO(woosuk): Move this to other place.
86 -def get_quant_config(
87 - quantization: str,
88 - model_name_or_path: str,
89 - hf_config: PretrainedConfig,
90 - cache_dir: Optional[str] = None,
91 -) -> QuantizationConfig:
92 - quant_cls = get_quantization_config(quantization)
86 +def get_quant_config(model_config: ModelConfig) -> QuantizationConfig:
87 + quant_cls = get_quantization_config(model_config.quantization)
88 # Read the quantization config from the HF model config, if available.
94 - hf_quant_config = getattr(hf_config, "quantization_config", None)
89 + hf_quant_config = getattr(model_config.hf_config, "quantization_config",
90 + None)
91 if hf_quant_config is not None:
92 return quant_cls.from_config(hf_quant_config)
97 -
93 + model_name_or_path = model_config.model
94 is_local = os.path.isdir(model_name_or_path)
95 if not is_local:
96 # Download the config files.
101 - with get_lock(model_name_or_path, cache_dir):
97 + with get_lock(model_name_or_path, model_config.download_dir):
98 hf_folder = snapshot_download(model_name_or_path,
99 + revision=model_config.revision,
100 allow_patterns="*.json",
104 - cache_dir=cache_dir,
101 + cache_dir=model_config.download_dir,
102 tqdm_class=Disabledtqdm)
103 else:
104 hf_folder = model_name_or_path
@@ -112,10 +109,12 @@ def get_quant_config(
109 f.endswith(x) for x in quant_cls.get_config_filenames())
110 ]
111 if len(quant_config_files) == 0:
115 - raise ValueError(f"Cannot find the config file for {quantization}")
112 + raise ValueError(
113 + f"Cannot find the config file for {model_config.quantization}")
114 if len(quant_config_files) > 1:
117 - raise ValueError(f"Found multiple config files for {quantization}: "
118 - f"{quant_config_files}")
115 + raise ValueError(
116 + f"Found multiple config files for {model_config.quantization}: "
117 + f"{quant_config_files}")
118
119 quant_config_file = quant_config_files[0]
120 with open(quant_config_file, "r") as f: