A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[4/N] make quant config first-class citizen (#9978)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 4, 2024 at 08:51 UTC
8d72bb20fae1a8a9d6ec6dcb2a833a190e1225d3
2 files changed
+41
-31
vllm/config.py
+38
@@ -23,9 +23,13 @@ if TYPE_CHECKING:
23
from ray.util.placement_group import PlacementGroup
24
25
from vllm.executor.executor_base import ExecutorBase
26
+ from vllm.model_executor.layers.quantization.base_config import (
27
+ QuantizationConfig)
28
from vllm.model_executor.model_loader.loader import BaseModelLoader
29
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
30
BaseTokenizerGroup)
31
+else:
32
+ QuantizationConfig = None
33
34
logger = init_logger(__name__)
35
@@ -1966,6 +1970,35 @@ class VllmConfig:
1970
decoding_config: Optional[DecodingConfig] = None
1971
observability_config: Optional[ObservabilityConfig] = None
1972
prompt_adapter_config: Optional[PromptAdapterConfig] = None
1973
+ quant_config: Optional[QuantizationConfig] = None
1974
+
1975
+ @staticmethod
1976
+ def _get_quantization_config(
1977
+ model_config: ModelConfig,
1978
+ load_config: LoadConfig) -> Optional[QuantizationConfig]:
1979
+ """Get the quantization config."""
1980
+ if model_config.quantization is not None:
1981
+ from vllm.model_executor.model_loader.weight_utils import (
1982
+ get_quant_config)
1983
+ quant_config = get_quant_config(model_config, load_config)
1984
+ capability_tuple = current_platform.get_device_capability()
1985
+
1986
+ if capability_tuple is not None:
1987
+ capability = capability_tuple.to_int()
1988
+ if capability < quant_config.get_min_capability():
1989
+ raise ValueError(
1990
+ f"The quantization method {model_config.quantization} "
1991
+ "is not supported for the current GPU. Minimum "
1992
+ f"capability: {quant_config.get_min_capability()}. "
1993
+ f"Current capability: {capability}.")
1994
+ supported_dtypes = quant_config.get_supported_act_dtypes()
1995
+ if model_config.dtype not in supported_dtypes:
1996
+ raise ValueError(
1997
+ f"{model_config.dtype} is not supported for quantization "
1998
+ f"method {model_config.quantization}. Supported dtypes: "
1999
+ f"{supported_dtypes}")
2000
+ return quant_config
2001
+ return None
2002
2003
def __post_init__(self):
2004
"""Verify configs are valid & consistent with each other.
@@ -1983,3 +2016,8 @@ class VllmConfig:
2016
if self.prompt_adapter_config:
2017
self.prompt_adapter_config.verify_with_model_config(
2018
self.model_config)
2019
+
2020
+ if self.quant_config is None and \
2021
+ self.model_config is not None and self.load_config is not None:
2022
+ self.quant_config = VllmConfig._get_quantization_config(
2023
+ self.model_config, self.load_config)
vllm/model_executor/model_loader/loader.py
+3
-31
@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.utils import (get_model_architecture,
39
from vllm.model_executor.model_loader.weight_utils import (
40
download_safetensors_index_file_from_hf, download_weights_from_hf,
41
filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
42
- get_gguf_extra_tensor_names, get_quant_config, gguf_quant_weights_iterator,
42
+ get_gguf_extra_tensor_names, gguf_quant_weights_iterator,
43
initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
44
safetensors_weights_iterator)
45
from vllm.model_executor.models import (has_inner_state, supports_lora,
@@ -93,32 +93,6 @@ def device_loading_context(module: torch.nn.Module,
93
logger = init_logger(__name__)
94
95
96
-def _get_quantization_config(
97
- model_config: ModelConfig,
98
- load_config: LoadConfig) -> Optional[QuantizationConfig]:
99
- """Get the quantization config."""
100
- if model_config.quantization is not None:
101
- quant_config = get_quant_config(model_config, load_config)
102
- capability_tuple = current_platform.get_device_capability()
103
-
104
- if capability_tuple is not None:
105
- capability = capability_tuple.to_int()
106
- if capability < quant_config.get_min_capability():
107
- raise ValueError(
108
- f"The quantization method {model_config.quantization} "
109
- "is not supported for the current GPU. "
110
- f"Minimum capability: {quant_config.get_min_capability()}. "
111
- f"Current capability: {capability}.")
112
- supported_dtypes = quant_config.get_supported_act_dtypes()
113
- if model_config.dtype not in supported_dtypes:
114
- raise ValueError(
115
- f"{model_config.dtype} is not supported for quantization "
116
- f"method {model_config.quantization}. Supported dtypes: "
117
- f"{supported_dtypes}")
118
- return quant_config
119
- return None
120
-
121
-
96
def _get_model_initialization_kwargs(
97
model_class: Type[nn.Module],
98
lora_config: Optional[LoRAConfig],
@@ -185,7 +159,6 @@ def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
159
lora_config = vllm_config.lora_config
160
scheduler_config = vllm_config.scheduler_config
161
cache_config = vllm_config.cache_config
188
- load_config = vllm_config.load_config
162
model_class, _ = get_model_architecture(model_config)
163
164
return build_model(
@@ -193,7 +166,7 @@ def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
166
vllm_config,
167
model_config.hf_config,
168
cache_config=cache_config,
196
- quant_config=_get_quantization_config(model_config, load_config),
169
+ quant_config=vllm_config.quant_config,
170
lora_config=lora_config,
171
multimodal_config=model_config.multimodal_config,
172
scheduler_config=scheduler_config,
@@ -518,8 +491,7 @@ class TensorizerLoader(BaseModelLoader):
491
with set_default_torch_dtype(model_config.dtype):
492
with torch.device(device_config.device):
493
model_class = get_model_architecture(model_config)[0]
521
- quant_config = _get_quantization_config(
522
- model_config, self.load_config)
494
+ quant_config = vllm_config.quant_config
495
extra_kwargs = _get_model_initialization_kwargs(
496
model_class, lora_config, model_config.multimodal_config)
497
extra_kwargs["quant_config"] = quant_config