@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d72bb20fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[4/N] make quant config first-class citizen (#9978)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 4, 2024 at 08:51 UTC 8d72bb20fae1a8a9d6ec6dcb2a833a190e1225d3
2 files changed +41 -31
vllm/config.py
+38
@@ -23,9 +23,13 @@ if TYPE_CHECKING:
23 from ray.util.placement_group import PlacementGroup
24
25 from vllm.executor.executor_base import ExecutorBase
26 + from vllm.model_executor.layers.quantization.base_config import (
27 + QuantizationConfig)
28 from vllm.model_executor.model_loader.loader import BaseModelLoader
29 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
30 BaseTokenizerGroup)
31 +else:
32 + QuantizationConfig = None
33
34 logger = init_logger(__name__)
35
@@ -1966,6 +1970,35 @@ class VllmConfig:
1970 decoding_config: Optional[DecodingConfig] = None
1971 observability_config: Optional[ObservabilityConfig] = None
1972 prompt_adapter_config: Optional[PromptAdapterConfig] = None
1973 + quant_config: Optional[QuantizationConfig] = None
1974 +
1975 + @staticmethod
1976 + def _get_quantization_config(
1977 + model_config: ModelConfig,
1978 + load_config: LoadConfig) -> Optional[QuantizationConfig]:
1979 + """Get the quantization config."""
1980 + if model_config.quantization is not None:
1981 + from vllm.model_executor.model_loader.weight_utils import (
1982 + get_quant_config)
1983 + quant_config = get_quant_config(model_config, load_config)
1984 + capability_tuple = current_platform.get_device_capability()
1985 +
1986 + if capability_tuple is not None:
1987 + capability = capability_tuple.to_int()
1988 + if capability < quant_config.get_min_capability():
1989 + raise ValueError(
1990 + f"The quantization method {model_config.quantization} "
1991 + "is not supported for the current GPU. Minimum "
1992 + f"capability: {quant_config.get_min_capability()}. "
1993 + f"Current capability: {capability}.")
1994 + supported_dtypes = quant_config.get_supported_act_dtypes()
1995 + if model_config.dtype not in supported_dtypes:
1996 + raise ValueError(
1997 + f"{model_config.dtype} is not supported for quantization "
1998 + f"method {model_config.quantization}. Supported dtypes: "
1999 + f"{supported_dtypes}")
2000 + return quant_config
2001 + return None
2002
2003 def __post_init__(self):
2004 """Verify configs are valid & consistent with each other.
@@ -1983,3 +2016,8 @@ class VllmConfig:
2016 if self.prompt_adapter_config:
2017 self.prompt_adapter_config.verify_with_model_config(
2018 self.model_config)
2019 +
2020 + if self.quant_config is None and \
2021 + self.model_config is not None and self.load_config is not None:
2022 + self.quant_config = VllmConfig._get_quantization_config(
2023 + self.model_config, self.load_config)
vllm/model_executor/model_loader/loader.py
+3 -31
@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.utils import (get_model_architecture,
39 from vllm.model_executor.model_loader.weight_utils import (
40 download_safetensors_index_file_from_hf, download_weights_from_hf,
41 filter_duplicate_safetensors_files, filter_files_not_needed_for_inference,
42 - get_gguf_extra_tensor_names, get_quant_config, gguf_quant_weights_iterator,
42 + get_gguf_extra_tensor_names, gguf_quant_weights_iterator,
43 initialize_dummy_weights, np_cache_weights_iterator, pt_weights_iterator,
44 safetensors_weights_iterator)
45 from vllm.model_executor.models import (has_inner_state, supports_lora,
@@ -93,32 +93,6 @@ def device_loading_context(module: torch.nn.Module,
93 logger = init_logger(__name__)
94
95
96 -def _get_quantization_config(
97 - model_config: ModelConfig,
98 - load_config: LoadConfig) -> Optional[QuantizationConfig]:
99 - """Get the quantization config."""
100 - if model_config.quantization is not None:
101 - quant_config = get_quant_config(model_config, load_config)
102 - capability_tuple = current_platform.get_device_capability()
103 -
104 - if capability_tuple is not None:
105 - capability = capability_tuple.to_int()
106 - if capability < quant_config.get_min_capability():
107 - raise ValueError(
108 - f"The quantization method {model_config.quantization} "
109 - "is not supported for the current GPU. "
110 - f"Minimum capability: {quant_config.get_min_capability()}. "
111 - f"Current capability: {capability}.")
112 - supported_dtypes = quant_config.get_supported_act_dtypes()
113 - if model_config.dtype not in supported_dtypes:
114 - raise ValueError(
115 - f"{model_config.dtype} is not supported for quantization "
116 - f"method {model_config.quantization}. Supported dtypes: "
117 - f"{supported_dtypes}")
118 - return quant_config
119 - return None
120 -
121 -
96 def _get_model_initialization_kwargs(
97 model_class: Type[nn.Module],
98 lora_config: Optional[LoRAConfig],
@@ -185,7 +159,6 @@ def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
159 lora_config = vllm_config.lora_config
160 scheduler_config = vllm_config.scheduler_config
161 cache_config = vllm_config.cache_config
188 - load_config = vllm_config.load_config
162 model_class, _ = get_model_architecture(model_config)
163
164 return build_model(
@@ -193,7 +166,7 @@ def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
166 vllm_config,
167 model_config.hf_config,
168 cache_config=cache_config,
196 - quant_config=_get_quantization_config(model_config, load_config),
169 + quant_config=vllm_config.quant_config,
170 lora_config=lora_config,
171 multimodal_config=model_config.multimodal_config,
172 scheduler_config=scheduler_config,
@@ -518,8 +491,7 @@ class TensorizerLoader(BaseModelLoader):
491 with set_default_torch_dtype(model_config.dtype):
492 with torch.device(device_config.device):
493 model_class = get_model_architecture(model_config)[0]
521 - quant_config = _get_quantization_config(
522 - model_config, self.load_config)
494 + quant_config = vllm_config.quant_config
495 extra_kwargs = _get_model_initialization_kwargs(
496 model_class, lora_config, model_config.multimodal_config)
497 extra_kwargs["quant_config"] = quant_config