@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 13f9f7a3d0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[[Misc]Upgrade bitsandbytes to the latest version 0.44.0 (#8768)

Jee Jee Li committed Sep 25, 2024 at 08:08 UTC 13f9f7a3d0373421ee9fd7498e450214e134aa6c
7 files changed +44 -34
docs/source/quantization/bnb.rst
+1 -1
@@ -11,7 +11,7 @@ Below are the steps to utilize BitsAndBytes with vLLM.
11
12 .. code-block:: console
13
14 - $ pip install bitsandbytes>=0.42.0
14 + $ pip install bitsandbytes>=0.44.0
15
16 vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint.
17
examples/lora_with_quantization_inference.py
+10 -16
@@ -79,23 +79,17 @@ def initialize_engine(model: str, quantization: str,
79 # It quantizes the model when loading, with some config info from the
80 # LoRA adapter repo. So need to set the parameter of load_format and
81 # qlora_adapter_name_or_path as below.
82 - engine_args = EngineArgs(
83 - model=model,
84 - quantization=quantization,
85 - qlora_adapter_name_or_path=lora_repo,
86 - load_format="bitsandbytes",
87 - enable_lora=True,
88 - max_lora_rank=64,
89 - # set it only in GPUs of limited memory
90 - enforce_eager=True)
82 + engine_args = EngineArgs(model=model,
83 + quantization=quantization,
84 + qlora_adapter_name_or_path=lora_repo,
85 + load_format="bitsandbytes",
86 + enable_lora=True,
87 + max_lora_rank=64)
88 else:
92 - engine_args = EngineArgs(
93 - model=model,
94 - quantization=quantization,
95 - enable_lora=True,
96 - max_loras=4,
97 - # set it only in GPUs of limited memory
98 - enforce_eager=True)
89 + engine_args = EngineArgs(model=model,
90 + quantization=quantization,
91 + enable_lora=True,
92 + max_loras=4)
93 return LLMEngine.from_engine_args(engine_args)
94
95
requirements-test.txt
+1 -1
@@ -30,5 +30,5 @@ datamodel_code_generator # required for minicpm3 test
30 aiohttp
31
32 # quantization
33 -bitsandbytes==0.42.0
33 +bitsandbytes>=0.44.0
34 buildkite-test-collector==0.1.8
tests/quantization/test_bitsandbytes.py
+1 -1
@@ -107,7 +107,7 @@ def validate_generated_texts(hf_runner,
107 quantization='bitsandbytes',
108 load_format='bitsandbytes',
109 tensor_parallel_size=vllm_tp_size,
110 - enforce_eager=True,
110 + enforce_eager=False,
111 gpu_memory_utilization=0.8) as llm:
112 vllm_outputs = llm.generate_greedy(prompts, 8)
113 vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
vllm/config.py
+23 -7
@@ -222,6 +222,7 @@ class ModelConfig:
222 self._verify_embedding_mode()
223 self._verify_quantization()
224 self._verify_cuda_graph()
225 + self._verify_bnb_config()
226
227 def _init_multimodal_config(
228 self, limit_mm_per_prompt: Optional[Mapping[str, int]]
@@ -337,6 +338,28 @@ class ModelConfig:
338 self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
339 self.max_model_len)
340
341 + def _verify_bnb_config(self) -> None:
342 + """
343 + The current version of bitsandbytes (0.44.0) with 8-bit models does not
344 + yet support CUDA graph.
345 + """
346 + is_bitsandbytes = self.quantization == "bitsandbytes"
347 + has_quantization_config = (getattr(self.hf_config,
348 + "quantization_config", None)
349 + is not None)
350 + is_8bit = (self.hf_config.quantization_config.get(
351 + "load_in_8bit", False) if has_quantization_config else False)
352 + if all([
353 + is_bitsandbytes,
354 + has_quantization_config,
355 + is_8bit,
356 + not self.enforce_eager,
357 + ]):
358 + logger.warning(
359 + "CUDA graph is not supported on BitAndBytes 8bit yet, "
360 + "fallback to the eager mode.")
361 + self.enforce_eager = True
362 +
363 def verify_async_output_proc(self, parallel_config, speculative_config,
364 device_config) -> None:
365 if not self.use_async_output_proc:
@@ -401,13 +424,6 @@ class ModelConfig:
424 "Pipeline parallelism is only supported for the following "
425 f" architectures: {_PP_SUPPORTED_MODELS}.")
426
404 - # Remove the constraint after the bitsandbytes issue is fixed:
405 - # https://github.com/bitsandbytes-foundation/bitsandbytes/issues/1308
406 - if self.quantization == "bitsandbytes" and self.enforce_eager is False:
407 - logger.warning("CUDA graph is not supported on BitAndBytes yet, "
408 - "fallback to the eager mode.")
409 - self.enforce_eager = True
410 -
427 if pipeline_parallel_size > 1 and self.use_async_output_proc:
428 logger.warning("Async output processor is not supported with "
429 "pipeline parallelism currently. Disabling it.")
vllm/model_executor/layers/quantization/bitsandbytes.py
+4 -4
@@ -121,12 +121,12 @@ class BitsAndBytesLinearMethod(LinearMethodBase):
121 def __init__(self, quant_config: BitsAndBytesConfig):
122 try:
123 import bitsandbytes
124 - if bitsandbytes.__version__ < "0.42.0":
124 + if bitsandbytes.__version__ < "0.44.0":
125 raise ImportError("bitsandbytes version is wrong. Please "
126 - "install bitsandbytes>=0.42.0.")
126 + "install bitsandbytes>=0.44.0.")
127 except ImportError as err:
128 - raise ImportError("Please install bitsandbytes>=0.42.0 via "
129 - "`pip install bitsandbytes>=0.42.0` to use "
128 + raise ImportError("Please install bitsandbytes>=0.44.0 via "
129 + "`pip install bitsandbytes>=0.44.0` to use "
130 "bitsandbytes quantizer.") from err
131
132 self.quant_config = quant_config
vllm/model_executor/model_loader/loader.py
+4 -4
@@ -851,12 +851,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
851 # only load the bitsandbytes module when needed
852 try:
853 import bitsandbytes
854 - if bitsandbytes.__version__ < "0.42.0":
854 + if bitsandbytes.__version__ < "0.44.0":
855 raise ImportError("bitsandbytes version is wrong. Please "
856 - "install bitsandbytes>=0.42.0.")
856 + "install bitsandbytes>=0.44.0.")
857 except ImportError as err:
858 - raise ImportError("Please install bitsandbytes>=0.42.0 via "
859 - "`pip install bitsandbytes>=0.42.0` to use "
858 + raise ImportError("Please install bitsandbytes>=0.44.0 via "
859 + "`pip install bitsandbytes>=0.44.0` to use "
860 "bitsandbytes quantizer.") from err
861
862 hf_weights_files, use_safetensors = self._prepare_weights(