A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] bitsandbytes models fail to run pipeline parallel (#10200)
Signed-off-by: Hoang Cong Duc <hoangcongducltt@gmail.com>
HoangCongDuc committed
Nov 14, 2024 at 00:56 UTC
ac49b59d8b01ffb9979e18e67b252d45410bc1e6
2 files changed
+35
-1
tests/quantization/test_bitsandbytes.py
+29
-1
@@ -9,7 +9,7 @@ import pytest
9
import torch
10
11
from tests.quantization.utils import is_quant_method_supported
12
-from tests.utils import fork_new_process_for_each_test
12
+from tests.utils import compare_two_settings, fork_new_process_for_each_test
13
14
models_4bit_to_test = [
15
("facebook/opt-125m", "quantize opt model inflight"),
@@ -82,6 +82,34 @@ def test_load_tp_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
82
vllm_tp_size=2)
83
84
85
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
86
+ reason='Test requires at least 2 GPUs.')
87
+@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
88
+ reason='bitsandbytes is not supported on this GPU type.')
89
+@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
90
+@fork_new_process_for_each_test
91
+def test_load_pp_4bit_bnb_model(model_name, description) -> None:
92
+ common_args = [
93
+ "--disable-log-stats",
94
+ "--disable-log-requests",
95
+ "--dtype",
96
+ "bfloat16",
97
+ "--enable-prefix-caching",
98
+ "--quantization",
99
+ "bitsandbytes",
100
+ "--load-format",
101
+ "bitsandbytes",
102
+ "--gpu-memory-utilization",
103
+ "0.7",
104
+ ]
105
+ pp_args = [
106
+ *common_args,
107
+ "--pipeline-parallel-size",
108
+ "2",
109
+ ]
110
+ compare_two_settings(model_name, common_args, pp_args)
111
+
112
+
113
def log_generated_texts(prompts, outputs, runner_name):
114
logged_texts = []
115
for i, (_, generated_text) in enumerate(outputs):
vllm/model_executor/model_loader/loader.py
+6
@@ -991,7 +991,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
991
992
param_dict = dict(model.named_parameters())
993
stacked_quant_state_dict: Dict[str, Dict[int, Any]] = {}
994
+ # TODO: Change this lazy import to normal import
995
+ # after the checks are updated to run on a new version
996
+ from vllm.model_executor.models.utils import is_pp_missing_parameter
997
for quant_param_name in quant_state_dict:
998
+ if is_pp_missing_parameter(quant_param_name, model):
999
+ continue
1000
+
1001
non_stacked_param_name = quant_param_name
1002
1003
shard_index = 0