@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ac49b59d8b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] bitsandbytes models fail to run pipeline parallel (#10200)

Signed-off-by: Hoang Cong Duc <hoangcongducltt@gmail.com>

HoangCongDuc committed Nov 14, 2024 at 00:56 UTC ac49b59d8b01ffb9979e18e67b252d45410bc1e6
2 files changed +35 -1
tests/quantization/test_bitsandbytes.py
+29 -1
@@ -9,7 +9,7 @@ import pytest
9 import torch
10
11 from tests.quantization.utils import is_quant_method_supported
12 -from tests.utils import fork_new_process_for_each_test
12 +from tests.utils import compare_two_settings, fork_new_process_for_each_test
13
14 models_4bit_to_test = [
15 ("facebook/opt-125m", "quantize opt model inflight"),
@@ -82,6 +82,34 @@ def test_load_tp_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,
82 vllm_tp_size=2)
83
84
85 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
86 + reason='Test requires at least 2 GPUs.')
87 +@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
88 + reason='bitsandbytes is not supported on this GPU type.')
89 +@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
90 +@fork_new_process_for_each_test
91 +def test_load_pp_4bit_bnb_model(model_name, description) -> None:
92 + common_args = [
93 + "--disable-log-stats",
94 + "--disable-log-requests",
95 + "--dtype",
96 + "bfloat16",
97 + "--enable-prefix-caching",
98 + "--quantization",
99 + "bitsandbytes",
100 + "--load-format",
101 + "bitsandbytes",
102 + "--gpu-memory-utilization",
103 + "0.7",
104 + ]
105 + pp_args = [
106 + *common_args,
107 + "--pipeline-parallel-size",
108 + "2",
109 + ]
110 + compare_two_settings(model_name, common_args, pp_args)
111 +
112 +
113 def log_generated_texts(prompts, outputs, runner_name):
114 logged_texts = []
115 for i, (_, generated_text) in enumerate(outputs):
vllm/model_executor/model_loader/loader.py
+6
@@ -991,7 +991,13 @@ class BitsAndBytesModelLoader(BaseModelLoader):
991
992 param_dict = dict(model.named_parameters())
993 stacked_quant_state_dict: Dict[str, Dict[int, Any]] = {}
994 + # TODO: Change this lazy import to normal import
995 + # after the checks are updated to run on a new version
996 + from vllm.model_executor.models.utils import is_pp_missing_parameter
997 for quant_param_name in quant_state_dict:
998 + if is_pp_missing_parameter(quant_param_name, model):
999 + continue
1000 +
1001 non_stacked_param_name = quant_param_name
1002
1003 shard_index = 0