A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix BNB loader target_modules (#10720)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 5, 2024 at 13:20 UTC
1f958a7d52b24314e41c4bb56c51b1dce5405e05
1 file changed
+6
-58
vllm/model_executor/model_loader/loader.py
+6
-58
@@ -6,7 +6,6 @@ import fnmatch
6
import glob
7
import inspect
8
import itertools
9
-import json
9
import math
10
import os
11
import warnings
@@ -18,7 +17,7 @@ import gguf
17
import huggingface_hub
18
import numpy as np
19
import torch
21
-from huggingface_hub import HfApi, hf_hub_download
20
+from huggingface_hub import HfApi
21
from torch import nn
22
from transformers import AutoModelForCausalLM
23
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
@@ -704,51 +703,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
703
self.unsharded_weights_modules: List[str] = []
704
# Save the module names that are sharded by column.
705
self.column_sharded_weights_modules: List[str] = []
707
- # we don't need to quantize the whole model, only the target modules
708
- # that are specified in the adapter config file. If the adapter config
709
- # file is not provided, we will quantize the default modules.
710
- if (not load_config.model_loader_extra_config
711
- or "qlora_adapter_name_or_path"
712
- not in load_config.model_loader_extra_config):
713
- self.target_modules = []
714
- return
715
-
716
- qlora_adapter = load_config.model_loader_extra_config[
717
- "qlora_adapter_name_or_path"]
718
-
719
- config_file_path = self._get_config_file(qlora_adapter)
720
-
721
- with open(config_file_path) as f:
722
- config = json.load(f)
723
- self.target_modules = config["target_modules"]
724
- # TODO: target_modules could be either a list or a regex string.
725
- # We need to handle both cases.
726
- assert isinstance(self.target_modules,
727
- list), "Unsupported target_modules: "
728
- f"{self.target_modules}"
729
-
730
- def _get_config_file(self, qlora_adapter: str) -> str:
731
- is_local = os.path.isdir(qlora_adapter)
732
- config_file_path = None
733
- if is_local:
734
- for file in self.possible_config_file_names:
735
- config_file_path = os.path.join(qlora_adapter, file)
736
- if os.path.exists(config_file_path):
737
- break
738
- else:
739
- hf_api = HfApi()
740
- repo_files = hf_api.list_repo_files(repo_id=qlora_adapter)
741
- for file in self.possible_config_file_names:
742
- if file in repo_files:
743
- config_file_path = hf_hub_download(repo_id=qlora_adapter,
744
- filename=file)
745
- break
746
-
747
- if not config_file_path:
748
- raise ValueError(
749
- f"Cannot find adapter config file in {qlora_adapter}")
750
-
751
- return config_file_path
706
+ # Store all module names (from transformers) that support
707
+ # BNB quantization.
708
+ self.target_modules: List[str] = []
709
710
def _get_weight_files(
711
self,
@@ -1030,25 +987,16 @@ class BitsAndBytesModelLoader(BaseModelLoader):
987
inverse_stacked_mapping[packed] = []
988
inverse_stacked_mapping[packed].insert(idx, orig)
989
1033
- linear_module_lst = []
990
for name, module in model.named_modules():
991
if isinstance(module, (LinearBase, )):
992
last_name = name.split(".")[-1]
993
if sub_modules := inverse_stacked_mapping.get(last_name, []):
994
# Map vllm's names to transformers' names.
995
for sub_name in sub_modules:
1040
- linear_module_lst.append(
996
+ self.target_modules.append(
997
name.replace(last_name, sub_name))
998
else:
1043
- linear_module_lst.append(name)
1044
- if self.target_modules:
1045
- # Update self.target_modules
1046
- self.target_modules = [
1047
- qual_name for qual_name in linear_module_lst
1048
- if any(t in qual_name for t in self.target_modules)
1049
- ]
1050
- else:
1051
- self.target_modules = linear_module_lst
999
+ self.target_modules.append(name)
1000
assert (self.target_modules
1001
), "vllm currently does not support BNB quantization for"
1002
f" {type(model).__name__}"