@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1f958a7d52
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix BNB loader target_modules (#10720)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 5, 2024 at 13:20 UTC 1f958a7d52b24314e41c4bb56c51b1dce5405e05
1 file changed +6 -58
vllm/model_executor/model_loader/loader.py
+6 -58
@@ -6,7 +6,6 @@ import fnmatch
6 import glob
7 import inspect
8 import itertools
9 -import json
9 import math
10 import os
11 import warnings
@@ -18,7 +17,7 @@ import gguf
17 import huggingface_hub
18 import numpy as np
19 import torch
21 -from huggingface_hub import HfApi, hf_hub_download
20 +from huggingface_hub import HfApi
21 from torch import nn
22 from transformers import AutoModelForCausalLM
23 from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
@@ -704,51 +703,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
703 self.unsharded_weights_modules: List[str] = []
704 # Save the module names that are sharded by column.
705 self.column_sharded_weights_modules: List[str] = []
707 - # we don't need to quantize the whole model, only the target modules
708 - # that are specified in the adapter config file. If the adapter config
709 - # file is not provided, we will quantize the default modules.
710 - if (not load_config.model_loader_extra_config
711 - or "qlora_adapter_name_or_path"
712 - not in load_config.model_loader_extra_config):
713 - self.target_modules = []
714 - return
715 -
716 - qlora_adapter = load_config.model_loader_extra_config[
717 - "qlora_adapter_name_or_path"]
718 -
719 - config_file_path = self._get_config_file(qlora_adapter)
720 -
721 - with open(config_file_path) as f:
722 - config = json.load(f)
723 - self.target_modules = config["target_modules"]
724 - # TODO: target_modules could be either a list or a regex string.
725 - # We need to handle both cases.
726 - assert isinstance(self.target_modules,
727 - list), "Unsupported target_modules: "
728 - f"{self.target_modules}"
729 -
730 - def _get_config_file(self, qlora_adapter: str) -> str:
731 - is_local = os.path.isdir(qlora_adapter)
732 - config_file_path = None
733 - if is_local:
734 - for file in self.possible_config_file_names:
735 - config_file_path = os.path.join(qlora_adapter, file)
736 - if os.path.exists(config_file_path):
737 - break
738 - else:
739 - hf_api = HfApi()
740 - repo_files = hf_api.list_repo_files(repo_id=qlora_adapter)
741 - for file in self.possible_config_file_names:
742 - if file in repo_files:
743 - config_file_path = hf_hub_download(repo_id=qlora_adapter,
744 - filename=file)
745 - break
746 -
747 - if not config_file_path:
748 - raise ValueError(
749 - f"Cannot find adapter config file in {qlora_adapter}")
750 -
751 - return config_file_path
706 + # Store all module names (from transformers) that support
707 + # BNB quantization.
708 + self.target_modules: List[str] = []
709
710 def _get_weight_files(
711 self,
@@ -1030,25 +987,16 @@ class BitsAndBytesModelLoader(BaseModelLoader):
987 inverse_stacked_mapping[packed] = []
988 inverse_stacked_mapping[packed].insert(idx, orig)
989
1033 - linear_module_lst = []
990 for name, module in model.named_modules():
991 if isinstance(module, (LinearBase, )):
992 last_name = name.split(".")[-1]
993 if sub_modules := inverse_stacked_mapping.get(last_name, []):
994 # Map vllm's names to transformers' names.
995 for sub_name in sub_modules:
1040 - linear_module_lst.append(
996 + self.target_modules.append(
997 name.replace(last_name, sub_name))
998 else:
1043 - linear_module_lst.append(name)
1044 - if self.target_modules:
1045 - # Update self.target_modules
1046 - self.target_modules = [
1047 - qual_name for qual_name in linear_module_lst
1048 - if any(t in qual_name for t in self.target_modules)
1049 - ]
1050 - else:
1051 - self.target_modules = linear_module_lst
999 + self.target_modules.append(name)
1000 assert (self.target_modules
1001 ), "vllm currently does not support BNB quantization for"
1002 f" {type(model).__name__}"