@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b6374e09b0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Fix Phi-3 BNB quantization with tensor parallel (#9948)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 22, 2024 at 15:01 UTC b6374e09b0af4f8fa4c0b911b3cd1bd45342ead6
2 files changed +56 -6
vllm/model_executor/layers/linear.py
+14 -5
@@ -1,3 +1,4 @@
1 +import itertools
2 from abc import abstractmethod
3 from typing import Dict, List, Optional, Tuple
4
@@ -41,12 +42,12 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
42
43
44 def adjust_bitsandbytes_4bit_shard(param: Parameter,
44 - qkv_offsets: Dict[str, Tuple[int, int]],
45 + shard_offsets: Dict[str, Tuple[int, int]],
46 loaded_shard_id: str) -> Tuple[int, int]:
47 """Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
48
48 - total, _ = qkv_offsets["total"]
49 - orig_offset, orig_size = qkv_offsets[loaded_shard_id]
49 + total, _ = shard_offsets["total"]
50 + orig_offset, orig_size = shard_offsets[loaded_shard_id]
51
52 quantized_total = param.data.shape[0]
53 quantized_offset = orig_offset * quantized_total // total
@@ -499,9 +500,17 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
500 # Special case for Marlin.
501 shard_size, shard_offset = adjust_marlin_shard(
502 param, shard_size, shard_offset)
503 +
504 if use_bitsandbytes_4bit:
503 - shard_size = loaded_weight.shape[output_dim] // 2
504 - shard_offset = shard_size * shard_id
505 + index = list(itertools.accumulate([0] + self.output_sizes))
506 + orig_offsets = {
507 + str(i): (index[i], size)
508 + for i, size in enumerate(self.output_sizes)
509 + }
510 + orig_offsets["total"] = (self.output_size, 0)
511 + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(
512 + param, orig_offsets, str(shard_id))
513 +
514 loaded_weight_shard = loaded_weight.narrow(
515 output_dim, shard_offset, shard_size)
516 self.weight_loader(param, loaded_weight_shard, shard_id)
vllm/model_executor/model_loader/loader.py
+42 -1
@@ -5,6 +5,7 @@ import dataclasses
5 import fnmatch
6 import glob
7 import inspect
8 +import itertools
9 import json
10 import math
11 import os
@@ -27,7 +28,9 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.envs import VLLM_USE_MODELSCOPE
30 from vllm.logger import init_logger
30 -from vllm.model_executor.layers.linear import (ReplicatedLinear,
31 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
32 + QKVParallelLinear,
33 + ReplicatedLinear,
34 RowParallelLinear)
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizeMethodBase)
@@ -936,6 +939,34 @@ class BitsAndBytesModelLoader(BaseModelLoader):
939 end_index = total_size // tp_size * (tp_rank + 1)
940 weight_sub_tensor = weight_tensor[...,
941 start_index:end_index]
942 + # Weights have fused on disk. In this case, we assume that the
943 + # weight and module use same name.
944 + elif any(
945 + weight_name.startswith(module)
946 + for module in self.maybe_fused_weights_modules):
947 + # special case for fused weights
948 + # get the size of each shard weight tensor
949 + total_shard_sizes = next(
950 + (sizes for module, sizes in
951 + self.maybe_fused_weights_modules.items()
952 + if weight_name.startswith(module)))
953 + total_size = weight_tensor.size(0)
954 + assert total_size == sum(total_shard_sizes)
955 + # get the start/end index of each shard weight tensor
956 + total_start_index = list(
957 + itertools.accumulate([0] + total_shard_sizes))[:-1]
958 + shard_weights_index = [
959 + (idx + size // tp_size * tp_rank,
960 + idx + size // tp_size * (tp_rank + 1))
961 + for idx, size in zip(total_start_index,
962 + total_shard_sizes)
963 + ]
964 + # slice and reorder the weight tensor
965 + weight_tensor = [
966 + weight_tensor[start_index:end_index, ...]
967 + for start_index, end_index in shard_weights_index
968 + ]
969 + weight_sub_tensor = torch.cat(weight_tensor, dim=0)
970 # Shard by row
971 else:
972 total_size = weight_tensor.size(0)
@@ -985,12 +1016,22 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1016 else:
1017 self.target_modules = self.default_target_modules
1018
1019 + # Modules whose weights might have fused on disk
1020 + # we need their output_sizes to make shard in flight correctly with TP
1021 + self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
1022 +
1023 for name, module in model.named_modules():
1024 # Some modules like `ReplicatedLinear` should not have their weights
1025 # sharded. The reason for implementing it this way is to avoid new
1026 # static variable in the model implementation.
1027 if isinstance(module, (ReplicatedLinear, )):
1028 self.unsharded_weights_modules.append(name)
1029 + # `QKVParallelLinear` and `MergedColumnParallelLinear` might have
1030 + # fused weights on disk. We need to use the output sizes of these
1031 + # modules to shard the weights correctly.
1032 + elif isinstance(module,
1033 + (QKVParallelLinear, MergedColumnParallelLinear)):
1034 + self.maybe_fused_weights_modules[name] = module.output_sizes
1035 # In TP, these weights are partitioned along the column
1036 # dimension (dim=-1)
1037 elif isinstance(module, (RowParallelLinear, )):