A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Fix Phi-3 BNB quantization with tensor parallel (#9948)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 22, 2024 at 15:01 UTC
b6374e09b0af4f8fa4c0b911b3cd1bd45342ead6
2 files changed
+56
-6
vllm/model_executor/layers/linear.py
+14
-5
@@ -1,3 +1,4 @@
1
+import itertools
2
from abc import abstractmethod
3
from typing import Dict, List, Optional, Tuple
4
@@ -41,12 +42,12 @@ def adjust_marlin_shard(param, shard_size, shard_offset):
42
43
44
def adjust_bitsandbytes_4bit_shard(param: Parameter,
44
- qkv_offsets: Dict[str, Tuple[int, int]],
45
+ shard_offsets: Dict[str, Tuple[int, int]],
46
loaded_shard_id: str) -> Tuple[int, int]:
47
"""Adjust the quantization offsets and sizes for BitsAndBytes sharding."""
48
48
- total, _ = qkv_offsets["total"]
49
- orig_offset, orig_size = qkv_offsets[loaded_shard_id]
49
+ total, _ = shard_offsets["total"]
50
+ orig_offset, orig_size = shard_offsets[loaded_shard_id]
51
52
quantized_total = param.data.shape[0]
53
quantized_offset = orig_offset * quantized_total // total
@@ -499,9 +500,17 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
500
# Special case for Marlin.
501
shard_size, shard_offset = adjust_marlin_shard(
502
param, shard_size, shard_offset)
503
+
504
if use_bitsandbytes_4bit:
503
- shard_size = loaded_weight.shape[output_dim] // 2
504
- shard_offset = shard_size * shard_id
505
+ index = list(itertools.accumulate([0] + self.output_sizes))
506
+ orig_offsets = {
507
+ str(i): (index[i], size)
508
+ for i, size in enumerate(self.output_sizes)
509
+ }
510
+ orig_offsets["total"] = (self.output_size, 0)
511
+ shard_size, shard_offset = adjust_bitsandbytes_4bit_shard(
512
+ param, orig_offsets, str(shard_id))
513
+
514
loaded_weight_shard = loaded_weight.narrow(
515
output_dim, shard_offset, shard_size)
516
self.weight_loader(param, loaded_weight_shard, shard_id)
vllm/model_executor/model_loader/loader.py
+42
-1
@@ -5,6 +5,7 @@ import dataclasses
5
import fnmatch
6
import glob
7
import inspect
8
+import itertools
9
import json
10
import math
11
import os
@@ -27,7 +28,9 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
28
get_tensor_model_parallel_world_size)
29
from vllm.envs import VLLM_USE_MODELSCOPE
30
from vllm.logger import init_logger
30
-from vllm.model_executor.layers.linear import (ReplicatedLinear,
31
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
32
+ QKVParallelLinear,
33
+ ReplicatedLinear,
34
RowParallelLinear)
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizeMethodBase)
@@ -936,6 +939,34 @@ class BitsAndBytesModelLoader(BaseModelLoader):
939
end_index = total_size // tp_size * (tp_rank + 1)
940
weight_sub_tensor = weight_tensor[...,
941
start_index:end_index]
942
+ # Weights have fused on disk. In this case, we assume that the
943
+ # weight and module use same name.
944
+ elif any(
945
+ weight_name.startswith(module)
946
+ for module in self.maybe_fused_weights_modules):
947
+ # special case for fused weights
948
+ # get the size of each shard weight tensor
949
+ total_shard_sizes = next(
950
+ (sizes for module, sizes in
951
+ self.maybe_fused_weights_modules.items()
952
+ if weight_name.startswith(module)))
953
+ total_size = weight_tensor.size(0)
954
+ assert total_size == sum(total_shard_sizes)
955
+ # get the start/end index of each shard weight tensor
956
+ total_start_index = list(
957
+ itertools.accumulate([0] + total_shard_sizes))[:-1]
958
+ shard_weights_index = [
959
+ (idx + size // tp_size * tp_rank,
960
+ idx + size // tp_size * (tp_rank + 1))
961
+ for idx, size in zip(total_start_index,
962
+ total_shard_sizes)
963
+ ]
964
+ # slice and reorder the weight tensor
965
+ weight_tensor = [
966
+ weight_tensor[start_index:end_index, ...]
967
+ for start_index, end_index in shard_weights_index
968
+ ]
969
+ weight_sub_tensor = torch.cat(weight_tensor, dim=0)
970
# Shard by row
971
else:
972
total_size = weight_tensor.size(0)
@@ -985,12 +1016,22 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1016
else:
1017
self.target_modules = self.default_target_modules
1018
1019
+ # Modules whose weights might have fused on disk
1020
+ # we need their output_sizes to make shard in flight correctly with TP
1021
+ self.maybe_fused_weights_modules: Dict[str, List[int]] = {}
1022
+
1023
for name, module in model.named_modules():
1024
# Some modules like `ReplicatedLinear` should not have their weights
1025
# sharded. The reason for implementing it this way is to avoid new
1026
# static variable in the model implementation.
1027
if isinstance(module, (ReplicatedLinear, )):
1028
self.unsharded_weights_modules.append(name)
1029
+ # `QKVParallelLinear` and `MergedColumnParallelLinear` might have
1030
+ # fused weights on disk. We need to use the output sizes of these
1031
+ # modules to shard the weights correctly.
1032
+ elif isinstance(module,
1033
+ (QKVParallelLinear, MergedColumnParallelLinear)):
1034
+ self.maybe_fused_weights_modules[name] = module.output_sizes
1035
# In TP, these weights are partitioned along the column
1036
# dimension (dim=-1)
1037
elif isinstance(module, (RowParallelLinear, )):