A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix]Fix Phi-3 BNB online quantization (#10417)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 19, 2024 at 11:21 UTC
7eb719df13cf8059485f52648a6a115700158301
2 files changed
+19
-3
vllm/model_executor/layers/linear.py
+9
-3
@@ -470,7 +470,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
470
needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False)
471
472
if loaded_shard_id is None:
473
- # Loaded weight is already fused on disk (qkv/mlp).
473
+ # Loaded weight is already fused on disk (mlp).
474
+ # (e.g., Phi-3's gate_up_proj).
475
if output_dim is None:
476
if needs_scalar_to_array:
477
param_data, loaded_weight = adjust_scalar_to_fused_array(
@@ -480,6 +481,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
481
param_data.copy_(loaded_weight)
482
return
483
current_shard_offset = 0
484
+ use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
485
+ False)
486
shard_offsets: List[Tuple[int, int, int]] = []
487
for i, output_size in enumerate(self.output_sizes):
488
shard_offsets.append((i, current_shard_offset, output_size))
@@ -495,7 +498,9 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
498
# Special case for Marlin.
499
shard_size, shard_offset = adjust_marlin_shard(
500
param, shard_size, shard_offset)
498
-
501
+ if use_bitsandbytes_4bit:
502
+ shard_size = loaded_weight.shape[output_dim] // 2
503
+ shard_offset = shard_size * shard_id
504
loaded_weight_shard = loaded_weight.narrow(
505
output_dim, shard_offset, shard_size)
506
self.weight_loader(param, loaded_weight_shard, shard_id)
@@ -808,7 +813,8 @@ class QKVParallelLinear(ColumnParallelLinear):
813
needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False)
814
815
if loaded_shard_id is None:
811
- # Loaded weight is already fused on disk (qkv/mlp).
816
+ # Loaded weight is already fused on disk (qkv).
817
+ # (e.g., Phi-3's qkv_proj).
818
if output_dim is None:
819
if needs_scalar_to_array:
820
param_data, loaded_weight = adjust_scalar_to_fused_array(
vllm/model_executor/models/phi3.py
+10
@@ -14,3 +14,13 @@ class Phi3ForCausalLM(LlamaForCausalLM):
14
"gate_up_proj",
15
],
16
}
17
+
18
+ # BitandBytes specific attributes
19
+ default_bitsandbytes_target_modules = [
20
+ ".gate_up_proj.",
21
+ ".down_proj.",
22
+ ".qkv_proj.",
23
+ ".o_proj.",
24
+ ]
25
+ # Initialize an empty dict when there is no stacked parameter mapping.
26
+ bitsandbytes_stacked_params_mapping = {}