@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7eb719df13
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix]Fix Phi-3 BNB online quantization (#10417)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 19, 2024 at 11:21 UTC 7eb719df13cf8059485f52648a6a115700158301
2 files changed +19 -3
vllm/model_executor/layers/linear.py
+9 -3
@@ -470,7 +470,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
470 needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False)
471
472 if loaded_shard_id is None:
473 - # Loaded weight is already fused on disk (qkv/mlp).
473 + # Loaded weight is already fused on disk (mlp).
474 + # (e.g., Phi-3's gate_up_proj).
475 if output_dim is None:
476 if needs_scalar_to_array:
477 param_data, loaded_weight = adjust_scalar_to_fused_array(
@@ -480,6 +481,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
481 param_data.copy_(loaded_weight)
482 return
483 current_shard_offset = 0
484 + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit",
485 + False)
486 shard_offsets: List[Tuple[int, int, int]] = []
487 for i, output_size in enumerate(self.output_sizes):
488 shard_offsets.append((i, current_shard_offset, output_size))
@@ -495,7 +498,9 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
498 # Special case for Marlin.
499 shard_size, shard_offset = adjust_marlin_shard(
500 param, shard_size, shard_offset)
498 -
501 + if use_bitsandbytes_4bit:
502 + shard_size = loaded_weight.shape[output_dim] // 2
503 + shard_offset = shard_size * shard_id
504 loaded_weight_shard = loaded_weight.narrow(
505 output_dim, shard_offset, shard_size)
506 self.weight_loader(param, loaded_weight_shard, shard_id)
@@ -808,7 +813,8 @@ class QKVParallelLinear(ColumnParallelLinear):
813 needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False)
814
815 if loaded_shard_id is None:
811 - # Loaded weight is already fused on disk (qkv/mlp).
816 + # Loaded weight is already fused on disk (qkv).
817 + # (e.g., Phi-3's qkv_proj).
818 if output_dim is None:
819 if needs_scalar_to_array:
820 param_data, loaded_weight = adjust_scalar_to_fused_array(
vllm/model_executor/models/phi3.py
+10
@@ -14,3 +14,13 @@ class Phi3ForCausalLM(LlamaForCausalLM):
14 "gate_up_proj",
15 ],
16 }
17 +
18 + # BitandBytes specific attributes
19 + default_bitsandbytes_target_modules = [
20 + ".gate_up_proj.",
21 + ".down_proj.",
22 + ".qkv_proj.",
23 + ".o_proj.",
24 + ]
25 + # Initialize an empty dict when there is no stacked parameter mapping.
26 + bitsandbytes_stacked_params_mapping = {}