@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a5314e8698
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] RowParallelLinear: pass bias to quant_method.apply (#6327)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Jul 19, 2024 at 15:15 UTC a5314e8698b7c0f20cf3facf921c54917c89a9ba
2 files changed +14 -9
tests/spec_decode/e2e/test_integration_dist_tp2.py
+3
@@ -83,6 +83,9 @@ def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
83 # cleaned up properly, and its server host thread leaks, causing the
84 # second run of the test to fail with internal NCCL error.
85 "use_async": True,
86 +
87 + # precision
88 + "dtype": "float32",
89 }])
90 @pytest.mark.parametrize("baseline_llm_kwargs", [{}])
91 @pytest.mark.parametrize(
vllm/model_executor/layers/linear.py
+11 -9
@@ -715,6 +715,7 @@ class RowParallelLinear(LinearBase):
715 self.reduce_results = reduce_results
716
717 # Divide the weight matrix along the last dimension.
718 + self.tp_rank = get_tensor_model_parallel_rank()
719 self.tp_size = get_tensor_model_parallel_world_size()
720 self.input_size_per_partition = divide(input_size, self.tp_size)
721 assert self.quant_method is not None
@@ -770,18 +771,19 @@ class RowParallelLinear(LinearBase):
771
772 # Matrix multiply.
773 assert self.quant_method is not None
773 - output_parallel = self.quant_method.apply(self, input_parallel)
774 + # Only fuse bias add into GEMM for rank 0 (this ensures that
775 + # bias will not get added more than once in TP>1 case)
776 + bias_ = None if (self.tp_rank > 0 or self.skip_bias_add) else self.bias
777 + output_parallel = self.quant_method.apply(self,
778 + input_parallel,
779 + bias=bias_)
780 if self.reduce_results and self.tp_size > 1:
775 - output_ = tensor_model_parallel_all_reduce(output_parallel)
781 + output = tensor_model_parallel_all_reduce(output_parallel)
782 else:
777 - output_ = output_parallel
783 + output = output_parallel
784 +
785 + output_bias = self.bias if self.skip_bias_add else None
786
779 - if not self.skip_bias_add:
780 - output = output_ + self.bias if self.bias is not None else output_
781 - output_bias = None
782 - else:
783 - output = output_
784 - output_bias = self.bias
787 return output, output_bias
788
789 def extra_repr(self) -> str: