A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Model] RowParallelLinear: pass bias to quant_method.apply (#6327)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jul 19, 2024 at 15:15 UTC
a5314e8698b7c0f20cf3facf921c54917c89a9ba
2 files changed
+14
-9
tests/spec_decode/e2e/test_integration_dist_tp2.py
+3
@@ -83,6 +83,9 @@ def test_target_model_tp_gt_1(baseline_llm_generator, test_llm_generator,
83
# cleaned up properly, and its server host thread leaks, causing the
84
# second run of the test to fail with internal NCCL error.
85
"use_async": True,
86
+
87
+ # precision
88
+ "dtype": "float32",
89
}])
90
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
91
@pytest.mark.parametrize(
vllm/model_executor/layers/linear.py
+11
-9
@@ -715,6 +715,7 @@ class RowParallelLinear(LinearBase):
715
self.reduce_results = reduce_results
716
717
# Divide the weight matrix along the last dimension.
718
+ self.tp_rank = get_tensor_model_parallel_rank()
719
self.tp_size = get_tensor_model_parallel_world_size()
720
self.input_size_per_partition = divide(input_size, self.tp_size)
721
assert self.quant_method is not None
@@ -770,18 +771,19 @@ class RowParallelLinear(LinearBase):
771
772
# Matrix multiply.
773
assert self.quant_method is not None
773
- output_parallel = self.quant_method.apply(self, input_parallel)
774
+ # Only fuse bias add into GEMM for rank 0 (this ensures that
775
+ # bias will not get added more than once in TP>1 case)
776
+ bias_ = None if (self.tp_rank > 0 or self.skip_bias_add) else self.bias
777
+ output_parallel = self.quant_method.apply(self,
778
+ input_parallel,
779
+ bias=bias_)
780
if self.reduce_results and self.tp_size > 1:
775
- output_ = tensor_model_parallel_all_reduce(output_parallel)
781
+ output = tensor_model_parallel_all_reduce(output_parallel)
782
else:
777
- output_ = output_parallel
783
+ output = output_parallel
784
+
785
+ output_bias = self.bias if self.skip_bias_add else None
786
779
- if not self.skip_bias_add:
780
- output = output_ + self.bias if self.bias is not None else output_
781
- output_bias = None
782
- else:
783
- output = output_
784
- output_bias = self.bias
787
return output, output_bias
788
789
def extra_repr(self) -> str: