A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[FIX] Fix the case when `input_is_parallel=False` for `ScaledActivation` (#1737)
Zhuohan Li committed
Nov 20, 2023 at 23:56 UTC
7d761fe3c12e87df37383467c43c97dec2bb8470
1 file changed
+6
-4
vllm/model_executor/layers/activation.py
+6
-4
@@ -61,6 +61,7 @@ class ScaledActivation(nn.Module):
61
):
62
super().__init__()
63
self.act = act_module
64
+ self.input_is_parallel = input_is_parallel
65
if input_is_parallel:
66
tp_size = get_tensor_model_parallel_world_size()
67
intermediate_size_per_partition = divide(intermediate_size,
@@ -79,11 +80,12 @@ class ScaledActivation(nn.Module):
80
return self.act(x) / self.scales
81
82
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
82
- tp_rank = get_tensor_model_parallel_rank()
83
param_data = param.data
84
- shard_size = param_data.shape[0]
85
- start_idx = tp_rank * shard_size
86
- loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
84
+ if self.input_is_parallel:
85
+ tp_rank = get_tensor_model_parallel_rank()
86
+ shard_size = param_data.shape[0]
87
+ start_idx = tp_rank * shard_size
88
+ loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
89
assert param_data.shape == loaded_weight.shape
90
param_data.copy_(loaded_weight)
91