@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7d761fe3c1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[FIX] Fix the case when `input_is_parallel=False` for `ScaledActivation` (#1737)

Zhuohan Li committed Nov 20, 2023 at 23:56 UTC 7d761fe3c12e87df37383467c43c97dec2bb8470
1 file changed +6 -4
vllm/model_executor/layers/activation.py
+6 -4
@@ -61,6 +61,7 @@ class ScaledActivation(nn.Module):
61 ):
62 super().__init__()
63 self.act = act_module
64 + self.input_is_parallel = input_is_parallel
65 if input_is_parallel:
66 tp_size = get_tensor_model_parallel_world_size()
67 intermediate_size_per_partition = divide(intermediate_size,
@@ -79,11 +80,12 @@ class ScaledActivation(nn.Module):
80 return self.act(x) / self.scales
81
82 def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
82 - tp_rank = get_tensor_model_parallel_rank()
83 param_data = param.data
84 - shard_size = param_data.shape[0]
85 - start_idx = tp_rank * shard_size
86 - loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
84 + if self.input_is_parallel:
85 + tp_rank = get_tensor_model_parallel_rank()
86 + shard_size = param_data.shape[0]
87 + start_idx = tp_rank * shard_size
88 + loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
89 assert param_data.shape == loaded_weight.shape
90 param_data.copy_(loaded_weight)
91