@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9d852d601
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Remove Mixtral device="cuda" declarations (#4543)

Remove the device="cuda" declarations in mixtral as promised in #4343

Philipp Moritz committed May 1, 2024 at 16:30 UTC c9d852d601ce1a02f6748ab62db8694c22772583
1 file changed +4 -8
vllm/model_executor/models/mixtral.py
+4 -8
@@ -96,13 +96,11 @@ class MixtralMoE(nn.Module):
96 torch.empty(self.num_total_experts,
97 2 * self.intermediate_size,
98 self.hidden_size,
99 - device="cuda",
99 dtype=self.params_dtype))
100 self.w2s = nn.Parameter(
101 torch.empty(self.num_total_experts,
102 self.hidden_size,
103 self.intermediate_size,
105 - device="cuda",
104 dtype=self.params_dtype))
105
106 set_weight_attrs(self.ws, {
@@ -114,22 +112,20 @@ class MixtralMoE(nn.Module):
112
113 # Scaling factors for FP8 weights
114 self.ws_scale = nn.Parameter(
117 - torch.ones(
118 - self.num_total_experts, device="cuda", dtype=torch.float32),
115 + torch.ones(self.num_total_experts, dtype=torch.float32),
116 requires_grad=False) if self.use_fp8 else None
117 self.w2s_scale = nn.Parameter(
121 - torch.ones(
122 - self.num_total_experts, device="cuda", dtype=torch.float32),
118 + torch.ones(self.num_total_experts, dtype=torch.float32),
119 requires_grad=False) if self.use_fp8 else None
120
121 # Scaling factors for FP8 activations
122 need_act_scales = (self.use_fp8
123 and quant_config.activation_scheme == "static")
124 self.as_scale = nn.Parameter(
129 - torch.zeros(1, device="cuda", dtype=torch.float32),
125 + torch.zeros(1, dtype=torch.float32),
126 requires_grad=False) if need_act_scales else None
127 self.a2s_scale = nn.Parameter(
132 - torch.zeros(1, device="cuda", dtype=torch.float32),
128 + torch.zeros(1, dtype=torch.float32),
129 requires_grad=False) if need_act_scales else None
130
131 if need_act_scales: