A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Remove Mixtral device="cuda" declarations (#4543)
Remove the device="cuda" declarations in mixtral as promised in #4343
Philipp Moritz committed
May 1, 2024 at 16:30 UTC
c9d852d601ce1a02f6748ab62db8694c22772583
1 file changed
+4
-8
vllm/model_executor/models/mixtral.py
+4
-8
@@ -96,13 +96,11 @@ class MixtralMoE(nn.Module):
96
torch.empty(self.num_total_experts,
97
2 * self.intermediate_size,
98
self.hidden_size,
99
- device="cuda",
99
dtype=self.params_dtype))
100
self.w2s = nn.Parameter(
101
torch.empty(self.num_total_experts,
102
self.hidden_size,
103
self.intermediate_size,
105
- device="cuda",
104
dtype=self.params_dtype))
105
106
set_weight_attrs(self.ws, {
@@ -114,22 +112,20 @@ class MixtralMoE(nn.Module):
112
113
# Scaling factors for FP8 weights
114
self.ws_scale = nn.Parameter(
117
- torch.ones(
118
- self.num_total_experts, device="cuda", dtype=torch.float32),
115
+ torch.ones(self.num_total_experts, dtype=torch.float32),
116
requires_grad=False) if self.use_fp8 else None
117
self.w2s_scale = nn.Parameter(
121
- torch.ones(
122
- self.num_total_experts, device="cuda", dtype=torch.float32),
118
+ torch.ones(self.num_total_experts, dtype=torch.float32),
119
requires_grad=False) if self.use_fp8 else None
120
121
# Scaling factors for FP8 activations
122
need_act_scales = (self.use_fp8
123
and quant_config.activation_scheme == "static")
124
self.as_scale = nn.Parameter(
129
- torch.zeros(1, device="cuda", dtype=torch.float32),
125
+ torch.zeros(1, dtype=torch.float32),
126
requires_grad=False) if need_act_scales else None
127
self.a2s_scale = nn.Parameter(
132
- torch.zeros(1, device="cuda", dtype=torch.float32),
128
+ torch.zeros(1, dtype=torch.float32),
129
requires_grad=False) if need_act_scales else None
130
131
if need_act_scales: