@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7ca9934fe7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc] Update w2 scale loading for GPTQMarlinMoE (#12757)

Dipika Sikka committed Feb 6, 2025 at 04:02 UTC 7ca9934fe773edf8680aed287b0a05cb195bd8e4
3 files changed +21 -8
tests/weight_loading/models-large.txt
+2
@@ -1,5 +1,7 @@
1 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4 +compressed-tensors, nm-testing/test-w4a16-mixtral-actorder-group, main
5 gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
6 +gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, gptq-8bit-128g-actorder_True
7 awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
\ No newline at end of file
vllm/model_executor/layers/fused_moe/layer.py
+2 -2
@@ -302,8 +302,8 @@ class FusedMoE(torch.nn.Module):
302 "weight_loader": self.weight_loader,
303 }
304 # need full intermediate size pre-sharding for WNA16 act order
305 - if (self.quant_method.__class__.__name__ ==
306 - "CompressedTensorsWNA16MoEMethod"):
305 + if (self.quant_method.__class__.__name__
306 + in ("GPTQMarlinMoEMethod", "CompressedTensorsWNA16MoEMethod")):
307 moe_quant_params["intermediate_size_full"] = intermediate_size
308
309 self.quant_method.create_weights(layer=self, **moe_quant_params)
vllm/model_executor/layers/quantization/gptq_marlin.py
+17 -6
@@ -323,13 +323,18 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
323 params_dtype: torch.dtype,
324 **extra_weight_attrs,
325 ):
326 - # Currently assuming is_k_full is always True
327 - # (input size per partition is the same as full input size)
328 - # Supports only sym for now (no zp)
326 + intermediate_size_full = extra_weight_attrs.pop(
327 + "intermediate_size_full")
328 +
329 + self.is_k_full = (not self.quant_config.desc_act) or (
330 + intermediate_size_per_partition == intermediate_size_full)
331 +
332 if self.quant_config.group_size != -1:
333 scales_size13 = hidden_size // self.quant_config.group_size
331 - scales_size2 = (intermediate_size_per_partition //
332 - self.quant_config.group_size)
334 + w2_scales_size = (intermediate_size_full
335 + if self.quant_config.desc_act else
336 + intermediate_size_per_partition)
337 + scales_size2 = (w2_scales_size // self.quant_config.group_size)
338 strategy = FusedMoeWeightScaleSupported.GROUP.value
339 else:
340 scales_size13 = 1
@@ -385,6 +390,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
390 )
391 layer.register_parameter("w2_scales", w2_scales)
392 set_weight_attrs(w2_scales, extra_weight_attrs)
393 + # dont shard the w2 scales when running act order
394 + set_weight_attrs(w2_scales,
395 + {"load_full_w2": self.quant_config.desc_act})
396 # up_proj scales
397 w13_qzeros = torch.nn.Parameter(
398 torch.empty(num_experts,
@@ -406,6 +414,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
414 )
415 layer.register_parameter("w2_qzeros", w2_qzeros)
416 set_weight_attrs(w2_qzeros, extra_weight_attrs)
417 + # dont shard the w2 scales when running act order
418 + set_weight_attrs(w2_qzeros,
419 + {"load_full_w2": self.quant_config.desc_act})
420 w13_g_idx = torch.nn.Parameter(
421 torch.empty(
422 num_experts,
@@ -575,4 +586,4 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
586 sort_indices1=layer.w13_g_idx_sort_indices,
587 sort_indices2=layer.w2_g_idx_sort_indices,
588 num_bits=self.quant_config.quant_type.size_bits,
578 - ).to(orig_dtype)
589 + is_k_full=self.is_k_full).to(orig_dtype)