@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 325f679f32
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[BugFix] Fix Torch.Compile For DeepSeek (#12594)

Co-authored-by: simon-mo <xmo@berkeley.edu>

Robert Shaw committed Jan 31, 2025 at 15:06 UTC 325f679f324c1044cfaa0c594bf0d817eeda4451
1 file changed +29 -25
vllm/model_executor/layers/quantization/fp8.py
+29 -25
@@ -245,20 +245,24 @@ class Fp8LinearMethod(LinearMethodBase):
245 layer.register_parameter("input_scale", None)
246
247 def process_weights_after_loading(self, layer: Module) -> None:
248 - # Block quant doesn't need to process weights after loading
248 + # TODO(rob): refactor block quant into separate class.
249 if self.block_quant:
250 + assert self.quant_config.activation_scheme == "dynamic"
251 if current_platform.is_rocm():
251 - weight, weight_scale, _ = \
252 + weight, weight_scale_inv, _ = \
253 normalize_e4m3fn_to_e4m3fnuz(
254 weight=layer.weight,
254 - weight_scale=layer.weight_scale_inv,
255 - input_scale=layer.input_scale)
256 - layer.weight = Parameter(weight, requires_grad=False)
257 - layer.weight_scale_inv = Parameter(weight_scale,
258 - requires_grad=False)
255 + weight_scale=layer.weight_scale_inv)
256 + else:
257 + weight = layer.weight.data
258 + weight_scale_inv = layer.weight_scale_inv.data
259 +
260 + # Torch.compile cannot use Parameter subclasses.
261 + layer.weight = Parameter(weight, requires_grad=False)
262 + layer.weight_scale_inv = Parameter(weight_scale_inv,
263 + requires_grad=False)
264 return
260 - layer.weight = torch.nn.Parameter(layer.weight.data,
261 - requires_grad=False)
265 +
266 # If checkpoint not serialized fp8, quantize the weights.
267 if not self.quant_config.is_checkpoint_fp8_serialized:
268 qweight, weight_scale = ops.scaled_fp8_quant(layer.weight,
@@ -507,8 +511,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
511 layer.w2_input_scale = None
512
513 def process_weights_after_loading(self, layer: Module) -> None:
510 - # Block quant doesn't need to process weights after loading
514 + # TODO (rob): refactor block quant into separate class.
515 if self.block_quant:
516 + assert self.quant_config.activation_scheme == "dynamic"
517 if current_platform.is_rocm():
518 w13_weight, w13_weight_scale_inv, w13_input_scale = \
519 normalize_e4m3fn_to_e4m3fnuz(
@@ -518,22 +523,21 @@ class Fp8MoEMethod(FusedMoEMethodBase):
523 normalize_e4m3fn_to_e4m3fnuz(
524 layer.w2_weight, layer.w2_weight_scale_inv,
525 layer.w2_input_scale)
521 - # Reset the parameter
522 - layer.w13_weight = torch.nn.Parameter(w13_weight,
523 - requires_grad=False)
524 - layer.w13_weight_scale_inv = torch.nn.Parameter(
525 - w13_weight_scale_inv, requires_grad=False)
526 - if w13_input_scale is not None:
527 - layer.w13_input_scale = torch.nn.Parameter(
528 - w13_input_scale, requires_grad=False)
529 - layer.w2_weight = torch.nn.Parameter(w2_weight,
530 - requires_grad=False)
531 - layer.w2_weight_scale_inv = torch.nn.Parameter(
532 - w2_weight_scale_inv, requires_grad=False)
533 - if w2_input_scale is not None:
534 - layer.w2_input_scale = torch.nn.Parameter(
535 - w2_input_scale, requires_grad=False)
526 + else:
527 + w13_weight = layer.w13_weight.data
528 + w13_weight_scale_inv = layer.w13_weight_scale_inv.data
529 + w2_weight = layer.w2_weight
530 + w2_weight_scale_inv = layer.w2_weight_scale_inv
531 +
532 + # torch.compile() cannot use Parameter subclasses.
533 + layer.w13_weight = Parameter(w13_weight, requires_grad=False)
534 + layer.w13_weight_scale_inv = Parameter(w13_weight_scale_inv,
535 + requires_grad=False)
536 + layer.w2_weight = Parameter(w2_weight, requires_grad=False)
537 + layer.w2_weight_scale_inv = Parameter(w2_weight_scale_inv,
538 + requires_grad=False)
539 return
540 +
541 # If checkpoint is fp16, quantize in place.
542 if not self.quant_config.is_checkpoint_fp8_serialized:
543 # If rocm, use float8_e4m3fnuz as dtype