@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ac79799403
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Fix for ROCM compressed tensor support (#11561)

Selali committed Dec 27, 2024 at 12:12 UTC ac797994039ba9e6ed0c2b3a503099cb122a936e
1 file changed +7 -3
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+7 -3
@@ -41,10 +41,12 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
41 )
42
43 if current_platform.is_rocm():
44 + input_scale = getattr(layer, 'input_scale', None)
45 +
46 weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
47 weight=weight,
48 weight_scale=max_w_scale,
47 - input_scale=layer.input_scale)
49 + input_scale=input_scale)
50 if input_scale is not None:
51 layer.input_scale = Parameter(input_scale,
52 requires_grad=False)
@@ -57,11 +59,13 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
59 weight = layer.weight
60
61 if current_platform.is_rocm():
62 + input_scale = getattr(layer, 'input_scale', None)
63 +
64 weight, weight_scale, input_scale = \
65 normalize_e4m3fn_to_e4m3fnuz(
66 weight=weight,
67 weight_scale=layer.weight_scale,
64 - input_scale=layer.input_scale)
68 + input_scale=input_scale)
69 if input_scale is not None:
70 layer.input_scale = Parameter(input_scale,
71 requires_grad=False)
@@ -76,7 +80,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
80 raise ValueError(f"Unknown quantization strategy {self.strategy}")
81
82 # INPUT SCALE
79 - if self.is_static_input_scheme:
83 + if self.is_static_input_scheme and hasattr(layer, 'input_scale'):
84 layer.input_scale = Parameter(layer.input_scale.max(),
85 requires_grad=False)
86 else: