A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Fix for ROCM compressed tensor support (#11561)
Selali committed
Dec 27, 2024 at 12:12 UTC
ac797994039ba9e6ed0c2b3a503099cb122a936e
1 file changed
+7
-3
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+7
-3
@@ -41,10 +41,12 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
41
)
42
43
if current_platform.is_rocm():
44
+ input_scale = getattr(layer, 'input_scale', None)
45
+
46
weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
47
weight=weight,
48
weight_scale=max_w_scale,
47
- input_scale=layer.input_scale)
49
+ input_scale=input_scale)
50
if input_scale is not None:
51
layer.input_scale = Parameter(input_scale,
52
requires_grad=False)
@@ -57,11 +59,13 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
59
weight = layer.weight
60
61
if current_platform.is_rocm():
62
+ input_scale = getattr(layer, 'input_scale', None)
63
+
64
weight, weight_scale, input_scale = \
65
normalize_e4m3fn_to_e4m3fnuz(
66
weight=weight,
67
weight_scale=layer.weight_scale,
64
- input_scale=layer.input_scale)
68
+ input_scale=input_scale)
69
if input_scale is not None:
70
layer.input_scale = Parameter(input_scale,
71
requires_grad=False)
@@ -76,7 +80,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
80
raise ValueError(f"Unknown quantization strategy {self.strategy}")
81
82
# INPUT SCALE
79
- if self.is_static_input_scheme:
83
+ if self.is_static_input_scheme and hasattr(layer, 'input_scale'):
84
layer.input_scale = Parameter(layer.input_scale.max(),
85
requires_grad=False)
86
else: