A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix/Core] Flashinfer k_scale and v_scale (#9861)
Pavani Majety committed
Nov 1, 2024 at 12:15 UTC
598b6d7b070149aae5884aa8b17a0c91c93172f5
3 files changed
+25
-12
tests/kernels/test_cache.py
+14
-7
@@ -258,19 +258,20 @@ def test_reshape_and_cache_flash(
258
del key_caches
259
del value_caches
260
261
+ k_scale = key.amax().item() / 256
262
+ v_scale = value.amax().item() / 256
263
+
264
# Clone the KV caches.
265
if kv_cache_dtype == "fp8":
266
cloned_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
264
- ops.convert_fp8(cloned_key_cache, key_cache)
267
+ ops.convert_fp8(cloned_key_cache, key_cache, k_scale, kv_cache_dtype)
268
cloned_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
266
- ops.convert_fp8(cloned_value_cache, value_cache)
269
+ ops.convert_fp8(cloned_value_cache, value_cache, v_scale,
270
+ kv_cache_dtype)
271
else:
272
cloned_key_cache = key_cache.clone()
273
cloned_value_cache = value_cache.clone()
274
271
- # Using default kv_scale
272
- k_scale = v_scale = 1.0
273
-
275
# Call the reshape_and_cache kernel.
276
opcheck(torch.ops._C_cache_ops.reshape_and_cache_flash,
277
(key, value, key_cache, value_cache, slot_mapping, kv_cache_dtype,
@@ -281,9 +282,15 @@ def test_reshape_and_cache_flash(
282
283
if kv_cache_dtype == "fp8":
284
result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
284
- ops.convert_fp8(result_key_cache, key_cache)
285
+ ops.convert_fp8(result_key_cache,
286
+ key_cache,
287
+ k_scale,
288
+ kv_dtype=kv_cache_dtype)
289
result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
286
- ops.convert_fp8(result_value_cache, value_cache)
290
+ ops.convert_fp8(result_value_cache,
291
+ value_cache,
292
+ v_scale,
293
+ kv_dtype=kv_cache_dtype)
294
295
# Run the reference implementation.
296
block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
vllm/attention/backends/flashinfer.py
+6
-3
@@ -759,8 +759,6 @@ class FlashInferImpl(AttentionImpl):
759
v_scale: float = 1.0,
760
attn_type: AttentionType = AttentionType.DECODER,
761
) -> torch.Tensor:
762
- assert k_scale == 1.0 and v_scale == 1.0, (
763
- "key/v_scale is not supported in FlashInfer.")
762
if attn_type != AttentionType.DECODER:
763
raise NotImplementedError("Encoder self-attention and "
764
"encoder/decoder cross-attention "
@@ -874,7 +872,12 @@ def unified_flash_infer(
872
assert prefill_meta is not None
873
assert prefill_meta.prefill_wrapper is not None
874
prefill_output = prefill_meta.prefill_wrapper.forward(
877
- query, kv_cache, logits_soft_cap=logits_soft_cap, causal=True)
875
+ query,
876
+ kv_cache,
877
+ logits_soft_cap=logits_soft_cap,
878
+ causal=True,
879
+ k_scale=k_scale,
880
+ v_scale=v_scale)
881
if decode_meta := attn_metadata.decode_metadata:
882
assert attn_metadata.decode_metadata is not None
883
assert attn_metadata.decode_metadata.decode_wrapper is not None
vllm/model_executor/layers/quantization/modelopt.py
+5
-2
@@ -141,8 +141,11 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
141
layer.register_parameter("input_scale", scale)
142
143
def process_weights_after_loading(self, layer: Module) -> None:
144
- max_w_scale, weight = requantize_with_max_scale(
145
- layer.weight, layer.weight_scale, layer.logical_widths)
144
+ weight = layer.weight
145
+ max_w_scale = layer.weight_scale.max()
146
+ if not (layer.weight_scale == layer.weight_scale[0]).all():
147
+ max_w_scale, weight = requantize_with_max_scale(
148
+ layer.weight, layer.weight_scale, layer.logical_widths)
149
layer.weight = Parameter(weight.t(), requires_grad=False)
150
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
151
layer.input_scale = Parameter(layer.input_scale.max(),