@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 598b6d7b07
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix/Core] Flashinfer k_scale and v_scale (#9861)

Pavani Majety committed Nov 1, 2024 at 12:15 UTC 598b6d7b070149aae5884aa8b17a0c91c93172f5
3 files changed +25 -12
tests/kernels/test_cache.py
+14 -7
@@ -258,19 +258,20 @@ def test_reshape_and_cache_flash(
258 del key_caches
259 del value_caches
260
261 + k_scale = key.amax().item() / 256
262 + v_scale = value.amax().item() / 256
263 +
264 # Clone the KV caches.
265 if kv_cache_dtype == "fp8":
266 cloned_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
264 - ops.convert_fp8(cloned_key_cache, key_cache)
267 + ops.convert_fp8(cloned_key_cache, key_cache, k_scale, kv_cache_dtype)
268 cloned_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
266 - ops.convert_fp8(cloned_value_cache, value_cache)
269 + ops.convert_fp8(cloned_value_cache, value_cache, v_scale,
270 + kv_cache_dtype)
271 else:
272 cloned_key_cache = key_cache.clone()
273 cloned_value_cache = value_cache.clone()
274
271 - # Using default kv_scale
272 - k_scale = v_scale = 1.0
273 -
275 # Call the reshape_and_cache kernel.
276 opcheck(torch.ops._C_cache_ops.reshape_and_cache_flash,
277 (key, value, key_cache, value_cache, slot_mapping, kv_cache_dtype,
@@ -281,9 +282,15 @@ def test_reshape_and_cache_flash(
282
283 if kv_cache_dtype == "fp8":
284 result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
284 - ops.convert_fp8(result_key_cache, key_cache)
285 + ops.convert_fp8(result_key_cache,
286 + key_cache,
287 + k_scale,
288 + kv_dtype=kv_cache_dtype)
289 result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
286 - ops.convert_fp8(result_value_cache, value_cache)
290 + ops.convert_fp8(result_value_cache,
291 + value_cache,
292 + v_scale,
293 + kv_dtype=kv_cache_dtype)
294
295 # Run the reference implementation.
296 block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
vllm/attention/backends/flashinfer.py
+6 -3
@@ -759,8 +759,6 @@ class FlashInferImpl(AttentionImpl):
759 v_scale: float = 1.0,
760 attn_type: AttentionType = AttentionType.DECODER,
761 ) -> torch.Tensor:
762 - assert k_scale == 1.0 and v_scale == 1.0, (
763 - "key/v_scale is not supported in FlashInfer.")
762 if attn_type != AttentionType.DECODER:
763 raise NotImplementedError("Encoder self-attention and "
764 "encoder/decoder cross-attention "
@@ -874,7 +872,12 @@ def unified_flash_infer(
872 assert prefill_meta is not None
873 assert prefill_meta.prefill_wrapper is not None
874 prefill_output = prefill_meta.prefill_wrapper.forward(
877 - query, kv_cache, logits_soft_cap=logits_soft_cap, causal=True)
875 + query,
876 + kv_cache,
877 + logits_soft_cap=logits_soft_cap,
878 + causal=True,
879 + k_scale=k_scale,
880 + v_scale=v_scale)
881 if decode_meta := attn_metadata.decode_metadata:
882 assert attn_metadata.decode_metadata is not None
883 assert attn_metadata.decode_metadata.decode_wrapper is not None
vllm/model_executor/layers/quantization/modelopt.py
+5 -2
@@ -141,8 +141,11 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
141 layer.register_parameter("input_scale", scale)
142
143 def process_weights_after_loading(self, layer: Module) -> None:
144 - max_w_scale, weight = requantize_with_max_scale(
145 - layer.weight, layer.weight_scale, layer.logical_widths)
144 + weight = layer.weight
145 + max_w_scale = layer.weight_scale.max()
146 + if not (layer.weight_scale == layer.weight_scale[0]).all():
147 + max_w_scale, weight = requantize_with_max_scale(
148 + layer.weight, layer.weight_scale, layer.logical_widths)
149 layer.weight = Parameter(weight.t(), requires_grad=False)
150 layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
151 layer.input_scale = Parameter(layer.input_scale.max(),