@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65b1f121c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix `kv_cache_dtype=fp8` without scales for FP8 checkpoints (#6761)

Michael Goin committed Jul 25, 2024 at 12:46 UTC 65b1f121c885f169da210946eddb0d52524677f1
2 files changed +12 -6
tests/quantization/test_fp8.py
+10 -2
@@ -60,12 +60,20 @@ def test_kv_cache_model_load_and_run(vllm_runner, model_id: str):
60
61 @pytest.mark.skipif(not is_quant_method_supported("fp8"),
62 reason="FP8 is not supported on this GPU type.")
63 -def test_load_fp16_model(vllm_runner) -> None:
64 - with vllm_runner("facebook/opt-125m", quantization="fp8") as llm:
63 +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
64 +def test_load_fp16_model(vllm_runner, kv_cache_dtype: str) -> None:
65 + with vllm_runner("facebook/opt-125m",
66 + quantization="fp8",
67 + kv_cache_dtype=kv_cache_dtype) as llm:
68
69 model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
70 fc1 = model.model.decoder.layers[0].fc1
71 assert isinstance(fc1.quant_method, Fp8LinearMethod)
72 + if kv_cache_dtype == "fp8":
73 + attn = model.model.decoder.layers[0].self_attn.attn
74 + assert isinstance(attn.quant_method, Fp8KVCacheMethod)
75 + assert attn._k_scale == 1.0
76 + assert attn._v_scale == 1.0
77
78 capability = torch.cuda.get_device_capability()
79 capability = capability[0] * 10 + capability[1]
vllm/model_executor/layers/quantization/kv_cache.py
+2 -4
@@ -46,10 +46,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
46 elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
47 # If no scales were loaded (both scales are invalid negative
48 # values), use the default value of 1.0
49 - k_scale = torch.nn.Parameter(torch.tensor(1.0),
50 - requires_grad=False)
51 - v_scale = torch.nn.Parameter(torch.tensor(1.0),
52 - requires_grad=False)
49 + k_scale = 1.0
50 + v_scale = 1.0
51 else:
52 # If we find a single kv_scale in the checkpoint, we remap
53 # kv_scale to k_scale during weight loading, and duplicate