A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix `kv_cache_dtype=fp8` without scales for FP8 checkpoints (#6761)
Michael Goin committed
Jul 25, 2024 at 12:46 UTC
65b1f121c885f169da210946eddb0d52524677f1
2 files changed
+12
-6
tests/quantization/test_fp8.py
+10
-2
@@ -60,12 +60,20 @@ def test_kv_cache_model_load_and_run(vllm_runner, model_id: str):
60
61
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
62
reason="FP8 is not supported on this GPU type.")
63
-def test_load_fp16_model(vllm_runner) -> None:
64
- with vllm_runner("facebook/opt-125m", quantization="fp8") as llm:
63
+@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
64
+def test_load_fp16_model(vllm_runner, kv_cache_dtype: str) -> None:
65
+ with vllm_runner("facebook/opt-125m",
66
+ quantization="fp8",
67
+ kv_cache_dtype=kv_cache_dtype) as llm:
68
69
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
70
fc1 = model.model.decoder.layers[0].fc1
71
assert isinstance(fc1.quant_method, Fp8LinearMethod)
72
+ if kv_cache_dtype == "fp8":
73
+ attn = model.model.decoder.layers[0].self_attn.attn
74
+ assert isinstance(attn.quant_method, Fp8KVCacheMethod)
75
+ assert attn._k_scale == 1.0
76
+ assert attn._v_scale == 1.0
77
78
capability = torch.cuda.get_device_capability()
79
capability = capability[0] * 10 + capability[1]
vllm/model_executor/layers/quantization/kv_cache.py
+2
-4
@@ -46,10 +46,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
46
elif layer.k_scale < 0.0 and layer.v_scale < 0.0:
47
# If no scales were loaded (both scales are invalid negative
48
# values), use the default value of 1.0
49
- k_scale = torch.nn.Parameter(torch.tensor(1.0),
50
- requires_grad=False)
51
- v_scale = torch.nn.Parameter(torch.tensor(1.0),
52
- requires_grad=False)
49
+ k_scale = 1.0
50
+ v_scale = 1.0
51
else:
52
# If we find a single kv_scale in the checkpoint, we remap
53
# kv_scale to k_scale during weight loading, and duplicate