| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | """End-to-end accuracy tests for per-token-head KV cache quantization. |
| 5 | |
| 6 | Compares logprobs between a baseline bf16 model and the same model with |
| 7 | per-token-head quantized KV cache (int8 or fp8) using the Triton attention |
| 8 | backend. |
| 9 | |
| 10 | Run: pytest tests/models/quantization/test_per_token_kv_cache.py -v -s |
| 11 | """ |
| 12 | |
| 13 | import pytest |
| 14 | |
| 15 | from vllm.platforms import current_platform |
| 16 | |
| 17 | from ..utils import check_logprobs_close |
| 18 | |
| 19 | |
| 20 | @pytest.mark.skipif( |
| 21 | not current_platform.is_cuda_alike(), |
| 22 | reason="Per-token-head KV cache requires CUDA or ROCm GPU.", |
| 23 | ) |
| 24 | @pytest.mark.parametrize( |
| 25 | "base_model,test_model", |
| 26 | [ |
| 27 | ( |
| 28 | "meta-llama/Llama-3.2-1B-Instruct", |
| 29 | "meta-llama/Llama-3.2-1B-Instruct", |
| 30 | ), |
| 31 | ], |
| 32 | ) |
| 33 | @pytest.mark.parametrize( |
| 34 | "kv_cache_dtype", |
| 35 | ["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"], |
| 36 | ) |
| 37 | @pytest.mark.parametrize("max_tokens", [4]) |
| 38 | @pytest.mark.parametrize("enforce_eager", [True]) |
| 39 | @pytest.mark.parametrize("backend", ["TRITON_ATTN"]) |
| 40 | @pytest.mark.parametrize("tensor_parallel_size", [1]) |
| 41 | def test_per_token_head_kv_cache_accuracy( |
| 42 | vllm_runner, |
| 43 | example_prompts, |
| 44 | base_model: str, |
| 45 | test_model: str, |
| 46 | kv_cache_dtype: str, |
| 47 | max_tokens: int, |
| 48 | enforce_eager: bool, |
| 49 | backend: str, |
| 50 | tensor_parallel_size: int, |
| 51 | monkeypatch: pytest.MonkeyPatch, |
| 52 | ) -> None: |
| 53 | """Compare logprobs between bf16 baseline and per-token-head quantized KV |
| 54 | cache. |
| 55 | """ |
| 56 | with monkeypatch.context() as m: |
| 57 | m.setenv("TOKENIZERS_PARALLELISM", "true") |
| 58 | |
| 59 | MAX_MODEL_LEN = 1024 |
| 60 | NUM_LOG_PROBS = 8 |
| 61 | |
| 62 | with vllm_runner( |
| 63 | base_model, |
| 64 | max_model_len=MAX_MODEL_LEN, |
| 65 | tensor_parallel_size=tensor_parallel_size, |
| 66 | enforce_eager=enforce_eager, |
| 67 | kv_cache_dtype="auto", |
| 68 | attention_config={"backend": backend}, |
| 69 | ) as vllm_model: |
| 70 | baseline_outputs = vllm_model.generate_greedy_logprobs( |
| 71 | example_prompts, max_tokens, NUM_LOG_PROBS |
| 72 | ) |
| 73 | |
| 74 | with vllm_runner( |
| 75 | test_model, |
| 76 | max_model_len=MAX_MODEL_LEN, |
| 77 | tensor_parallel_size=tensor_parallel_size, |
| 78 | enforce_eager=enforce_eager, |
| 79 | kv_cache_dtype=kv_cache_dtype, |
| 80 | attention_config={"backend": backend}, |
| 81 | ) as vllm_model: |
| 82 | test_outputs = vllm_model.generate_greedy_logprobs( |
| 83 | example_prompts, max_tokens, NUM_LOG_PROBS |
| 84 | ) |
| 85 | |
| 86 | check_logprobs_close( |
| 87 | outputs_0_lst=baseline_outputs, |
| 88 | outputs_1_lst=test_outputs, |
| 89 | name_0="bf16_kv_cache", |
| 90 | name_1=f"{kv_cache_dtype}_kv_cache", |
| 91 | ) |