A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[MISC] Add prefix cache hit rate to metrics (#7606)
Cody Yu committed
Aug 19, 2024 at 11:52 UTC
3ac50b47d0f718364d05b1bcb93743e15be6a37c
16 files changed
+200
-16
tests/core/block/test_prefix_caching_block.py
+26
@@ -682,6 +682,32 @@ class TestPrefixCachingBlockAllocator:
682
683
assert new_block[0].block_id == last_block_id
684
685
+ # Test case for cache mertics
686
+ @staticmethod
687
+ def test_metric():
688
+ block_size = 16
689
+ allocator = PrefixCachingBlockAllocator(num_blocks=4,
690
+ block_size=block_size)
691
+ # Test when no query (0/0)
692
+ assert allocator.get_prefix_cache_hit_rate() == 0.0
693
+
694
+ token_ids = list(range(block_size))
695
+ allocator.allocate_immutable_block(prev_block=None,
696
+ token_ids=token_ids)
697
+ # Test 0/1 hit rate
698
+ assert allocator.get_prefix_cache_hit_rate() == 0.0
699
+
700
+ allocator.allocate_immutable_block(prev_block=None,
701
+ token_ids=token_ids)
702
+ # Test 1/2 hit rate
703
+ assert allocator.get_prefix_cache_hit_rate() == 0.5
704
+
705
+ # Test more than one block
706
+ for _ in range(2, 1005):
707
+ allocator.allocate_immutable_block(prev_block=None,
708
+ token_ids=token_ids)
709
+ assert allocator.get_prefix_cache_hit_rate() > 0.99
710
+
711
@staticmethod
712
def create_immutable_chain(
713
block_size: int,
tests/prefix_caching/test_prefix_caching.py
+7
@@ -34,6 +34,9 @@ def test_block_allocator(
34
assert (first_block == second_block)
35
assert (second_block.ref_count == 2)
36
37
+ # Check metric: 1 hit of 2 queries
38
+ assert block_allocator.get_prefix_cache_hit_rate() == 0.5
39
+
40
# Free the first_block and confirm that the ref_count is correctly
41
# decremented on the second block
42
block_allocator.free(first_block)
@@ -48,6 +51,10 @@ def test_block_allocator(
51
assert (first_block == second_block)
52
assert (first_block.block_hash == block_hash)
53
54
+ # Allocate one more time to get 3/4 hit rate for easy checking
55
+ block_allocator.allocate(block_hash, 0)
56
+ assert block_allocator.get_prefix_cache_hit_rate() == 0.75
57
+
58
59
@pytest.mark.parametrize("num_blocks", [16])
60
def test_eviction(num_blocks: int, ):
vllm/core/block/common.py
+53
@@ -1,4 +1,5 @@
1
from collections import deque
2
+from dataclasses import dataclass
3
from typing import Deque, Dict, Iterable, List, Optional, Protocol, Tuple
4
5
from vllm.core.block.interfaces import Block, BlockAllocator
@@ -282,6 +283,58 @@ class BlockList:
283
return self._block_ids
284
285
286
+@dataclass
287
+class CacheMetricData:
288
+ """A utility dataclass to maintain cache metric.
289
+ To avoid overflow, we maintain the hit rate in block granularity, so that
290
+ we can maintain a single hit rate for n_completed_block x block_size,
291
+ and calculate the real time hit rate by the following:
292
+ BS = The number of queries per block.
293
+ nB = The number of completed blocks.
294
+ HR = hit rate of (nB x BS) queries.
295
+ Q = current number of queries (< BS).
296
+ H = current number of hits (< BS).
297
+ hit rate = ((HR x nB) + (H / Q) x (Q / BS)) / (nB + Q / BS)
298
+ """
299
+ num_completed_blocks: int = 0
300
+ completed_block_cache_hit_rate: float = 0.0
301
+ num_incompleted_block_queries: int = 0
302
+ num_incompleted_block_hit: int = 0
303
+ block_size: int = 1000
304
+
305
+ def query(self, hit: bool):
306
+ self.num_incompleted_block_queries += 1
307
+ self.num_incompleted_block_hit += 1 if hit else 0
308
+
309
+ # When a block is completed, update the cache hit rate
310
+ # and reset the incomplete numbers.
311
+ if self.num_incompleted_block_queries == self.block_size:
312
+ hit_rate = (self.num_incompleted_block_hit /
313
+ self.num_incompleted_block_queries)
314
+ self.completed_block_cache_hit_rate = (
315
+ self.completed_block_cache_hit_rate * self.num_completed_blocks
316
+ + hit_rate) / (self.num_completed_blocks + 1)
317
+ self.num_incompleted_block_queries = 0
318
+ self.num_incompleted_block_hit = 0
319
+ self.num_completed_blocks += 1
320
+
321
+ def get_hit_rate(self):
322
+ incomplete_ratio = self.num_incompleted_block_queries / self.block_size
323
+ total_blocks = self.num_completed_blocks + incomplete_ratio
324
+ if total_blocks == 0:
325
+ return 0.0
326
+
327
+ completed_block_hit, incompleted_block_hit = 0.0, 0.0
328
+ if self.num_completed_blocks > 0:
329
+ completed_block_hit = (self.completed_block_cache_hit_rate *
330
+ self.num_completed_blocks)
331
+ if self.num_incompleted_block_queries > 0:
332
+ incompleted_hit_rate = (self.num_incompleted_block_hit /
333
+ self.num_incompleted_block_queries)
334
+ incompleted_block_hit = (incompleted_hit_rate * incomplete_ratio)
335
+ return (completed_block_hit + incompleted_block_hit) / total_blocks
336
+
337
+
338
def get_all_blocks_recursively(last_block: Block) -> List[Block]:
339
"""Retrieves all the blocks in a sequence starting from the last block.
340
vllm/core/block/cpu_gpu_block_allocator.py
+5
@@ -323,6 +323,11 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
323
def all_block_ids(self) -> FrozenSet[int]:
324
return frozenset(self._block_ids_to_allocator.keys())
325
326
+ def get_prefix_cache_hit_rate(self, device: Device) -> float:
327
+ """Prefix cache hit rate. -1 means not supported or disabled."""
328
+ assert device in self._allocators
329
+ return self._allocators[device].get_prefix_cache_hit_rate()
330
+
331
def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
332
"""Returns and clears the mapping of source to destination block IDs.
333
Will be called after every swapping operations for now, and after every
vllm/core/block/interfaces.py
+10
@@ -186,6 +186,11 @@ class BlockAllocator(ABC):
186
num_lookahead_slots: int = 0) -> int:
187
pass
188
189
+ @abstractmethod
190
+ def get_prefix_cache_hit_rate(self) -> float:
191
+ """Prefix cache hit rate. -1 means not supported or disabled."""
192
+ pass
193
+
194
class NoFreeBlocksError(ValueError):
195
pass
196
@@ -278,3 +283,8 @@ class DeviceAwareBlockAllocator(ABC):
283
There is at most one null block per allocator.
284
"""
285
pass
286
+
287
+ @abstractmethod
288
+ def get_prefix_cache_hit_rate(self, device: Device) -> float:
289
+ """Prefix cache hit rate. -1 means not supported or disabled."""
290
+ pass
vllm/core/block/naive_block.py
+3
@@ -341,6 +341,9 @@ class NaiveBlockAllocator(BlockAllocator):
341
342
block.block_id = block_id # Assign block_id
343
344
+ def get_prefix_cache_hit_rate(self) -> float:
345
+ return -1
346
+
347
348
class NaiveBlock(Block):
349
"""An implementation of the Block class that does not support prefix
vllm/core/block/prefix_caching_block.py
+8
-2
@@ -1,9 +1,8 @@
1
"""Token blocks."""
2
-
2
from os.path import commonprefix
3
from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
4
6
-from vllm.core.block.common import (CopyOnWriteTracker,
5
+from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
6
get_all_blocks_recursively)
7
from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
8
from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
@@ -107,6 +106,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
106
self._cow_tracker = CopyOnWriteTracker(
107
refcounter=self._refcounter.as_readonly())
108
109
+ self.metric_data = CacheMetricData()
110
+
111
# Implements Block.Factory.
112
def _create_block(
113
self,
@@ -155,9 +156,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
156
157
cached_block_id = self._cached_blocks.get(block.content_hash, None)
158
if cached_block_id is not None:
159
+ self.metric_data.query(hit=True)
160
block.block_id = cached_block_id
161
self._incr_refcount_cached_block(block)
162
return block
163
+ self.metric_data.query(hit=False)
164
self._block_pool.free_block(block)
165
166
# No cached block => Allocate a new block
@@ -404,6 +407,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
407
def all_block_ids(self) -> FrozenSet[int]:
408
return self._hashless_allocator.all_block_ids
409
410
+ def get_prefix_cache_hit_rate(self) -> float:
411
+ return self.metric_data.get_hit_rate()
412
+
413
def is_block_cached(self, block: Block) -> bool:
414
assert block.content_hash is not None
415
if block.content_hash in self._cached_blocks:
vllm/core/block_manager_v1.py
+27
-4
@@ -8,6 +8,7 @@ from typing import Sequence as GenericSequence
8
from typing import Set, Tuple
9
10
from vllm.block import BlockTable, PhysicalTokenBlock
11
+from vllm.core.block.common import CacheMetricData
12
from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
13
from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
14
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
@@ -60,6 +61,11 @@ class BlockAllocatorBase(ABC):
61
def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
62
pass
63
64
+ @abstractmethod
65
+ def get_prefix_cache_hit_rate(self) -> float:
66
+ """Prefix cache hit rate. -1 means not supported or disabled."""
67
+ pass
68
+
69
70
class CachedBlockAllocator(BlockAllocatorBase):
71
"""Manages free physical token blocks for a device.
@@ -85,6 +91,8 @@ class CachedBlockAllocator(BlockAllocatorBase):
91
92
self.default_hash_ctr = count()
93
94
+ self.cache_metric_data = CacheMetricData()
95
+
96
def allocate_block(self, block_hash: int,
97
num_hashed_tokens: int) -> PhysicalTokenBlock:
98
if self.current_num_blocks == self.num_blocks:
@@ -105,15 +113,17 @@ class CachedBlockAllocator(BlockAllocatorBase):
113
num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
114
if block_hash is None:
115
block_hash = next(self.default_hash_ctr)
116
+
117
if block_hash in self.evictor:
118
assert block_hash not in self.cached_blocks
119
block = self.evictor.remove(block_hash)
120
assert block.ref_count == 0
121
self.cached_blocks[block_hash] = block
113
- block.ref_count += 1
114
- assert block.block_hash == block_hash
115
- return block
116
- if block_hash not in self.cached_blocks:
122
+
123
+ if block_hash in self.cached_blocks:
124
+ self.cache_metric_data.query(hit=True)
125
+ else:
126
+ self.cache_metric_data.query(hit=False)
127
self.cached_blocks[block_hash] = self.allocate_block(
128
block_hash, num_hashed_tokens)
129
block = self.cached_blocks[block_hash]
@@ -150,6 +160,9 @@ class CachedBlockAllocator(BlockAllocatorBase):
160
del self.cached_blocks[old_hash]
161
self.cached_blocks[block_hash] = block
162
163
+ def get_prefix_cache_hit_rate(self) -> float:
164
+ return self.cache_metric_data.get_hit_rate()
165
+
166
167
class UncachedBlockAllocator(BlockAllocatorBase):
168
"""Manages free physical token blocks for a device.
@@ -209,6 +222,9 @@ class UncachedBlockAllocator(BlockAllocatorBase):
222
raise NotImplementedError(
223
"Invalid codepath for uncached block allocator.")
224
225
+ def get_prefix_cache_hit_rate(self) -> float:
226
+ return -1
227
+
228
229
class BlockSpaceManagerV1(BlockSpaceManager):
230
"""Manages the mapping between logical and physical token blocks."""
@@ -705,3 +721,10 @@ class BlockSpaceManagerV1(BlockSpaceManager):
721
if self.enable_caching:
722
for seq in seq_group.get_seqs():
723
self.compute_full_blocks_in_seq(seq)
724
+
725
+ def get_prefix_cache_hit_rate(self, device: Device) -> float:
726
+ if device == Device.GPU:
727
+ return self.gpu_allocator.get_prefix_cache_hit_rate()
728
+ if device == Device.CPU:
729
+ return self.cpu_allocator.get_prefix_cache_hit_rate()
730
+ raise ValueError(f"Invalid device: {device}")
vllm/core/block_manager_v2.py
+3
@@ -441,6 +441,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
441
def get_num_free_cpu_blocks(self) -> int:
442
return self.block_allocator.get_num_free_blocks(Device.CPU)
443
444
+ def get_prefix_cache_hit_rate(self, device: Device) -> float:
445
+ return self.block_allocator.get_prefix_cache_hit_rate(device)
446
+
447
def _can_swap(self,
448
seq_group: SequenceGroup,
449
device: Device,
vllm/core/embedding_model_block_manager.py
+4
@@ -2,6 +2,7 @@ from typing import List, Tuple
2
3
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
4
from vllm.sequence import Sequence, SequenceGroup
5
+from vllm.utils import Device
6
7
8
class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
@@ -81,3 +82,6 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
82
83
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
84
pass
85
+
86
+ def get_prefix_cache_hit_rate(self, device: Device) -> float:
87
+ return -1
vllm/core/evictor_v2.py
+8
-7
@@ -85,19 +85,21 @@ class LRUEvictor(Evictor):
85
if len(self.free_table) == 0:
86
raise ValueError("No usable cache memory left")
87
88
- evicted_block = next(iter(self.free_table.values()))
89
- evicted_block_id = next(iter(self.free_table.keys()))
88
+ evicted_block, evicted_block_id = None, None
89
# The blocks with the lowest timestamps should be placed consecutively
90
# at the start of OrderedDict. Loop through all these blocks to
91
# find the one with maximum number of hashed tokens.
92
for _id, block in self.free_table.items():
93
+ if evicted_block is None:
94
+ evicted_block, evicted_block_id = block, _id
95
+ continue
96
if evicted_block.last_accessed < block.last_accessed:
97
break
96
- if (evicted_block.last_accessed == block.last_accessed and
97
- evicted_block.num_hashed_tokens < block.num_hashed_tokens):
98
- evicted_block = block
99
- evicted_block_id = _id
98
+ if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
99
+ evicted_block, evicted_block_id = block, _id
100
101
+ assert evicted_block is not None
102
+ assert evicted_block_id is not None
103
self.free_table.pop(evicted_block_id)
104
105
return evicted_block_id, evicted_block.content_hash
@@ -110,7 +112,6 @@ class LRUEvictor(Evictor):
112
113
def update(self, block_id: int, last_accessed: float):
114
self.free_table[block_id].last_accessed = last_accessed
113
- self.free_table.move_to_end(block_id)
115
116
def remove(self, block_id: int):
117
if block_id not in self.free_table:
vllm/core/interfaces.py
+6
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5
from typing import Tuple
6
7
from vllm.sequence import Sequence, SequenceGroup
8
+from vllm.utils import Device
9
10
11
class AllocStatus(enum.Enum):
@@ -116,3 +117,8 @@ class BlockSpaceManager(ABC):
117
@abstractmethod
118
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
119
pass
120
+
121
+ @abstractmethod
122
+ def get_prefix_cache_hit_rate(self, device: Device) -> float:
123
+ """Prefix cache hit rate. -1 means not supported or disabled."""
124
+ pass
vllm/core/scheduler.py
+4
-1
@@ -14,7 +14,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
15
SequenceGroupMetadata, SequenceGroupMetadataDelta,
16
SequenceStatus)
17
-from vllm.utils import PyObjectCache
17
+from vllm.utils import Device, PyObjectCache
18
19
logger = init_logger(__name__)
20
@@ -447,6 +447,9 @@ class Scheduler:
447
return len(self.waiting) != 0 or len(self.running) != 0 or len(
448
self.swapped) != 0
449
450
+ def get_prefix_cache_hit_rate(self, device: Device) -> float:
451
+ return self.block_manager.get_prefix_cache_hit_rate(device)
452
+
453
def get_num_unfinished_seq_groups(self) -> int:
454
return len(self.waiting) + len(self.running) + len(self.swapped)
455
vllm/engine/llm_engine.py
+11
-1
@@ -47,7 +47,7 @@ from vllm.transformers_utils.tokenizer_group import (
47
AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
48
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
49
usage_message)
50
-from vllm.utils import Counter
50
+from vllm.utils import Counter, Device
51
from vllm.version import __version__ as VLLM_VERSION
52
53
logger = init_logger(__name__)
@@ -1390,6 +1390,13 @@ class LLMEngine:
1390
for scheduler in self.scheduler)
1391
cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
1392
1393
+ # Prefix Cache Hit Rate. Note that we always use
1394
+ # the cache hit rate of the first virtual engine.
1395
+ cpu_prefix_cache_hit_rate = self.scheduler[
1396
+ 0].get_prefix_cache_hit_rate(Device.CPU)
1397
+ gpu_prefix_cache_hit_rate = self.scheduler[
1398
+ 0].get_prefix_cache_hit_rate(Device.GPU)
1399
+
1400
# Iteration stats
1401
num_prompt_tokens_iter = 0
1402
num_generation_tokens_iter = 0
@@ -1498,6 +1505,9 @@ class LLMEngine:
1505
# KV Cache Usage in %
1506
gpu_cache_usage_sys=gpu_cache_usage_sys,
1507
cpu_cache_usage_sys=cpu_cache_usage_sys,
1508
+ # Prefix Cache Hit Rate
1509
+ cpu_prefix_cache_hit_rate=cpu_prefix_cache_hit_rate,
1510
+ gpu_prefix_cache_hit_rate=gpu_prefix_cache_hit_rate,
1511
1512
# Iteration stats
1513
num_prompt_tokens_iter=num_prompt_tokens_iter,
vllm/engine/metrics.py
+22
-1
@@ -71,6 +71,17 @@ class Metrics:
71
documentation="CPU KV-cache usage. 1 means 100 percent usage.",
72
labelnames=labelnames,
73
multiprocess_mode="sum")
74
+ # Prefix caching block hit rate
75
+ self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
76
+ name="vllm:cpu_prefix_cache_hit_rate",
77
+ documentation="CPU prefix cache block hit rate.",
78
+ labelnames=labelnames,
79
+ multiprocess_mode="sum")
80
+ self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
81
+ name="vllm:gpu_prefix_cache_hit_rate",
82
+ documentation="GPU prefix cache block hit rate.",
83
+ labelnames=labelnames,
84
+ multiprocess_mode="sum")
85
86
# Iteration stats
87
self.counter_num_preemption = self._counter_cls(
@@ -351,7 +362,13 @@ class LoggingStatLogger(StatLoggerBase):
362
stats.gpu_cache_usage_sys * 100,
363
stats.cpu_cache_usage_sys * 100,
364
)
354
-
365
+ if (stats.cpu_prefix_cache_hit_rate >= 0
366
+ or stats.gpu_prefix_cache_hit_rate >= 0):
367
+ logger.info(
368
+ "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
369
+ stats.gpu_prefix_cache_hit_rate * 100,
370
+ stats.cpu_prefix_cache_hit_rate * 100,
371
+ )
372
if self.spec_decode_metrics is not None:
373
logger.info(
374
self._format_spec_decode_metrics_str(
@@ -423,6 +440,10 @@ class PrometheusStatLogger(StatLoggerBase):
440
stats.gpu_cache_usage_sys)
441
self._log_gauge(self.metrics.gauge_cpu_cache_usage,
442
stats.cpu_cache_usage_sys)
443
+ self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
444
+ stats.cpu_prefix_cache_hit_rate)
445
+ self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
446
+ stats.gpu_prefix_cache_hit_rate)
447
448
# Iteration level data
449
self._log_counter(self.metrics.counter_num_preemption,
vllm/engine/metrics_types.py
+3
@@ -32,6 +32,9 @@ class Stats:
32
# KV Cache Usage in %
33
gpu_cache_usage_sys: float
34
cpu_cache_usage_sys: float
35
+ # Prefix caching block hit rate
36
+ cpu_prefix_cache_hit_rate: float
37
+ gpu_prefix_cache_hit_rate: float
38
39
# Iteration stats (should have _iter suffix)
40
num_prompt_tokens_iter: int