@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ac50b47d0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[MISC] Add prefix cache hit rate to metrics (#7606)

Cody Yu committed Aug 19, 2024 at 11:52 UTC 3ac50b47d0f718364d05b1bcb93743e15be6a37c
16 files changed +200 -16
tests/core/block/test_prefix_caching_block.py
+26
@@ -682,6 +682,32 @@ class TestPrefixCachingBlockAllocator:
682
683 assert new_block[0].block_id == last_block_id
684
685 + # Test case for cache mertics
686 + @staticmethod
687 + def test_metric():
688 + block_size = 16
689 + allocator = PrefixCachingBlockAllocator(num_blocks=4,
690 + block_size=block_size)
691 + # Test when no query (0/0)
692 + assert allocator.get_prefix_cache_hit_rate() == 0.0
693 +
694 + token_ids = list(range(block_size))
695 + allocator.allocate_immutable_block(prev_block=None,
696 + token_ids=token_ids)
697 + # Test 0/1 hit rate
698 + assert allocator.get_prefix_cache_hit_rate() == 0.0
699 +
700 + allocator.allocate_immutable_block(prev_block=None,
701 + token_ids=token_ids)
702 + # Test 1/2 hit rate
703 + assert allocator.get_prefix_cache_hit_rate() == 0.5
704 +
705 + # Test more than one block
706 + for _ in range(2, 1005):
707 + allocator.allocate_immutable_block(prev_block=None,
708 + token_ids=token_ids)
709 + assert allocator.get_prefix_cache_hit_rate() > 0.99
710 +
711 @staticmethod
712 def create_immutable_chain(
713 block_size: int,
tests/prefix_caching/test_prefix_caching.py
+7
@@ -34,6 +34,9 @@ def test_block_allocator(
34 assert (first_block == second_block)
35 assert (second_block.ref_count == 2)
36
37 + # Check metric: 1 hit of 2 queries
38 + assert block_allocator.get_prefix_cache_hit_rate() == 0.5
39 +
40 # Free the first_block and confirm that the ref_count is correctly
41 # decremented on the second block
42 block_allocator.free(first_block)
@@ -48,6 +51,10 @@ def test_block_allocator(
51 assert (first_block == second_block)
52 assert (first_block.block_hash == block_hash)
53
54 + # Allocate one more time to get 3/4 hit rate for easy checking
55 + block_allocator.allocate(block_hash, 0)
56 + assert block_allocator.get_prefix_cache_hit_rate() == 0.75
57 +
58
59 @pytest.mark.parametrize("num_blocks", [16])
60 def test_eviction(num_blocks: int, ):
vllm/core/block/common.py
+53
@@ -1,4 +1,5 @@
1 from collections import deque
2 +from dataclasses import dataclass
3 from typing import Deque, Dict, Iterable, List, Optional, Protocol, Tuple
4
5 from vllm.core.block.interfaces import Block, BlockAllocator
@@ -282,6 +283,58 @@ class BlockList:
283 return self._block_ids
284
285
286 +@dataclass
287 +class CacheMetricData:
288 + """A utility dataclass to maintain cache metric.
289 + To avoid overflow, we maintain the hit rate in block granularity, so that
290 + we can maintain a single hit rate for n_completed_block x block_size,
291 + and calculate the real time hit rate by the following:
292 + BS = The number of queries per block.
293 + nB = The number of completed blocks.
294 + HR = hit rate of (nB x BS) queries.
295 + Q = current number of queries (< BS).
296 + H = current number of hits (< BS).
297 + hit rate = ((HR x nB) + (H / Q) x (Q / BS)) / (nB + Q / BS)
298 + """
299 + num_completed_blocks: int = 0
300 + completed_block_cache_hit_rate: float = 0.0
301 + num_incompleted_block_queries: int = 0
302 + num_incompleted_block_hit: int = 0
303 + block_size: int = 1000
304 +
305 + def query(self, hit: bool):
306 + self.num_incompleted_block_queries += 1
307 + self.num_incompleted_block_hit += 1 if hit else 0
308 +
309 + # When a block is completed, update the cache hit rate
310 + # and reset the incomplete numbers.
311 + if self.num_incompleted_block_queries == self.block_size:
312 + hit_rate = (self.num_incompleted_block_hit /
313 + self.num_incompleted_block_queries)
314 + self.completed_block_cache_hit_rate = (
315 + self.completed_block_cache_hit_rate * self.num_completed_blocks
316 + + hit_rate) / (self.num_completed_blocks + 1)
317 + self.num_incompleted_block_queries = 0
318 + self.num_incompleted_block_hit = 0
319 + self.num_completed_blocks += 1
320 +
321 + def get_hit_rate(self):
322 + incomplete_ratio = self.num_incompleted_block_queries / self.block_size
323 + total_blocks = self.num_completed_blocks + incomplete_ratio
324 + if total_blocks == 0:
325 + return 0.0
326 +
327 + completed_block_hit, incompleted_block_hit = 0.0, 0.0
328 + if self.num_completed_blocks > 0:
329 + completed_block_hit = (self.completed_block_cache_hit_rate *
330 + self.num_completed_blocks)
331 + if self.num_incompleted_block_queries > 0:
332 + incompleted_hit_rate = (self.num_incompleted_block_hit /
333 + self.num_incompleted_block_queries)
334 + incompleted_block_hit = (incompleted_hit_rate * incomplete_ratio)
335 + return (completed_block_hit + incompleted_block_hit) / total_blocks
336 +
337 +
338 def get_all_blocks_recursively(last_block: Block) -> List[Block]:
339 """Retrieves all the blocks in a sequence starting from the last block.
340
vllm/core/block/cpu_gpu_block_allocator.py
+5
@@ -323,6 +323,11 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
323 def all_block_ids(self) -> FrozenSet[int]:
324 return frozenset(self._block_ids_to_allocator.keys())
325
326 + def get_prefix_cache_hit_rate(self, device: Device) -> float:
327 + """Prefix cache hit rate. -1 means not supported or disabled."""
328 + assert device in self._allocators
329 + return self._allocators[device].get_prefix_cache_hit_rate()
330 +
331 def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
332 """Returns and clears the mapping of source to destination block IDs.
333 Will be called after every swapping operations for now, and after every
vllm/core/block/interfaces.py
+10
@@ -186,6 +186,11 @@ class BlockAllocator(ABC):
186 num_lookahead_slots: int = 0) -> int:
187 pass
188
189 + @abstractmethod
190 + def get_prefix_cache_hit_rate(self) -> float:
191 + """Prefix cache hit rate. -1 means not supported or disabled."""
192 + pass
193 +
194 class NoFreeBlocksError(ValueError):
195 pass
196
@@ -278,3 +283,8 @@ class DeviceAwareBlockAllocator(ABC):
283 There is at most one null block per allocator.
284 """
285 pass
286 +
287 + @abstractmethod
288 + def get_prefix_cache_hit_rate(self, device: Device) -> float:
289 + """Prefix cache hit rate. -1 means not supported or disabled."""
290 + pass
vllm/core/block/naive_block.py
+3
@@ -341,6 +341,9 @@ class NaiveBlockAllocator(BlockAllocator):
341
342 block.block_id = block_id # Assign block_id
343
344 + def get_prefix_cache_hit_rate(self) -> float:
345 + return -1
346 +
347
348 class NaiveBlock(Block):
349 """An implementation of the Block class that does not support prefix
vllm/core/block/prefix_caching_block.py
+8 -2
@@ -1,9 +1,8 @@
1 """Token blocks."""
2 -
2 from os.path import commonprefix
3 from typing import Dict, FrozenSet, Iterable, List, Optional, Tuple
4
6 -from vllm.core.block.common import (CopyOnWriteTracker,
5 +from vllm.core.block.common import (CacheMetricData, CopyOnWriteTracker,
6 get_all_blocks_recursively)
7 from vllm.core.block.interfaces import Block, BlockAllocator, BlockId, Device
8 from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
@@ -107,6 +106,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
106 self._cow_tracker = CopyOnWriteTracker(
107 refcounter=self._refcounter.as_readonly())
108
109 + self.metric_data = CacheMetricData()
110 +
111 # Implements Block.Factory.
112 def _create_block(
113 self,
@@ -155,9 +156,11 @@ class PrefixCachingBlockAllocator(BlockAllocator):
156
157 cached_block_id = self._cached_blocks.get(block.content_hash, None)
158 if cached_block_id is not None:
159 + self.metric_data.query(hit=True)
160 block.block_id = cached_block_id
161 self._incr_refcount_cached_block(block)
162 return block
163 + self.metric_data.query(hit=False)
164 self._block_pool.free_block(block)
165
166 # No cached block => Allocate a new block
@@ -404,6 +407,9 @@ class PrefixCachingBlockAllocator(BlockAllocator):
407 def all_block_ids(self) -> FrozenSet[int]:
408 return self._hashless_allocator.all_block_ids
409
410 + def get_prefix_cache_hit_rate(self) -> float:
411 + return self.metric_data.get_hit_rate()
412 +
413 def is_block_cached(self, block: Block) -> bool:
414 assert block.content_hash is not None
415 if block.content_hash in self._cached_blocks:
vllm/core/block_manager_v1.py
+27 -4
@@ -8,6 +8,7 @@ from typing import Sequence as GenericSequence
8 from typing import Set, Tuple
9
10 from vllm.block import BlockTable, PhysicalTokenBlock
11 +from vllm.core.block.common import CacheMetricData
12 from vllm.core.block.utils import check_no_caching_or_swa_for_blockmgr_encdec
13 from vllm.core.evictor_v1 import EvictionPolicy, Evictor, make_evictor
14 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
@@ -60,6 +61,11 @@ class BlockAllocatorBase(ABC):
61 def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
62 pass
63
64 + @abstractmethod
65 + def get_prefix_cache_hit_rate(self) -> float:
66 + """Prefix cache hit rate. -1 means not supported or disabled."""
67 + pass
68 +
69
70 class CachedBlockAllocator(BlockAllocatorBase):
71 """Manages free physical token blocks for a device.
@@ -85,6 +91,8 @@ class CachedBlockAllocator(BlockAllocatorBase):
91
92 self.default_hash_ctr = count()
93
94 + self.cache_metric_data = CacheMetricData()
95 +
96 def allocate_block(self, block_hash: int,
97 num_hashed_tokens: int) -> PhysicalTokenBlock:
98 if self.current_num_blocks == self.num_blocks:
@@ -105,15 +113,17 @@ class CachedBlockAllocator(BlockAllocatorBase):
113 num_hashed_tokens: int = 0) -> PhysicalTokenBlock:
114 if block_hash is None:
115 block_hash = next(self.default_hash_ctr)
116 +
117 if block_hash in self.evictor:
118 assert block_hash not in self.cached_blocks
119 block = self.evictor.remove(block_hash)
120 assert block.ref_count == 0
121 self.cached_blocks[block_hash] = block
113 - block.ref_count += 1
114 - assert block.block_hash == block_hash
115 - return block
116 - if block_hash not in self.cached_blocks:
122 +
123 + if block_hash in self.cached_blocks:
124 + self.cache_metric_data.query(hit=True)
125 + else:
126 + self.cache_metric_data.query(hit=False)
127 self.cached_blocks[block_hash] = self.allocate_block(
128 block_hash, num_hashed_tokens)
129 block = self.cached_blocks[block_hash]
@@ -150,6 +160,9 @@ class CachedBlockAllocator(BlockAllocatorBase):
160 del self.cached_blocks[old_hash]
161 self.cached_blocks[block_hash] = block
162
163 + def get_prefix_cache_hit_rate(self) -> float:
164 + return self.cache_metric_data.get_hit_rate()
165 +
166
167 class UncachedBlockAllocator(BlockAllocatorBase):
168 """Manages free physical token blocks for a device.
@@ -209,6 +222,9 @@ class UncachedBlockAllocator(BlockAllocatorBase):
222 raise NotImplementedError(
223 "Invalid codepath for uncached block allocator.")
224
225 + def get_prefix_cache_hit_rate(self) -> float:
226 + return -1
227 +
228
229 class BlockSpaceManagerV1(BlockSpaceManager):
230 """Manages the mapping between logical and physical token blocks."""
@@ -705,3 +721,10 @@ class BlockSpaceManagerV1(BlockSpaceManager):
721 if self.enable_caching:
722 for seq in seq_group.get_seqs():
723 self.compute_full_blocks_in_seq(seq)
724 +
725 + def get_prefix_cache_hit_rate(self, device: Device) -> float:
726 + if device == Device.GPU:
727 + return self.gpu_allocator.get_prefix_cache_hit_rate()
728 + if device == Device.CPU:
729 + return self.cpu_allocator.get_prefix_cache_hit_rate()
730 + raise ValueError(f"Invalid device: {device}")
vllm/core/block_manager_v2.py
+3
@@ -441,6 +441,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
441 def get_num_free_cpu_blocks(self) -> int:
442 return self.block_allocator.get_num_free_blocks(Device.CPU)
443
444 + def get_prefix_cache_hit_rate(self, device: Device) -> float:
445 + return self.block_allocator.get_prefix_cache_hit_rate(device)
446 +
447 def _can_swap(self,
448 seq_group: SequenceGroup,
449 device: Device,
vllm/core/embedding_model_block_manager.py
+4
@@ -2,6 +2,7 @@ from typing import List, Tuple
2
3 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
4 from vllm.sequence import Sequence, SequenceGroup
5 +from vllm.utils import Device
6
7
8 class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
@@ -81,3 +82,6 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
82
83 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
84 pass
85 +
86 + def get_prefix_cache_hit_rate(self, device: Device) -> float:
87 + return -1
vllm/core/evictor_v2.py
+8 -7
@@ -85,19 +85,21 @@ class LRUEvictor(Evictor):
85 if len(self.free_table) == 0:
86 raise ValueError("No usable cache memory left")
87
88 - evicted_block = next(iter(self.free_table.values()))
89 - evicted_block_id = next(iter(self.free_table.keys()))
88 + evicted_block, evicted_block_id = None, None
89 # The blocks with the lowest timestamps should be placed consecutively
90 # at the start of OrderedDict. Loop through all these blocks to
91 # find the one with maximum number of hashed tokens.
92 for _id, block in self.free_table.items():
93 + if evicted_block is None:
94 + evicted_block, evicted_block_id = block, _id
95 + continue
96 if evicted_block.last_accessed < block.last_accessed:
97 break
96 - if (evicted_block.last_accessed == block.last_accessed and
97 - evicted_block.num_hashed_tokens < block.num_hashed_tokens):
98 - evicted_block = block
99 - evicted_block_id = _id
98 + if evicted_block.num_hashed_tokens < block.num_hashed_tokens:
99 + evicted_block, evicted_block_id = block, _id
100
101 + assert evicted_block is not None
102 + assert evicted_block_id is not None
103 self.free_table.pop(evicted_block_id)
104
105 return evicted_block_id, evicted_block.content_hash
@@ -110,7 +112,6 @@ class LRUEvictor(Evictor):
112
113 def update(self, block_id: int, last_accessed: float):
114 self.free_table[block_id].last_accessed = last_accessed
113 - self.free_table.move_to_end(block_id)
115
116 def remove(self, block_id: int):
117 if block_id not in self.free_table:
vllm/core/interfaces.py
+6
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5 from typing import Tuple
6
7 from vllm.sequence import Sequence, SequenceGroup
8 +from vllm.utils import Device
9
10
11 class AllocStatus(enum.Enum):
@@ -116,3 +117,8 @@ class BlockSpaceManager(ABC):
117 @abstractmethod
118 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
119 pass
120 +
121 + @abstractmethod
122 + def get_prefix_cache_hit_rate(self, device: Device) -> float:
123 + """Prefix cache hit rate. -1 means not supported or disabled."""
124 + pass
vllm/core/scheduler.py
+4 -1
@@ -14,7 +14,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
14 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
15 SequenceGroupMetadata, SequenceGroupMetadataDelta,
16 SequenceStatus)
17 -from vllm.utils import PyObjectCache
17 +from vllm.utils import Device, PyObjectCache
18
19 logger = init_logger(__name__)
20
@@ -447,6 +447,9 @@ class Scheduler:
447 return len(self.waiting) != 0 or len(self.running) != 0 or len(
448 self.swapped) != 0
449
450 + def get_prefix_cache_hit_rate(self, device: Device) -> float:
451 + return self.block_manager.get_prefix_cache_hit_rate(device)
452 +
453 def get_num_unfinished_seq_groups(self) -> int:
454 return len(self.waiting) + len(self.running) + len(self.swapped)
455
vllm/engine/llm_engine.py
+11 -1
@@ -47,7 +47,7 @@ from vllm.transformers_utils.tokenizer_group import (
47 AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
48 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
49 usage_message)
50 -from vllm.utils import Counter
50 +from vllm.utils import Counter, Device
51 from vllm.version import __version__ as VLLM_VERSION
52
53 logger = init_logger(__name__)
@@ -1390,6 +1390,13 @@ class LLMEngine:
1390 for scheduler in self.scheduler)
1391 cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
1392
1393 + # Prefix Cache Hit Rate. Note that we always use
1394 + # the cache hit rate of the first virtual engine.
1395 + cpu_prefix_cache_hit_rate = self.scheduler[
1396 + 0].get_prefix_cache_hit_rate(Device.CPU)
1397 + gpu_prefix_cache_hit_rate = self.scheduler[
1398 + 0].get_prefix_cache_hit_rate(Device.GPU)
1399 +
1400 # Iteration stats
1401 num_prompt_tokens_iter = 0
1402 num_generation_tokens_iter = 0
@@ -1498,6 +1505,9 @@ class LLMEngine:
1505 # KV Cache Usage in %
1506 gpu_cache_usage_sys=gpu_cache_usage_sys,
1507 cpu_cache_usage_sys=cpu_cache_usage_sys,
1508 + # Prefix Cache Hit Rate
1509 + cpu_prefix_cache_hit_rate=cpu_prefix_cache_hit_rate,
1510 + gpu_prefix_cache_hit_rate=gpu_prefix_cache_hit_rate,
1511
1512 # Iteration stats
1513 num_prompt_tokens_iter=num_prompt_tokens_iter,
vllm/engine/metrics.py
+22 -1
@@ -71,6 +71,17 @@ class Metrics:
71 documentation="CPU KV-cache usage. 1 means 100 percent usage.",
72 labelnames=labelnames,
73 multiprocess_mode="sum")
74 + # Prefix caching block hit rate
75 + self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
76 + name="vllm:cpu_prefix_cache_hit_rate",
77 + documentation="CPU prefix cache block hit rate.",
78 + labelnames=labelnames,
79 + multiprocess_mode="sum")
80 + self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
81 + name="vllm:gpu_prefix_cache_hit_rate",
82 + documentation="GPU prefix cache block hit rate.",
83 + labelnames=labelnames,
84 + multiprocess_mode="sum")
85
86 # Iteration stats
87 self.counter_num_preemption = self._counter_cls(
@@ -351,7 +362,13 @@ class LoggingStatLogger(StatLoggerBase):
362 stats.gpu_cache_usage_sys * 100,
363 stats.cpu_cache_usage_sys * 100,
364 )
354 -
365 + if (stats.cpu_prefix_cache_hit_rate >= 0
366 + or stats.gpu_prefix_cache_hit_rate >= 0):
367 + logger.info(
368 + "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
369 + stats.gpu_prefix_cache_hit_rate * 100,
370 + stats.cpu_prefix_cache_hit_rate * 100,
371 + )
372 if self.spec_decode_metrics is not None:
373 logger.info(
374 self._format_spec_decode_metrics_str(
@@ -423,6 +440,10 @@ class PrometheusStatLogger(StatLoggerBase):
440 stats.gpu_cache_usage_sys)
441 self._log_gauge(self.metrics.gauge_cpu_cache_usage,
442 stats.cpu_cache_usage_sys)
443 + self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
444 + stats.cpu_prefix_cache_hit_rate)
445 + self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
446 + stats.gpu_prefix_cache_hit_rate)
447
448 # Iteration level data
449 self._log_counter(self.metrics.counter_num_preemption,
vllm/engine/metrics_types.py
+3
@@ -32,6 +32,9 @@ class Stats:
32 # KV Cache Usage in %
33 gpu_cache_usage_sys: float
34 cpu_cache_usage_sys: float
35 + # Prefix caching block hit rate
36 + cpu_prefix_cache_hit_rate: float
37 + gpu_prefix_cache_hit_rate: float
38
39 # Iteration stats (should have _iter suffix)
40 num_prompt_tokens_iter: int