@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7206ce4ce1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Core] Support `reset_prefix_cache` (#12284)

Cody Yu committed Jan 22, 2025 at 10:52 UTC 7206ce4ce112ed117796a59045c968a6d353f691
27 files changed +300 -21
tests/core/block/test_prefix_caching_block.py
+38
@@ -796,6 +796,44 @@ class TestPrefixCachingBlockAllocator:
796 block_hashes=block_hashes_seq1)
797 assert len(cached_blocks) == len(blocks_seq1) - num_evicted_blocks
798
799 + # Test reset prefix cache
800 + @staticmethod
801 + @pytest.mark.parametrize("num_blocks", [10])
802 + @pytest.mark.parametrize("block_size", [16])
803 + def test_reset_prefix_cache(num_blocks: int, block_size: int):
804 + """This test case simulates the case of resetting the prefix cache."""
805 +
806 + allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
807 + block_size=block_size)
808 + token_ids = list(range(3 * block_size))
809 +
810 + first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
811 + block_size=block_size,
812 + token_ids=token_ids,
813 + allocator=allocator,
814 + )
815 + second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
816 + block_size=block_size,
817 + token_ids=token_ids,
818 + allocator=allocator,
819 + )
820 +
821 + # Free each block in the first chain.
822 + for block in first_chain:
823 + allocator.free(block)
824 +
825 + # Failed to reset prefix cache because some blocks are not freed yet.
826 + assert not allocator.reset_prefix_cache()
827 + assert allocator.get_prefix_cache_hit_rate() > 0.0
828 +
829 + # Free each block in the second chain.
830 + for block in second_chain:
831 + allocator.free(block)
832 +
833 + # Reset prefix cache.
834 + assert allocator.reset_prefix_cache()
835 + assert allocator.get_prefix_cache_hit_rate() == 0.0
836 +
837 @staticmethod
838 def create_immutable_chain(
839 block_size: int,
tests/v1/core/test_prefix_caching.py
+39
@@ -587,3 +587,42 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
587 assert {block.ref_cnt for block in block_part1[:3]} == {1}
588 # Block 3-5 are free.
589 assert {block.ref_cnt for block in block_part1[3:]} == {0}
590 +
591 +
592 +def test_reset_prefix_cache():
593 + manager = KVCacheManager(
594 + block_size=16,
595 + num_gpu_blocks=10,
596 + max_model_len=8192,
597 + sliding_window=None,
598 + enable_caching=True,
599 + num_preallocate_tokens=0,
600 + )
601 +
602 + full_block_token_ids = [i for i in range(3) for _ in range(16)]
603 + unique_token_ids = [3] * 7
604 + all_token_ids = full_block_token_ids + unique_token_ids
605 + req0 = make_request("0", all_token_ids)
606 + blocks = manager.allocate_slots(req0, 55, [])
607 + assert [b.block_id for b in blocks] == [0, 1, 2, 3]
608 +
609 + unique_token_ids = [4] * 7
610 + all_token_ids = full_block_token_ids + unique_token_ids
611 + req1 = make_request("1", all_token_ids)
612 + computed_blocks, _ = manager.get_computed_blocks(req1)
613 + assert len(req1.kv_block_hashes) == 3
614 + assert len(computed_blocks) == 3
615 + blocks = manager.allocate_slots(req1, 7, computed_blocks)
616 + assert [b.block_id for b in blocks] == [4]
617 +
618 + # Failed to reset prefix cache because some blocks are not freed yet.
619 + assert not manager.reset_prefix_cache()
620 + assert manager.cached_block_hash_to_block
621 +
622 + # Free the blocks.
623 + manager.free(req0)
624 + manager.free(req1)
625 +
626 + assert manager.reset_prefix_cache()
627 + assert not manager.cached_block_hash_to_block
628 + assert all([blk.block_hash is None for blk in manager.block_pool])
vllm/core/block/cpu_gpu_block_allocator.py
+7
@@ -339,6 +339,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
339 assert device in self._allocators
340 return self._allocators[device].get_prefix_cache_hit_rate()
341
342 + def reset_prefix_cache(self) -> bool:
343 + """Reset prefix cache for all devices."""
344 + success = True
345 + for allocator in self._allocators.values():
346 + success = success and allocator.reset_prefix_cache()
347 + return success
348 +
349 def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
350 """Returns and clears the mapping of source to destination block IDs.
351 Will be called after every swapping operations for now, and after every
vllm/core/block/interfaces.py
+10
@@ -192,6 +192,11 @@ class BlockAllocator(ABC):
192 """Prefix cache hit rate. -1 means not supported or disabled."""
193 pass
194
195 + @abstractmethod
196 + def reset_prefix_cache(self) -> bool:
197 + """Reset prefix cache."""
198 + pass
199 +
200 class NoFreeBlocksError(ValueError):
201 pass
202
@@ -297,6 +302,11 @@ class DeviceAwareBlockAllocator(ABC):
302 """Prefix cache hit rate. -1 means not supported or disabled."""
303 pass
304
305 + @abstractmethod
306 + def reset_prefix_cache(self) -> bool:
307 + """Reset prefix cache."""
308 + pass
309 +
310 @abstractmethod
311 def find_cached_blocks_prefix(
312 self,
vllm/core/block/naive_block.py
+14 -5
@@ -1,5 +1,5 @@
1 from collections import deque
2 -from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple
2 +from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple, Union
3
4 from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter,
5 get_all_blocks_recursively)
@@ -136,16 +136,18 @@ class NaiveBlockAllocator(BlockAllocator):
136 self._refcounter.incr(block_id)
137 return block_id
138
139 - def _free_block_id(self, block: Block) -> None:
140 - block_id = block.block_id
139 + def _free_block_id(self, block: Union[Block, BlockId]) -> None:
140 + if isinstance(block, Block):
141 + block_id = block.block_id
142 + block.block_id = None
143 + else:
144 + block_id = block
145 assert block_id is not None
146
147 refcount = self._refcounter.decr(block_id)
148 if refcount == 0:
149 self._free_block_indices.appendleft(block_id)
150
147 - block.block_id = None
148 -
151 def free(self, block: Block, keep_block_object: bool = False) -> None:
152 # Release the physical block id
153 self._free_block_id(block)
@@ -154,6 +156,9 @@ class NaiveBlockAllocator(BlockAllocator):
156 if not keep_block_object:
157 self._block_pool.free_block(block)
158
159 + def free_block_id(self, block_id: BlockId) -> None:
160 + self._free_block_id(block_id)
161 +
162 def fork(self, last_block: Block) -> List[Block]:
163 """Creates a new sequence of blocks that shares the same underlying
164 memory as the original sequence.
@@ -325,6 +330,10 @@ class NaiveBlockAllocator(BlockAllocator):
330 def get_prefix_cache_hit_rate(self) -> float:
331 return -1
332
333 + def reset_prefix_cache(self) -> bool:
334 + """No prefix cache for naive block allocator."""
335 + return True
336 +
337 def find_cached_blocks_prefix(self, block_hashes: List[int]) -> List[int]:
338 # Not applicable for naive block allocator.
339 return []
vllm/core/block/prefix_caching_block.py
+43 -1
@@ -12,6 +12,7 @@ from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId, Device,
12 from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
13 NaiveBlockAllocator)
14 from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
15 +from vllm.logger import init_logger
16 from vllm.sequence import Sequence
17
18 PrefixHash = int
@@ -21,6 +22,8 @@ PrefixHash = int
22 # then we know this block hasn't been accessed yet.
23 _DEFAULT_LAST_ACCESSED_TIME = -1
24
25 +logger = init_logger(__name__)
26 +
27
28 class BlockTracker:
29 """Used to track the status of a block inside the prefix caching allocator
@@ -105,7 +108,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
108
109 # Evitor used to maintain how we want to handle those computed blocks
110 # if we find memory pressure is high.
108 - self.evictor: Evictor = make_evictor(eviction_policy)
111 + self.eviction_policy = eviction_policy
112 + self.evictor: Evictor = make_evictor(self.eviction_policy)
113
114 # We share the refcounter between allocators. This allows us to promote
115 # blocks originally allocated in the hashless allocator to immutable
@@ -428,6 +432,44 @@ class PrefixCachingBlockAllocator(BlockAllocator):
432 def get_prefix_cache_hit_rate(self) -> float:
433 return self.metric_data.get_hit_rate()
434
435 + def reset_prefix_cache(self) -> bool:
436 + """Reset prefix cache. This function may be used in RLHF
437 + flows to invalid prefix caching after the weights are updated,
438 + or used for resetting prefix caching status for benchmarking.
439 +
440 + Returns:
441 + bool: True if the prefix cache is successfully reset,
442 + False otherwise.
443 + """
444 + num_used_blocks = (self.get_num_total_blocks() -
445 + self.get_num_free_blocks())
446 + if num_used_blocks > 0:
447 + logger.warning(
448 + "Failed to reset prefix cache because some "
449 + "blocks (%d) are not freed yet", num_used_blocks)
450 + return False
451 +
452 + # Free all blocks in the evictor.
453 + while (block_id :=
454 + self._maybe_allocate_evicted_block_id()) is not None:
455 + self._hashless_allocator.free_block_id(block_id)
456 +
457 + # Should not have any cached blocks because all blocks are evicted.
458 + assert not self._cached_blocks
459 +
460 + # Reset the evictor.
461 + self.evictor = make_evictor(self.eviction_policy)
462 +
463 + # Reset the block tracker.
464 + for block_id in self._block_tracker:
465 + self._block_tracker[block_id] = BlockTracker()
466 +
467 + # Reset the metrics.
468 + self.metric_data = CacheMetricData()
469 +
470 + logger.info("Successfully reset prefix cache")
471 + return True
472 +
473 def is_block_cached(self, block: Block) -> bool:
474 assert block.content_hash is not None
475 return block.content_hash in self._cached_blocks
vllm/core/block_manager.py
+3
@@ -455,6 +455,9 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
455 def get_prefix_cache_hit_rate(self, device: Device) -> float:
456 return self.block_allocator.get_prefix_cache_hit_rate(device)
457
458 + def reset_prefix_cache(self) -> bool:
459 + return self.block_allocator.reset_prefix_cache()
460 +
461 def _can_swap(self,
462 seq_group: SequenceGroup,
463 device: Device,
vllm/core/interfaces.py
+5
@@ -122,6 +122,11 @@ class BlockSpaceManager(ABC):
122 """Prefix cache hit rate. -1 means not supported or disabled."""
123 pass
124
125 + @abstractmethod
126 + def reset_prefix_cache(self) -> bool:
127 + """Reset prefix cache for all devices."""
128 + pass
129 +
130 @abstractmethod
131 def get_num_cached_tokens(self, seq: Sequence) -> int:
132 pass
vllm/core/placeholder_block_space_manager.py
+3
@@ -90,5 +90,8 @@ class PlaceholderBlockSpaceManager(BlockSpaceManager):
90 def get_prefix_cache_hit_rate(self, device: Device) -> float:
91 return -1
92
93 + def reset_prefix_cache(self) -> bool:
94 + return True
95 +
96 def get_num_cached_tokens(self, seq: Sequence) -> int:
97 return 0
vllm/core/scheduler.py
+3
@@ -504,6 +504,9 @@ class Scheduler:
504 def get_prefix_cache_hit_rate(self, device: Device) -> float:
505 return self.block_manager.get_prefix_cache_hit_rate(device)
506
507 + def reset_prefix_cache(self) -> bool:
508 + return self.block_manager.reset_prefix_cache()
509 +
510 def get_num_unfinished_seq_groups(self) -> int:
511 return len(self.waiting) + len(self.running) + len(self.swapped)
512
vllm/engine/async_llm_engine.py
+3
@@ -1182,6 +1182,9 @@ class AsyncLLMEngine(EngineClient):
1182 async def stop_profile(self) -> None:
1183 self.engine.stop_profile()
1184
1185 + async def reset_prefix_cache(self) -> None:
1186 + self.engine.reset_prefix_cache()
1187 +
1188 async def add_lora(self, lora_request: LoRARequest) -> None:
1189 self.engine.add_lora(lora_request)
1190
vllm/engine/llm_engine.py
+8
@@ -914,6 +914,14 @@ class LLMEngine:
914 """
915 return self.scheduler[virtual_engine].has_unfinished_seqs()
916
917 + def reset_prefix_cache(self) -> bool:
918 + """Reset prefix cache for all devices."""
919 +
920 + success = True
921 + for scheduler in self.scheduler:
922 + success = success and scheduler.reset_prefix_cache()
923 + return success
924 +
925 @staticmethod
926 def _process_sequence_group_outputs(
927 seq_group: SequenceGroup,
vllm/engine/multiprocessing/__init__.py
+6 -1
@@ -121,6 +121,10 @@ class RPCUProfileRequest(Enum):
121 STOP_PROFILE = 2
122
123
124 +class RPCResetPrefixCacheRequest(Enum):
125 + RESET_PREFIX_CACHE = 1
126 +
127 +
128 @dataclass
129 class RPCLoadAdapterRequest:
130 lora_request: LoRARequest
@@ -134,7 +138,8 @@ class RPCAdapterLoadedResponse:
138
139
140 RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest,
137 - RPCUProfileRequest, RPCLoadAdapterRequest]
141 + RPCUProfileRequest, RPCLoadAdapterRequest,
142 + RPCResetPrefixCacheRequest]
143
144 REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCAdapterLoadedResponse,
145 RPCError]
vllm/engine/multiprocessing/client.py
+10 -2
@@ -27,8 +27,9 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
27 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
28 RPCAdapterLoadedResponse, RPCError,
29 RPCLoadAdapterRequest,
30 - RPCProcessRequest, RPCStartupRequest,
31 - RPCStartupResponse,
30 + RPCProcessRequest,
31 + RPCResetPrefixCacheRequest,
32 + RPCStartupRequest, RPCStartupResponse,
33 RPCUProfileRequest)
34 from vllm.engine.protocol import EngineClient
35 # yapf: enable
@@ -675,6 +676,13 @@ class MQLLMEngineClient(EngineClient):
676 await self._send_one_way_rpc_request(
677 request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
678
679 + async def reset_prefix_cache(self) -> None:
680 + """Reset the prefix cache"""
681 +
682 + await self._send_one_way_rpc_request(
683 + request=RPCResetPrefixCacheRequest.RESET_PREFIX_CACHE,
684 + socket=self.input_socket)
685 +
686 async def add_lora(self, lora_request: LoRARequest) -> None:
687 """Load a new LoRA adapter into the engine for future requests."""
688 # Uses the same I/O as generate requests
vllm/engine/multiprocessing/engine.py
+8 -2
@@ -16,8 +16,9 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
16 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
17 RPCAdapterLoadedResponse, RPCError,
18 RPCLoadAdapterRequest,
19 - RPCProcessRequest, RPCStartupRequest,
20 - RPCStartupResponse,
19 + RPCProcessRequest,
20 + RPCResetPrefixCacheRequest,
21 + RPCStartupRequest, RPCStartupResponse,
22 RPCUProfileRequest)
23 # yapf: enable
24 from vllm.logger import init_logger
@@ -237,6 +238,8 @@ class MQLLMEngine:
238 self.stop_profile()
239 elif isinstance(request, RPCLoadAdapterRequest):
240 self._handle_load_adapter_request(request)
241 + elif isinstance(request, RPCResetPrefixCacheRequest):
242 + self.reset_prefix_cache()
243 else:
244 raise ValueError("Unknown RPCRequest Type: "
245 f"{type(request)}")
@@ -361,6 +364,9 @@ class MQLLMEngine:
364 def stop_profile(self) -> None:
365 self.engine.stop_profile()
366
367 + def reset_prefix_cache(self) -> bool:
368 + return self.engine.reset_prefix_cache()
369 +
370
371 def signal_handler(*_) -> None:
372 raise KeyboardInterrupt("MQLLMEngine terminated")
vllm/engine/protocol.py
+5
@@ -271,6 +271,11 @@ class EngineClient(ABC):
271 """Start profiling the engine"""
272 ...
273
274 + @abstractmethod
275 + async def reset_prefix_cache(self) -> None:
276 + """Reset the prefix cache"""
277 + ...
278 +
279 @abstractmethod
280 async def add_lora(self, lora_request: LoRARequest) -> None:
281 """Load a new LoRA adapter into the engine for future requests."""
vllm/entrypoints/llm.py
+4
@@ -1132,6 +1132,9 @@ class LLM:
1132 def stop_profile(self) -> None:
1133 self.llm_engine.stop_profile()
1134
1135 + def reset_prefix_cache(self) -> bool:
1136 + return self.llm_engine.reset_prefix_cache()
1137 +
1138 def sleep(self, level: int = 1):
1139 """
1140 Put the engine to sleep. The engine should not process any requests.
@@ -1150,6 +1153,7 @@ class LLM:
1153 where previous model weights are not needed. It reduces CPU memory
1154 pressure.
1155 """
1156 + self.reset_prefix_cache()
1157 self.llm_engine.sleep(level=level)
1158
1159 def wake_up(self):
vllm/entrypoints/openai/api_server.py
+12
@@ -518,6 +518,18 @@ TASK_HANDLERS: Dict[str, Dict[str, tuple]] = {
518 },
519 }
520
521 +if envs.VLLM_SERVER_DEV_MODE:
522 +
523 + @router.post("/reset_prefix_cache")
524 + async def reset_prefix_cache(raw_request: Request):
525 + """
526 + Reset the prefix cache. Note that we currently do not check if the
527 + prefix cache is successfully reset in the API server.
528 + """
529 + logger.info("Resetting prefix cache...")
530 + await engine_client(raw_request).reset_prefix_cache()
531 + return Response(status_code=200)
532 +
533
534 @router.post("/invocations")
535 async def invocations(raw_request: Request):
vllm/envs.py
+7
@@ -72,6 +72,7 @@ if TYPE_CHECKING:
72 VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
73 VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
74 VLLM_DISABLE_COMPILE_CACHE: bool = False
75 + VLLM_SERVER_DEV_MODE: bool = False
76
77
78 def get_default_cache_root():
@@ -467,6 +468,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
468 lambda: float(os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")),
469 "VLLM_DISABLE_COMPILE_CACHE":
470 lambda: bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0"))),
471 +
472 + # If set, vllm will run in development mode, which will enable
473 + # some additional endpoints for developing and debugging,
474 + # e.g. `/reset_prefix_cache`
475 + "VLLM_SERVER_DEV_MODE":
476 + lambda: bool(int(os.getenv("VLLM_SERVER_DEV_MODE", "0"))),
477 }
478
479 # end-env-vars-definition
vllm/executor/executor_base.py
-5
@@ -194,11 +194,6 @@ class ExecutorBase(ABC):
194 self.collective_rpc("stop_profile")
195
196 def sleep(self, level: int = 1):
197 - if self.cache_config.enable_prefix_caching:
198 - # TODO: support sleep with prefix caching
199 - # by resetting the prefix cache state,
200 - # after https://github.com/vllm-project/vllm/pull/12284
201 - raise ValueError("Cannot sleep when prefix caching is enabled.")
197 self.collective_rpc("sleep", kwargs=dict(level=level))
198
199 def wake_up(self):
vllm/v1/core/kv_cache_manager.py
+27
@@ -285,6 +285,33 @@ class KVCacheManager:
285 if block.ref_cnt == 0:
286 self.free_block_queue.append(block)
287
288 + def reset_prefix_cache(self) -> bool:
289 + """Reset prefix cache. This function may be used in RLHF
290 + flows to invalid prefix caching after the weights are updated,
291 + or used for resetting prefix caching status for benchmarking.
292 +
293 + Returns:
294 + bool: True if the prefix cache is successfully reset,
295 + False otherwise.
296 + """
297 + num_used_blocks = (self.num_gpu_blocks -
298 + self.free_block_queue.num_free_blocks)
299 + if num_used_blocks > 0:
300 + logger.warning(
301 + "Failed to reset prefix cache because some "
302 + "blocks (%d) are not freed yet", num_used_blocks)
303 + return False
304 +
305 + # Remove all hashes so that no new blocks will hit.
306 + self.cached_block_hash_to_block = defaultdict(dict)
307 +
308 + # Remove all hashes from all blocks.
309 + for block in self.block_pool:
310 + block.reset_hash()
311 +
312 + logger.info("Successfully reset prefix cache")
313 + return True
314 +
315 def get_num_common_prefix_blocks(
316 self,
317 request: Request,
vllm/v1/core/scheduler.py
+3
@@ -529,6 +529,9 @@ class Scheduler:
529 def has_unfinished_requests(self) -> bool:
530 return self.get_num_unfinished_requests() > 0
531
532 + def reset_prefix_cache(self) -> bool:
533 + return self.kv_cache_manager.reset_prefix_cache()
534 +
535 def make_stats(self) -> SchedulerStats:
536 return SchedulerStats(
537 num_running_reqs=len(self.running),
vllm/v1/engine/__init__.py
+8 -1
@@ -66,6 +66,11 @@ class EngineCoreProfile:
66 is_start: bool
67
68
69 +@dataclass
70 +class EngineCoreResetPrefixCache:
71 + pass
72 +
73 +
74 class EngineCoreRequestType(enum.Enum):
75 """
76 Request types defined as hex byte strings, so it can be sent over sockets
@@ -74,6 +79,8 @@ class EngineCoreRequestType(enum.Enum):
79 ADD = b'\x00'
80 ABORT = b'\x01'
81 PROFILE = b'\x02'
82 + RESET_PREFIX_CACHE = b'\x03'
83
84
79 -EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile, List[str]]
85 +EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile,
86 + EngineCoreResetPrefixCache, List[str]]
vllm/v1/engine/async_llm.py
+3
@@ -321,6 +321,9 @@ class AsyncLLM(EngineClient):
321 async def stop_profile(self) -> None:
322 await self.engine_core.profile_async(False)
323
324 + async def reset_prefix_cache(self) -> None:
325 + await self.engine_core.reset_prefix_cache_async()
326 +
327 @property
328 def is_running(self) -> bool:
329 return True
vllm/v1/engine/core.py
+9 -2
@@ -20,7 +20,7 @@ from vllm.v1.core.kv_cache_utils import get_kv_cache_config
20 from vllm.v1.core.scheduler import Scheduler
21 from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
22 EngineCoreRequest, EngineCoreRequestType,
23 - EngineCoreRequestUnion)
23 + EngineCoreRequestUnion, EngineCoreResetPrefixCache)
24 from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
25 from vllm.v1.executor.abstract import Executor
26 from vllm.v1.request import Request, RequestStatus
@@ -135,6 +135,9 @@ class EngineCore:
135 def profile(self, is_start: bool = True):
136 self.model_executor.profile(is_start)
137
138 + def reset_prefix_cache(self):
139 + self.scheduler.reset_prefix_cache()
140 +
141
142 class EngineCoreProc(EngineCore):
143 """ZMQ-wrapper for running EngineCore in background process."""
@@ -247,6 +250,8 @@ class EngineCoreProc(EngineCore):
250 self.add_request(request)
251 elif isinstance(request, EngineCoreProfile):
252 self.model_executor.profile(request.is_start)
253 + elif isinstance(request, EngineCoreResetPrefixCache):
254 + self.reset_prefix_cache()
255 else:
256 # TODO: make an EngineCoreAbort wrapper
257 assert isinstance(request, list)
@@ -271,7 +276,9 @@ class EngineCoreProc(EngineCore):
276 request = decoder_add_req.decode(request_data)
277 elif request_type == EngineCoreRequestType.ABORT.value:
278 request = decoder_abort_req.decode(request_data)
274 - elif request_type == EngineCoreRequestType.PROFILE.value:
279 + elif request_type in (
280 + EngineCoreRequestType.PROFILE.value,
281 + EngineCoreRequestType.RESET_PREFIX_CACHE.value):
282 request = pickle.loads(request_data)
283 else:
284 raise ValueError(f"Unknown RequestType: {request_type}")
vllm/v1/engine/core_client.py
+19 -2
@@ -14,7 +14,7 @@ from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
14 make_zmq_socket)
15 from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
16 EngineCoreRequest, EngineCoreRequestType,
17 - EngineCoreRequestUnion)
17 + EngineCoreRequestUnion, EngineCoreResetPrefixCache)
18 from vllm.v1.engine.core import EngineCore, EngineCoreProc
19 from vllm.v1.executor.abstract import Executor
20 from vllm.v1.serial_utils import PickleEncoder
@@ -69,6 +69,9 @@ class EngineCoreClient(ABC):
69 def profile(self, is_start: bool = True) -> None:
70 raise NotImplementedError
71
72 + def reset_prefix_cache(self) -> None:
73 + raise NotImplementedError
74 +
75 def abort_requests(self, request_ids: List[str]) -> None:
76 raise NotImplementedError
77
@@ -81,6 +84,9 @@ class EngineCoreClient(ABC):
84 async def profile_async(self, is_start: bool = True) -> None:
85 raise NotImplementedError
86
87 + async def reset_prefix_cache_async(self) -> None:
88 + raise NotImplementedError
89 +
90 async def abort_requests_async(self, request_ids: List[str]) -> None:
91 raise NotImplementedError
92
@@ -108,12 +114,15 @@ class InprocClient(EngineCoreClient):
114 if len(request_ids) > 0:
115 self.engine_core.abort_requests(request_ids)
116
111 - def shutdown(self):
117 + def shutdown(self) -> None:
118 self.engine_core.shutdown()
119
120 def profile(self, is_start: bool = True) -> None:
121 self.engine_core.profile(is_start)
122
123 + def reset_prefix_cache(self) -> None:
124 + self.engine_core.reset_prefix_cache()
125 +
126
127 class MPClient(EngineCoreClient):
128 """
@@ -229,6 +238,10 @@ class SyncMPClient(MPClient):
238 self._send_input(EngineCoreRequestType.PROFILE,
239 EngineCoreProfile(is_start))
240
241 + def reset_prefix_cache(self) -> None:
242 + self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
243 + EngineCoreResetPrefixCache())
244 +
245
246 class AsyncMPClient(MPClient):
247 """Asyncio-compatible client for multi-proc EngineCore."""
@@ -266,3 +279,7 @@ class AsyncMPClient(MPClient):
279 async def profile_async(self, is_start: bool = True) -> None:
280 await self._send_input(EngineCoreRequestType.PROFILE,
281 EngineCoreProfile(is_start))
282 +
283 + async def reset_prefix_cache_async(self) -> None:
284 + await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
285 + EngineCoreResetPrefixCache())
vllm/v1/engine/llm_engine.py
+3
@@ -162,6 +162,9 @@ class LLMEngine:
162 def stop_profile(self):
163 self.engine_core.profile(False)
164
165 + def reset_prefix_cache(self):
166 + self.engine_core.reset_prefix_cache()
167 +
168 def get_tokenizer_group(
169 self,
170 group_type: Type[_G] = BaseTokenizerGroup,