A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Core] Support `reset_prefix_cache` (#12284)
Cody Yu committed
Jan 22, 2025 at 10:52 UTC
7206ce4ce112ed117796a59045c968a6d353f691
27 files changed
+300
-21
tests/core/block/test_prefix_caching_block.py
+38
@@ -796,6 +796,44 @@ class TestPrefixCachingBlockAllocator:
796
block_hashes=block_hashes_seq1)
797
assert len(cached_blocks) == len(blocks_seq1) - num_evicted_blocks
798
799
+ # Test reset prefix cache
800
+ @staticmethod
801
+ @pytest.mark.parametrize("num_blocks", [10])
802
+ @pytest.mark.parametrize("block_size", [16])
803
+ def test_reset_prefix_cache(num_blocks: int, block_size: int):
804
+ """This test case simulates the case of resetting the prefix cache."""
805
+
806
+ allocator = PrefixCachingBlockAllocator(num_blocks=num_blocks,
807
+ block_size=block_size)
808
+ token_ids = list(range(3 * block_size))
809
+
810
+ first_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
811
+ block_size=block_size,
812
+ token_ids=token_ids,
813
+ allocator=allocator,
814
+ )
815
+ second_chain = TestPrefixCachingBlockAllocator.create_immutable_chain(
816
+ block_size=block_size,
817
+ token_ids=token_ids,
818
+ allocator=allocator,
819
+ )
820
+
821
+ # Free each block in the first chain.
822
+ for block in first_chain:
823
+ allocator.free(block)
824
+
825
+ # Failed to reset prefix cache because some blocks are not freed yet.
826
+ assert not allocator.reset_prefix_cache()
827
+ assert allocator.get_prefix_cache_hit_rate() > 0.0
828
+
829
+ # Free each block in the second chain.
830
+ for block in second_chain:
831
+ allocator.free(block)
832
+
833
+ # Reset prefix cache.
834
+ assert allocator.reset_prefix_cache()
835
+ assert allocator.get_prefix_cache_hit_rate() == 0.0
836
+
837
@staticmethod
838
def create_immutable_chain(
839
block_size: int,
tests/v1/core/test_prefix_caching.py
+39
@@ -587,3 +587,42 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
587
assert {block.ref_cnt for block in block_part1[:3]} == {1}
588
# Block 3-5 are free.
589
assert {block.ref_cnt for block in block_part1[3:]} == {0}
590
+
591
+
592
+def test_reset_prefix_cache():
593
+ manager = KVCacheManager(
594
+ block_size=16,
595
+ num_gpu_blocks=10,
596
+ max_model_len=8192,
597
+ sliding_window=None,
598
+ enable_caching=True,
599
+ num_preallocate_tokens=0,
600
+ )
601
+
602
+ full_block_token_ids = [i for i in range(3) for _ in range(16)]
603
+ unique_token_ids = [3] * 7
604
+ all_token_ids = full_block_token_ids + unique_token_ids
605
+ req0 = make_request("0", all_token_ids)
606
+ blocks = manager.allocate_slots(req0, 55, [])
607
+ assert [b.block_id for b in blocks] == [0, 1, 2, 3]
608
+
609
+ unique_token_ids = [4] * 7
610
+ all_token_ids = full_block_token_ids + unique_token_ids
611
+ req1 = make_request("1", all_token_ids)
612
+ computed_blocks, _ = manager.get_computed_blocks(req1)
613
+ assert len(req1.kv_block_hashes) == 3
614
+ assert len(computed_blocks) == 3
615
+ blocks = manager.allocate_slots(req1, 7, computed_blocks)
616
+ assert [b.block_id for b in blocks] == [4]
617
+
618
+ # Failed to reset prefix cache because some blocks are not freed yet.
619
+ assert not manager.reset_prefix_cache()
620
+ assert manager.cached_block_hash_to_block
621
+
622
+ # Free the blocks.
623
+ manager.free(req0)
624
+ manager.free(req1)
625
+
626
+ assert manager.reset_prefix_cache()
627
+ assert not manager.cached_block_hash_to_block
628
+ assert all([blk.block_hash is None for blk in manager.block_pool])
vllm/core/block/cpu_gpu_block_allocator.py
+7
@@ -339,6 +339,13 @@ class CpuGpuBlockAllocator(DeviceAwareBlockAllocator):
339
assert device in self._allocators
340
return self._allocators[device].get_prefix_cache_hit_rate()
341
342
+ def reset_prefix_cache(self) -> bool:
343
+ """Reset prefix cache for all devices."""
344
+ success = True
345
+ for allocator in self._allocators.values():
346
+ success = success and allocator.reset_prefix_cache()
347
+ return success
348
+
349
def get_and_reset_swaps(self) -> List[Tuple[int, int]]:
350
"""Returns and clears the mapping of source to destination block IDs.
351
Will be called after every swapping operations for now, and after every
vllm/core/block/interfaces.py
+10
@@ -192,6 +192,11 @@ class BlockAllocator(ABC):
192
"""Prefix cache hit rate. -1 means not supported or disabled."""
193
pass
194
195
+ @abstractmethod
196
+ def reset_prefix_cache(self) -> bool:
197
+ """Reset prefix cache."""
198
+ pass
199
+
200
class NoFreeBlocksError(ValueError):
201
pass
202
@@ -297,6 +302,11 @@ class DeviceAwareBlockAllocator(ABC):
302
"""Prefix cache hit rate. -1 means not supported or disabled."""
303
pass
304
305
+ @abstractmethod
306
+ def reset_prefix_cache(self) -> bool:
307
+ """Reset prefix cache."""
308
+ pass
309
+
310
@abstractmethod
311
def find_cached_blocks_prefix(
312
self,
vllm/core/block/naive_block.py
+14
-5
@@ -1,5 +1,5 @@
1
from collections import deque
2
-from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple
2
+from typing import Deque, FrozenSet, Iterable, List, Optional, Tuple, Union
3
4
from vllm.core.block.common import (BlockPool, CopyOnWriteTracker, RefCounter,
5
get_all_blocks_recursively)
@@ -136,16 +136,18 @@ class NaiveBlockAllocator(BlockAllocator):
136
self._refcounter.incr(block_id)
137
return block_id
138
139
- def _free_block_id(self, block: Block) -> None:
140
- block_id = block.block_id
139
+ def _free_block_id(self, block: Union[Block, BlockId]) -> None:
140
+ if isinstance(block, Block):
141
+ block_id = block.block_id
142
+ block.block_id = None
143
+ else:
144
+ block_id = block
145
assert block_id is not None
146
147
refcount = self._refcounter.decr(block_id)
148
if refcount == 0:
149
self._free_block_indices.appendleft(block_id)
150
147
- block.block_id = None
148
-
151
def free(self, block: Block, keep_block_object: bool = False) -> None:
152
# Release the physical block id
153
self._free_block_id(block)
@@ -154,6 +156,9 @@ class NaiveBlockAllocator(BlockAllocator):
156
if not keep_block_object:
157
self._block_pool.free_block(block)
158
159
+ def free_block_id(self, block_id: BlockId) -> None:
160
+ self._free_block_id(block_id)
161
+
162
def fork(self, last_block: Block) -> List[Block]:
163
"""Creates a new sequence of blocks that shares the same underlying
164
memory as the original sequence.
@@ -325,6 +330,10 @@ class NaiveBlockAllocator(BlockAllocator):
330
def get_prefix_cache_hit_rate(self) -> float:
331
return -1
332
333
+ def reset_prefix_cache(self) -> bool:
334
+ """No prefix cache for naive block allocator."""
335
+ return True
336
+
337
def find_cached_blocks_prefix(self, block_hashes: List[int]) -> List[int]:
338
# Not applicable for naive block allocator.
339
return []
vllm/core/block/prefix_caching_block.py
+43
-1
@@ -12,6 +12,7 @@ from vllm.core.block.interfaces import (Block, BlockAllocator, BlockId, Device,
12
from vllm.core.block.naive_block import (BlockPool, NaiveBlock,
13
NaiveBlockAllocator)
14
from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
15
+from vllm.logger import init_logger
16
from vllm.sequence import Sequence
17
18
PrefixHash = int
@@ -21,6 +22,8 @@ PrefixHash = int
22
# then we know this block hasn't been accessed yet.
23
_DEFAULT_LAST_ACCESSED_TIME = -1
24
25
+logger = init_logger(__name__)
26
+
27
28
class BlockTracker:
29
"""Used to track the status of a block inside the prefix caching allocator
@@ -105,7 +108,8 @@ class PrefixCachingBlockAllocator(BlockAllocator):
108
109
# Evitor used to maintain how we want to handle those computed blocks
110
# if we find memory pressure is high.
108
- self.evictor: Evictor = make_evictor(eviction_policy)
111
+ self.eviction_policy = eviction_policy
112
+ self.evictor: Evictor = make_evictor(self.eviction_policy)
113
114
# We share the refcounter between allocators. This allows us to promote
115
# blocks originally allocated in the hashless allocator to immutable
@@ -428,6 +432,44 @@ class PrefixCachingBlockAllocator(BlockAllocator):
432
def get_prefix_cache_hit_rate(self) -> float:
433
return self.metric_data.get_hit_rate()
434
435
+ def reset_prefix_cache(self) -> bool:
436
+ """Reset prefix cache. This function may be used in RLHF
437
+ flows to invalid prefix caching after the weights are updated,
438
+ or used for resetting prefix caching status for benchmarking.
439
+
440
+ Returns:
441
+ bool: True if the prefix cache is successfully reset,
442
+ False otherwise.
443
+ """
444
+ num_used_blocks = (self.get_num_total_blocks() -
445
+ self.get_num_free_blocks())
446
+ if num_used_blocks > 0:
447
+ logger.warning(
448
+ "Failed to reset prefix cache because some "
449
+ "blocks (%d) are not freed yet", num_used_blocks)
450
+ return False
451
+
452
+ # Free all blocks in the evictor.
453
+ while (block_id :=
454
+ self._maybe_allocate_evicted_block_id()) is not None:
455
+ self._hashless_allocator.free_block_id(block_id)
456
+
457
+ # Should not have any cached blocks because all blocks are evicted.
458
+ assert not self._cached_blocks
459
+
460
+ # Reset the evictor.
461
+ self.evictor = make_evictor(self.eviction_policy)
462
+
463
+ # Reset the block tracker.
464
+ for block_id in self._block_tracker:
465
+ self._block_tracker[block_id] = BlockTracker()
466
+
467
+ # Reset the metrics.
468
+ self.metric_data = CacheMetricData()
469
+
470
+ logger.info("Successfully reset prefix cache")
471
+ return True
472
+
473
def is_block_cached(self, block: Block) -> bool:
474
assert block.content_hash is not None
475
return block.content_hash in self._cached_blocks
vllm/core/block_manager.py
+3
@@ -455,6 +455,9 @@ class SelfAttnBlockSpaceManager(BlockSpaceManager):
455
def get_prefix_cache_hit_rate(self, device: Device) -> float:
456
return self.block_allocator.get_prefix_cache_hit_rate(device)
457
458
+ def reset_prefix_cache(self) -> bool:
459
+ return self.block_allocator.reset_prefix_cache()
460
+
461
def _can_swap(self,
462
seq_group: SequenceGroup,
463
device: Device,
vllm/core/interfaces.py
+5
@@ -122,6 +122,11 @@ class BlockSpaceManager(ABC):
122
"""Prefix cache hit rate. -1 means not supported or disabled."""
123
pass
124
125
+ @abstractmethod
126
+ def reset_prefix_cache(self) -> bool:
127
+ """Reset prefix cache for all devices."""
128
+ pass
129
+
130
@abstractmethod
131
def get_num_cached_tokens(self, seq: Sequence) -> int:
132
pass
vllm/core/placeholder_block_space_manager.py
+3
@@ -90,5 +90,8 @@ class PlaceholderBlockSpaceManager(BlockSpaceManager):
90
def get_prefix_cache_hit_rate(self, device: Device) -> float:
91
return -1
92
93
+ def reset_prefix_cache(self) -> bool:
94
+ return True
95
+
96
def get_num_cached_tokens(self, seq: Sequence) -> int:
97
return 0
vllm/core/scheduler.py
+3
@@ -504,6 +504,9 @@ class Scheduler:
504
def get_prefix_cache_hit_rate(self, device: Device) -> float:
505
return self.block_manager.get_prefix_cache_hit_rate(device)
506
507
+ def reset_prefix_cache(self) -> bool:
508
+ return self.block_manager.reset_prefix_cache()
509
+
510
def get_num_unfinished_seq_groups(self) -> int:
511
return len(self.waiting) + len(self.running) + len(self.swapped)
512
vllm/engine/async_llm_engine.py
+3
@@ -1182,6 +1182,9 @@ class AsyncLLMEngine(EngineClient):
1182
async def stop_profile(self) -> None:
1183
self.engine.stop_profile()
1184
1185
+ async def reset_prefix_cache(self) -> None:
1186
+ self.engine.reset_prefix_cache()
1187
+
1188
async def add_lora(self, lora_request: LoRARequest) -> None:
1189
self.engine.add_lora(lora_request)
1190
vllm/engine/llm_engine.py
+8
@@ -914,6 +914,14 @@ class LLMEngine:
914
"""
915
return self.scheduler[virtual_engine].has_unfinished_seqs()
916
917
+ def reset_prefix_cache(self) -> bool:
918
+ """Reset prefix cache for all devices."""
919
+
920
+ success = True
921
+ for scheduler in self.scheduler:
922
+ success = success and scheduler.reset_prefix_cache()
923
+ return success
924
+
925
@staticmethod
926
def _process_sequence_group_outputs(
927
seq_group: SequenceGroup,
vllm/engine/multiprocessing/__init__.py
+6
-1
@@ -121,6 +121,10 @@ class RPCUProfileRequest(Enum):
121
STOP_PROFILE = 2
122
123
124
+class RPCResetPrefixCacheRequest(Enum):
125
+ RESET_PREFIX_CACHE = 1
126
+
127
+
128
@dataclass
129
class RPCLoadAdapterRequest:
130
lora_request: LoRARequest
@@ -134,7 +138,8 @@ class RPCAdapterLoadedResponse:
138
139
140
RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest,
137
- RPCUProfileRequest, RPCLoadAdapterRequest]
141
+ RPCUProfileRequest, RPCLoadAdapterRequest,
142
+ RPCResetPrefixCacheRequest]
143
144
REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCAdapterLoadedResponse,
145
RPCError]
vllm/engine/multiprocessing/client.py
+10
-2
@@ -27,8 +27,9 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
27
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
28
RPCAdapterLoadedResponse, RPCError,
29
RPCLoadAdapterRequest,
30
- RPCProcessRequest, RPCStartupRequest,
31
- RPCStartupResponse,
30
+ RPCProcessRequest,
31
+ RPCResetPrefixCacheRequest,
32
+ RPCStartupRequest, RPCStartupResponse,
33
RPCUProfileRequest)
34
from vllm.engine.protocol import EngineClient
35
# yapf: enable
@@ -675,6 +676,13 @@ class MQLLMEngineClient(EngineClient):
676
await self._send_one_way_rpc_request(
677
request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
678
679
+ async def reset_prefix_cache(self) -> None:
680
+ """Reset the prefix cache"""
681
+
682
+ await self._send_one_way_rpc_request(
683
+ request=RPCResetPrefixCacheRequest.RESET_PREFIX_CACHE,
684
+ socket=self.input_socket)
685
+
686
async def add_lora(self, lora_request: LoRARequest) -> None:
687
"""Load a new LoRA adapter into the engine for future requests."""
688
# Uses the same I/O as generate requests
vllm/engine/multiprocessing/engine.py
+8
-2
@@ -16,8 +16,9 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
16
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
17
RPCAdapterLoadedResponse, RPCError,
18
RPCLoadAdapterRequest,
19
- RPCProcessRequest, RPCStartupRequest,
20
- RPCStartupResponse,
19
+ RPCProcessRequest,
20
+ RPCResetPrefixCacheRequest,
21
+ RPCStartupRequest, RPCStartupResponse,
22
RPCUProfileRequest)
23
# yapf: enable
24
from vllm.logger import init_logger
@@ -237,6 +238,8 @@ class MQLLMEngine:
238
self.stop_profile()
239
elif isinstance(request, RPCLoadAdapterRequest):
240
self._handle_load_adapter_request(request)
241
+ elif isinstance(request, RPCResetPrefixCacheRequest):
242
+ self.reset_prefix_cache()
243
else:
244
raise ValueError("Unknown RPCRequest Type: "
245
f"{type(request)}")
@@ -361,6 +364,9 @@ class MQLLMEngine:
364
def stop_profile(self) -> None:
365
self.engine.stop_profile()
366
367
+ def reset_prefix_cache(self) -> bool:
368
+ return self.engine.reset_prefix_cache()
369
+
370
371
def signal_handler(*_) -> None:
372
raise KeyboardInterrupt("MQLLMEngine terminated")
vllm/engine/protocol.py
+5
@@ -271,6 +271,11 @@ class EngineClient(ABC):
271
"""Start profiling the engine"""
272
...
273
274
+ @abstractmethod
275
+ async def reset_prefix_cache(self) -> None:
276
+ """Reset the prefix cache"""
277
+ ...
278
+
279
@abstractmethod
280
async def add_lora(self, lora_request: LoRARequest) -> None:
281
"""Load a new LoRA adapter into the engine for future requests."""
vllm/entrypoints/llm.py
+4
@@ -1132,6 +1132,9 @@ class LLM:
1132
def stop_profile(self) -> None:
1133
self.llm_engine.stop_profile()
1134
1135
+ def reset_prefix_cache(self) -> bool:
1136
+ return self.llm_engine.reset_prefix_cache()
1137
+
1138
def sleep(self, level: int = 1):
1139
"""
1140
Put the engine to sleep. The engine should not process any requests.
@@ -1150,6 +1153,7 @@ class LLM:
1153
where previous model weights are not needed. It reduces CPU memory
1154
pressure.
1155
"""
1156
+ self.reset_prefix_cache()
1157
self.llm_engine.sleep(level=level)
1158
1159
def wake_up(self):
vllm/entrypoints/openai/api_server.py
+12
@@ -518,6 +518,18 @@ TASK_HANDLERS: Dict[str, Dict[str, tuple]] = {
518
},
519
}
520
521
+if envs.VLLM_SERVER_DEV_MODE:
522
+
523
+ @router.post("/reset_prefix_cache")
524
+ async def reset_prefix_cache(raw_request: Request):
525
+ """
526
+ Reset the prefix cache. Note that we currently do not check if the
527
+ prefix cache is successfully reset in the API server.
528
+ """
529
+ logger.info("Resetting prefix cache...")
530
+ await engine_client(raw_request).reset_prefix_cache()
531
+ return Response(status_code=200)
532
+
533
534
@router.post("/invocations")
535
async def invocations(raw_request: Request):
vllm/envs.py
+7
@@ -72,6 +72,7 @@ if TYPE_CHECKING:
72
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
73
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
74
VLLM_DISABLE_COMPILE_CACHE: bool = False
75
+ VLLM_SERVER_DEV_MODE: bool = False
76
77
78
def get_default_cache_root():
@@ -467,6 +468,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
468
lambda: float(os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")),
469
"VLLM_DISABLE_COMPILE_CACHE":
470
lambda: bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0"))),
471
+
472
+ # If set, vllm will run in development mode, which will enable
473
+ # some additional endpoints for developing and debugging,
474
+ # e.g. `/reset_prefix_cache`
475
+ "VLLM_SERVER_DEV_MODE":
476
+ lambda: bool(int(os.getenv("VLLM_SERVER_DEV_MODE", "0"))),
477
}
478
479
# end-env-vars-definition
vllm/executor/executor_base.py
-5
@@ -194,11 +194,6 @@ class ExecutorBase(ABC):
194
self.collective_rpc("stop_profile")
195
196
def sleep(self, level: int = 1):
197
- if self.cache_config.enable_prefix_caching:
198
- # TODO: support sleep with prefix caching
199
- # by resetting the prefix cache state,
200
- # after https://github.com/vllm-project/vllm/pull/12284
201
- raise ValueError("Cannot sleep when prefix caching is enabled.")
197
self.collective_rpc("sleep", kwargs=dict(level=level))
198
199
def wake_up(self):
vllm/v1/core/kv_cache_manager.py
+27
@@ -285,6 +285,33 @@ class KVCacheManager:
285
if block.ref_cnt == 0:
286
self.free_block_queue.append(block)
287
288
+ def reset_prefix_cache(self) -> bool:
289
+ """Reset prefix cache. This function may be used in RLHF
290
+ flows to invalid prefix caching after the weights are updated,
291
+ or used for resetting prefix caching status for benchmarking.
292
+
293
+ Returns:
294
+ bool: True if the prefix cache is successfully reset,
295
+ False otherwise.
296
+ """
297
+ num_used_blocks = (self.num_gpu_blocks -
298
+ self.free_block_queue.num_free_blocks)
299
+ if num_used_blocks > 0:
300
+ logger.warning(
301
+ "Failed to reset prefix cache because some "
302
+ "blocks (%d) are not freed yet", num_used_blocks)
303
+ return False
304
+
305
+ # Remove all hashes so that no new blocks will hit.
306
+ self.cached_block_hash_to_block = defaultdict(dict)
307
+
308
+ # Remove all hashes from all blocks.
309
+ for block in self.block_pool:
310
+ block.reset_hash()
311
+
312
+ logger.info("Successfully reset prefix cache")
313
+ return True
314
+
315
def get_num_common_prefix_blocks(
316
self,
317
request: Request,
vllm/v1/core/scheduler.py
+3
@@ -529,6 +529,9 @@ class Scheduler:
529
def has_unfinished_requests(self) -> bool:
530
return self.get_num_unfinished_requests() > 0
531
532
+ def reset_prefix_cache(self) -> bool:
533
+ return self.kv_cache_manager.reset_prefix_cache()
534
+
535
def make_stats(self) -> SchedulerStats:
536
return SchedulerStats(
537
num_running_reqs=len(self.running),
vllm/v1/engine/__init__.py
+8
-1
@@ -66,6 +66,11 @@ class EngineCoreProfile:
66
is_start: bool
67
68
69
+@dataclass
70
+class EngineCoreResetPrefixCache:
71
+ pass
72
+
73
+
74
class EngineCoreRequestType(enum.Enum):
75
"""
76
Request types defined as hex byte strings, so it can be sent over sockets
@@ -74,6 +79,8 @@ class EngineCoreRequestType(enum.Enum):
79
ADD = b'\x00'
80
ABORT = b'\x01'
81
PROFILE = b'\x02'
82
+ RESET_PREFIX_CACHE = b'\x03'
83
84
79
-EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile, List[str]]
85
+EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile,
86
+ EngineCoreResetPrefixCache, List[str]]
vllm/v1/engine/async_llm.py
+3
@@ -321,6 +321,9 @@ class AsyncLLM(EngineClient):
321
async def stop_profile(self) -> None:
322
await self.engine_core.profile_async(False)
323
324
+ async def reset_prefix_cache(self) -> None:
325
+ await self.engine_core.reset_prefix_cache_async()
326
+
327
@property
328
def is_running(self) -> bool:
329
return True
vllm/v1/engine/core.py
+9
-2
@@ -20,7 +20,7 @@ from vllm.v1.core.kv_cache_utils import get_kv_cache_config
20
from vllm.v1.core.scheduler import Scheduler
21
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
22
EngineCoreRequest, EngineCoreRequestType,
23
- EngineCoreRequestUnion)
23
+ EngineCoreRequestUnion, EngineCoreResetPrefixCache)
24
from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
25
from vllm.v1.executor.abstract import Executor
26
from vllm.v1.request import Request, RequestStatus
@@ -135,6 +135,9 @@ class EngineCore:
135
def profile(self, is_start: bool = True):
136
self.model_executor.profile(is_start)
137
138
+ def reset_prefix_cache(self):
139
+ self.scheduler.reset_prefix_cache()
140
+
141
142
class EngineCoreProc(EngineCore):
143
"""ZMQ-wrapper for running EngineCore in background process."""
@@ -247,6 +250,8 @@ class EngineCoreProc(EngineCore):
250
self.add_request(request)
251
elif isinstance(request, EngineCoreProfile):
252
self.model_executor.profile(request.is_start)
253
+ elif isinstance(request, EngineCoreResetPrefixCache):
254
+ self.reset_prefix_cache()
255
else:
256
# TODO: make an EngineCoreAbort wrapper
257
assert isinstance(request, list)
@@ -271,7 +276,9 @@ class EngineCoreProc(EngineCore):
276
request = decoder_add_req.decode(request_data)
277
elif request_type == EngineCoreRequestType.ABORT.value:
278
request = decoder_abort_req.decode(request_data)
274
- elif request_type == EngineCoreRequestType.PROFILE.value:
279
+ elif request_type in (
280
+ EngineCoreRequestType.PROFILE.value,
281
+ EngineCoreRequestType.RESET_PREFIX_CACHE.value):
282
request = pickle.loads(request_data)
283
else:
284
raise ValueError(f"Unknown RequestType: {request_type}")
vllm/v1/engine/core_client.py
+19
-2
@@ -14,7 +14,7 @@ from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
14
make_zmq_socket)
15
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
16
EngineCoreRequest, EngineCoreRequestType,
17
- EngineCoreRequestUnion)
17
+ EngineCoreRequestUnion, EngineCoreResetPrefixCache)
18
from vllm.v1.engine.core import EngineCore, EngineCoreProc
19
from vllm.v1.executor.abstract import Executor
20
from vllm.v1.serial_utils import PickleEncoder
@@ -69,6 +69,9 @@ class EngineCoreClient(ABC):
69
def profile(self, is_start: bool = True) -> None:
70
raise NotImplementedError
71
72
+ def reset_prefix_cache(self) -> None:
73
+ raise NotImplementedError
74
+
75
def abort_requests(self, request_ids: List[str]) -> None:
76
raise NotImplementedError
77
@@ -81,6 +84,9 @@ class EngineCoreClient(ABC):
84
async def profile_async(self, is_start: bool = True) -> None:
85
raise NotImplementedError
86
87
+ async def reset_prefix_cache_async(self) -> None:
88
+ raise NotImplementedError
89
+
90
async def abort_requests_async(self, request_ids: List[str]) -> None:
91
raise NotImplementedError
92
@@ -108,12 +114,15 @@ class InprocClient(EngineCoreClient):
114
if len(request_ids) > 0:
115
self.engine_core.abort_requests(request_ids)
116
111
- def shutdown(self):
117
+ def shutdown(self) -> None:
118
self.engine_core.shutdown()
119
120
def profile(self, is_start: bool = True) -> None:
121
self.engine_core.profile(is_start)
122
123
+ def reset_prefix_cache(self) -> None:
124
+ self.engine_core.reset_prefix_cache()
125
+
126
127
class MPClient(EngineCoreClient):
128
"""
@@ -229,6 +238,10 @@ class SyncMPClient(MPClient):
238
self._send_input(EngineCoreRequestType.PROFILE,
239
EngineCoreProfile(is_start))
240
241
+ def reset_prefix_cache(self) -> None:
242
+ self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
243
+ EngineCoreResetPrefixCache())
244
+
245
246
class AsyncMPClient(MPClient):
247
"""Asyncio-compatible client for multi-proc EngineCore."""
@@ -266,3 +279,7 @@ class AsyncMPClient(MPClient):
279
async def profile_async(self, is_start: bool = True) -> None:
280
await self._send_input(EngineCoreRequestType.PROFILE,
281
EngineCoreProfile(is_start))
282
+
283
+ async def reset_prefix_cache_async(self) -> None:
284
+ await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
285
+ EngineCoreResetPrefixCache())
vllm/v1/engine/llm_engine.py
+3
@@ -162,6 +162,9 @@ class LLMEngine:
162
def stop_profile(self):
163
self.engine_core.profile(False)
164
165
+ def reset_prefix_cache(self):
166
+ self.engine_core.reset_prefix_cache()
167
+
168
def get_tokenizer_group(
169
self,
170
group_type: Type[_G] = BaseTokenizerGroup,