A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[BUGFIX]: return fast when request requires prompt logprobs (#17251)
Ning Xie committed
May 9, 2025 at 12:25 UTC
d310e6de989384222c516ce83ccc3099e9a8d65e
2 files changed
+7
-7
tests/v1/core/test_prefix_caching.py
+2
-2
@@ -194,7 +194,7 @@ def test_prefill_plp():
194
all_token_ids = common_token_ids + unique_token_ids
195
req0 = make_request("0", all_token_ids, prompt_logprobs=5)
196
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
197
- assert len(manager.req_to_block_hashes[req0.request_id]) == 3
197
+ assert len(manager.req_to_block_hashes[req0.request_id]) == 0
198
assert not computed_blocks.blocks
199
assert num_computed_tokens == 0
200
blocks = manager.allocate_slots(req0, 55, computed_blocks)
@@ -256,7 +256,7 @@ def test_prefill_plp():
256
common_token_ids + unique_token_ids,
257
prompt_logprobs=5)
258
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
259
- assert len(manager.req_to_block_hashes[req2.request_id]) == 3
259
+ assert len(manager.req_to_block_hashes[req2.request_id]) == 0
260
assert not computed_blocks.blocks
261
assert num_computed_tokens == 0
262
blocks = manager.allocate_slots(req2, 55, computed_blocks)
vllm/v1/core/kv_cache_manager.py
+5
-5
@@ -126,8 +126,11 @@ class KVCacheManager:
126
- A list of blocks that are computed for the request.
127
- The number of computed tokens.
128
"""
129
- if not self.enable_caching:
130
- # Prefix caching is disabled.
129
+
130
+ # Prefix caching is disabled or
131
+ # When the request requires prompt logprobs, we skip prefix caching.
132
+ if (not self.enable_caching
133
+ or request.sampling_params.prompt_logprobs is not None):
134
return KVCacheBlocks.create_empty(), 0
135
136
# The block hashes for the request may already be computed
@@ -141,9 +144,6 @@ class KVCacheManager:
144
if self.log_stats:
145
assert self.prefix_cache_stats is not None
146
self.prefix_cache_stats.requests += 1
144
- # When the request requires prompt logprobs, we skip prefix caching.
145
- if request.sampling_params.prompt_logprobs is not None:
146
- return KVCacheBlocks.create_empty(), 0
147
148
if len(block_hashes) * self.block_size == request.num_tokens:
149
# When prompt length is divisible by the block size and all