A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix][V1] Fix 'NoneType' object has no attribute 'hash_value' (#11157)
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
Cody Yu committed
Dec 12, 2024 at 22:30 UTC
34f1a806d5771c4ee81fdaf4feb7f9fd4071d779
1 file changed
+15
-9
vllm/v1/core/kv_cache_manager.py
+15
-9
@@ -164,13 +164,14 @@ class KVCacheManager:
164
165
new_full_blocks = req_blocks[
166
num_computed_full_blocks:num_full_blocks_after_append]
167
- self._cache_full_blocks(
168
- request=request,
169
- blk_start_idx=num_computed_full_blocks,
170
- full_blocks=new_full_blocks,
171
- prev_block=req_blocks[num_computed_full_blocks - 1]
172
- if num_computed_full_blocks >= 1 else None,
173
- )
167
+ if new_full_blocks:
168
+ self._cache_full_blocks(
169
+ request=request,
170
+ blk_start_idx=num_computed_full_blocks,
171
+ full_blocks=new_full_blocks,
172
+ prev_block=req_blocks[num_computed_full_blocks - 1]
173
+ if num_computed_full_blocks >= 1 else None,
174
+ )
175
176
return new_blocks
177
@@ -375,8 +376,13 @@ class KVCacheManager:
376
prev_block: The previous block in the chain.
377
"""
378
# Update the new blocks with the block hashes through the chain.
378
- prev_block_hash_value = (prev_block.block_hash.hash_value
379
- if prev_block is not None else None)
379
+ prev_block_hash_value = None
380
+ if prev_block is not None:
381
+ # Previous block must have a block hash because it must be
382
+ # a full, cached block.
383
+ assert prev_block.block_hash is not None
384
+ prev_block_hash_value = prev_block.block_hash.hash_value
385
+
386
for i, blk in enumerate(full_blocks):
387
blk_idx = blk_start_idx + i
388