A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Consider pempty tensor
Woosuk Kwon committed
Feb 23, 2023 at 20:20 UTC
7f985166f727159424ae8ede07d8030c3fd36413
1 file changed
+4
-1
cacheflow/models/input_metadata.py
+4
-1
@@ -23,6 +23,9 @@ class InputMetadata:
23
24
self.num_prompts = len(prompt_lens)
25
self.num_generation_tokens = context_lens.shape[0]
26
- self.max_num_blocks_per_seq = block_tables.shape[1]
26
+ if block_tables.numel() > 0:
27
+ self.max_num_blocks_per_seq = block_tables.shape[1]
28
+ else:
29
+ self.max_num_blocks_per_seq = 0
30
assert self.num_generation_tokens == block_tables.shape[0]
31
assert self.num_prompts + self.num_generation_tokens == len(seq_ids)