@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7f985166f7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Consider pempty tensor

Woosuk Kwon committed Feb 23, 2023 at 20:20 UTC 7f985166f727159424ae8ede07d8030c3fd36413
1 file changed +4 -1
cacheflow/models/input_metadata.py
+4 -1
@@ -23,6 +23,9 @@ class InputMetadata:
23
24 self.num_prompts = len(prompt_lens)
25 self.num_generation_tokens = context_lens.shape[0]
26 - self.max_num_blocks_per_seq = block_tables.shape[1]
26 + if block_tables.numel() > 0:
27 + self.max_num_blocks_per_seq = block_tables.shape[1]
28 + else:
29 + self.max_num_blocks_per_seq = 0
30 assert self.num_generation_tokens == block_tables.shape[0]
31 assert self.num_prompts + self.num_generation_tokens == len(seq_ids)