A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix eager mode performance (#2377)
Woosuk Kwon committed
Jan 8, 2024 at 10:11 UTC
c88481913599daff6ca293bf778a620213e467c9
1 file changed
+3
-1
vllm/worker/model_runner.py
+3
-1
@@ -235,9 +235,11 @@ class ModelRunner:
235
input_block_tables[i, :len(block_table)] = block_table
236
block_tables = torch.tensor(input_block_tables, device="cuda")
237
else:
238
+ max_block_table_len = (max_context_len + self.block_size -
239
+ 1) // self.block_size
240
block_tables = _make_tensor_with_pad(
241
block_tables,
240
- max_len=max_context_len,
242
+ max_len=max_block_table_len,
243
pad=0,
244
dtype=torch.int,
245
device="cuda",