@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c884819135
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix eager mode performance (#2377)

Woosuk Kwon committed Jan 8, 2024 at 10:11 UTC c88481913599daff6ca293bf778a620213e467c9
1 file changed +3 -1
vllm/worker/model_runner.py
+3 -1
@@ -235,9 +235,11 @@ class ModelRunner:
235 input_block_tables[i, :len(block_table)] = block_table
236 block_tables = torch.tensor(input_block_tables, device="cuda")
237 else:
238 + max_block_table_len = (max_context_len + self.block_size -
239 + 1) // self.block_size
240 block_tables = _make_tensor_with_pad(
241 block_tables,
240 - max_len=max_context_len,
242 + max_len=max_block_table_len,
243 pad=0,
244 dtype=torch.int,
245 device="cuda",