A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
fix neuron performance issue (#13589)
ajayvohra2005 committed
Feb 20, 2025 at 13:59 UTC
6a417b8600d4d1e57698a91b71a38446e8fc5c45
1 file changed
+2
-2
vllm/worker/neuron_worker.py
+2
-2
@@ -76,7 +76,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
76
# Set the number of GPU blocks to be the same as the maximum number of
77
# sequences that can be processed in a single batch. This is equivalent
78
# to schedule without PagedAttention.
79
- num_gpu_blocks = self.scheduler_config.max_num_seqs
79
+ num_gpu_blocks = self.scheduler_config.max_num_seqs + 1
80
81
# Swap not yet supported with Neuron backend.
82
num_cpu_blocks = 0
@@ -90,7 +90,7 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
90
91
# Different values are not tested.
92
assert num_cpu_blocks == 0
93
- assert num_gpu_blocks == self.scheduler_config.max_num_seqs
93
+ assert num_gpu_blocks == self.scheduler_config.max_num_seqs + 1
94
95
self.cache_config.num_gpu_blocks = num_gpu_blocks
96
self.cache_config.num_cpu_blocks = num_cpu_blocks