A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix assertion in NeuronExecutor (#5841)
aws-patlange committed
Jun 25, 2024 at 19:52 UTC
82079729ccd0830ce77fcc5fd7ea2be3bf81ccaf
1 file changed
+3
-3
vllm/executor/neuron_executor.py
+3
-3
@@ -48,9 +48,9 @@ class NeuronExecutor(ExecutorBase):
48
def execute_model(
49
self,
50
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
51
- assert (execute_model_req.blocks_to_swap_in == {}
52
- and execute_model_req.blocks_to_swap_out == {}
53
- and execute_model_req.blocks_to_copy == {}), (
51
+ assert (not execute_model_req.blocks_to_swap_in
52
+ and not execute_model_req.blocks_to_swap_out
53
+ and not execute_model_req.blocks_to_copy), (
54
"Cache operations are not supported for Neuron backend.")
55
assert execute_model_req.num_lookahead_slots == 0, (
56
"lookahead not supported for Neuron backend.")