@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82079729cc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix assertion in NeuronExecutor (#5841)

aws-patlange committed Jun 25, 2024 at 19:52 UTC 82079729ccd0830ce77fcc5fd7ea2be3bf81ccaf
1 file changed +3 -3
vllm/executor/neuron_executor.py
+3 -3
@@ -48,9 +48,9 @@ class NeuronExecutor(ExecutorBase):
48 def execute_model(
49 self,
50 execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
51 - assert (execute_model_req.blocks_to_swap_in == {}
52 - and execute_model_req.blocks_to_swap_out == {}
53 - and execute_model_req.blocks_to_copy == {}), (
51 + assert (not execute_model_req.blocks_to_swap_in
52 + and not execute_model_req.blocks_to_swap_out
53 + and not execute_model_req.blocks_to_copy), (
54 "Cache operations are not supported for Neuron backend.")
55 assert execute_model_req.num_lookahead_slots == 0, (
56 "lookahead not supported for Neuron backend.")