A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix][Hardware][CPU] Fix CPU model input for decode (#9044)
Isotr0py committed
Oct 7, 2024 at 10:14 UTC
487678d046fe56560ff5dc6c91c3f3c31af7de6f
1 file changed
+1
-1
vllm/worker/cpu_model_runner.py
+1
-1
@@ -133,7 +133,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
133
(input_tokens, input_positions,
134
attn_metadata) = self._prepare_decode(
135
self.seq_group_metadata_list)
136
- seq_lens = []
136
+ seq_lens = None
137
138
return self.model_input_cls(
139
input_tokens=input_tokens,