@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 487678d046
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix][Hardware][CPU] Fix CPU model input for decode (#9044)

Isotr0py committed Oct 7, 2024 at 10:14 UTC 487678d046fe56560ff5dc6c91c3f3c31af7de6f
1 file changed +1 -1
vllm/worker/cpu_model_runner.py
+1 -1
@@ -133,7 +133,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
133 (input_tokens, input_positions,
134 attn_metadata) = self._prepare_decode(
135 self.seq_group_metadata_list)
136 - seq_lens = []
136 + seq_lens = None
137
138 return self.model_input_cls(
139 input_tokens=input_tokens,