A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix][CPU] fix missing input intermediate_tensors in the cpu_model_runner (#8733)
Li, Jiang committed
Sep 23, 2024 at 21:15 UTC
3e83c12b5caa466bf533b144a9ec7944a9ce9d49
1 file changed
+2
vllm/worker/cpu_model_runner.py
+2
@@ -456,6 +456,8 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
456
model_input.attn_metadata,
457
**MultiModalInputs.as_kwargs(model_input.multi_modal_kwargs or {},
458
device=self.device),
459
+ "intermediate_tensors":
460
+ intermediate_tensors,
461
}
462
463
hidden_states = model_executable(**execute_model_kwargs)