A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 13:45 UTC
[BugFix][CPU] fix error on CPU runner shutdown (#41034)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
Fadi Arafeh committed
Apr 29, 2026 at 05:08 UTC
d95d03c719cd69e634e567d6cad3228557151393
1 file changed
+11
vllm/v1/worker/cpu_model_runner.py
+11
@@ -21,6 +21,9 @@ logger = init_logger(__name__)
21
22
class CPUModelRunner(GPUModelRunner):
23
def __init__(self, vllm_config: VllmConfig, device: torch.device):
24
+ # avoid calling accelerator APIs for methods inherited from super class
25
+ _set_torch_accelerator_to_noop()
26
+
27
with _torch_cuda_wrapper():
28
super().__init__(vllm_config, device)
29
@@ -244,3 +247,11 @@ def _set_global_compilation_settings(config: VllmConfig):
247
yield
248
finally:
249
torch_inductor_config.freezing = freezing_value
250
+
251
+
252
+def _set_torch_accelerator_to_noop() -> None:
253
+ def noop(*args: Any, **kwargs: Any) -> None:
254
+ pass
255
+
256
+ torch.accelerator.synchronize = noop
257
+ torch.accelerator.empty_cache = noop