@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c2134beda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[torch.compile] force inductor threads (#10620)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 25, 2024 at 15:04 UTC 7c2134beda9a4f72c71c4faffcca22cebd4e1c3c
1 file changed +4 -1
vllm/plugins/__init__.py
+4 -1
@@ -3,6 +3,8 @@ import os
3 from contextlib import contextmanager
4 from typing import TYPE_CHECKING, Optional
5
6 +import torch
7 +
8 import vllm.envs as envs
9
10 if TYPE_CHECKING:
@@ -26,7 +28,8 @@ def load_general_plugins():
28
29 # see https://github.com/vllm-project/vllm/issues/10480
30 os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
29 -
31 + # see https://github.com/vllm-project/vllm/issues/10619
32 + torch._inductor.config.compile_threads = 1
33 global plugins_loaded
34 if plugins_loaded:
35 return