A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] force inductor threads (#10620)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 25, 2024 at 15:04 UTC
7c2134beda9a4f72c71c4faffcca22cebd4e1c3c
1 file changed
+4
-1
vllm/plugins/__init__.py
+4
-1
@@ -3,6 +3,8 @@ import os
3
from contextlib import contextmanager
4
from typing import TYPE_CHECKING, Optional
5
6
+import torch
7
+
8
import vllm.envs as envs
9
10
if TYPE_CHECKING:
@@ -26,7 +28,8 @@ def load_general_plugins():
28
29
# see https://github.com/vllm-project/vllm/issues/10480
30
os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
29
-
31
+ # see https://github.com/vllm-project/vllm/issues/10619
32
+ torch._inductor.config.compile_threads = 1
33
global plugins_loaded
34
if plugins_loaded:
35
return