A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] Fix torch.compile() error when using MultiprocessingGPUExecutor (#5229)
zifeitong committed
Jun 3, 2024 at 20:55 UTC
a58f24e590c1c4be2d1398f62f119a795e79d833
1 file changed
+3
vllm/executor/multiproc_gpu_executor.py
+3
@@ -34,6 +34,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
34
# Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
35
os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
36
37
+ # Disable torch async compiling which won't work with daemonic processes
38
+ os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
39
+
40
from torch.cuda import device_count
41
assert world_size <= device_count(), (
42
"please set tensor_parallel_size to less than max local gpu count")