@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a58f24e590
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] Fix torch.compile() error when using MultiprocessingGPUExecutor (#5229)

zifeitong committed Jun 3, 2024 at 20:55 UTC a58f24e590c1c4be2d1398f62f119a795e79d833
1 file changed +3
vllm/executor/multiproc_gpu_executor.py
+3
@@ -34,6 +34,9 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
34 # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
35 os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
36
37 + # Disable torch async compiling which won't work with daemonic processes
38 + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
39 +
40 from torch.cuda import device_count
41 assert world_size <= device_count(), (
42 "please set tensor_parallel_size to less than max local gpu count")