A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix https://github.com/vllm-project/vllm/issues/2540 (#2545)
Philipp Moritz committed
Jan 22, 2024 at 10:02 UTC
ab7e6006d62d77dca72b29721b7b346eeb6563d4
1 file changed
+1
-1
vllm/engine/llm_engine.py
+1
-1
@@ -209,7 +209,7 @@ class LLMEngine:
209
worker.set_cuda_visible_devices.remote(node_gpus[node_id])
210
211
distributed_init_method = get_distributed_init_method(
212
- driver_ip, get_open_port)
212
+ driver_ip, get_open_port())
213
214
# Lazy import the Worker to avoid importing torch.cuda/xformers
215
# before CUDA_VISIBLE_DEVICES is set in the Worker