A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[BugFix] Pop instead of del CUDA_VISIBLE_DEVICES (#12962)
Signed-off-by: Hollow Man <hollowman@opensuse.org>
ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟 committed
Feb 11, 2025 at 18:21 UTC
6c4dbe23eb85e5d1da00ccaf4923a275d8769a7f
4 files changed
+9
-9
examples/offline_inference/rlhf.py
+1
-1
@@ -92,7 +92,7 @@ class MyLLM(LLM):
92
# a hack to make the script work.
93
# stop ray from manipulating CUDA_VISIBLE_DEVICES
94
# at the top-level
95
- del os.environ["CUDA_VISIBLE_DEVICES"]
95
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
96
super().__init__(*args, **kwargs)
97
98
examples/offline_inference/rlhf_colocate.py
+1
-1
@@ -59,7 +59,7 @@ class MyLLM(LLM):
59
# a hack to make the script work.
60
# stop ray from manipulating CUDA_VISIBLE_DEVICES
61
# at the top-level
62
- del os.environ["CUDA_VISIBLE_DEVICES"]
62
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
63
# every worker will use 0.4 GPU, so that we can schedule
64
# 2 instances on the same GPUs.
65
os.environ["VLLM_RAY_PER_WORKER_GPUS"] = "0.4"
tests/distributed/test_comm_ops.py
+5
-5
@@ -22,7 +22,7 @@ def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
22
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
23
# so that each worker can see all the GPUs
24
# they will be able to set the device to the correct GPU
25
- del os.environ["CUDA_VISIBLE_DEVICES"]
25
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
26
device = torch.device(f"cuda:{rank}")
27
torch.cuda.set_device(device)
28
init_test_distributed_environment(tp_size, pp_size, rank,
@@ -44,7 +44,7 @@ def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
44
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
45
# so that each worker can see all the GPUs
46
# they will be able to set the device to the correct GPU
47
- del os.environ["CUDA_VISIBLE_DEVICES"]
47
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
48
device = torch.device(f"cuda:{rank}")
49
torch.cuda.set_device(device)
50
init_test_distributed_environment(tp_size, pp_size, rank,
@@ -72,7 +72,7 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
72
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
73
# so that each worker can see all the GPUs
74
# they will be able to set the device to the correct GPU
75
- del os.environ["CUDA_VISIBLE_DEVICES"]
75
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
76
device = torch.device(f"cuda:{rank}")
77
torch.cuda.set_device(device)
78
init_test_distributed_environment(tp_size, pp_size, rank,
@@ -108,7 +108,7 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
108
@ray.remote(num_gpus=1, max_calls=1)
109
def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
110
distributed_init_port: str):
111
- del os.environ["CUDA_VISIBLE_DEVICES"]
111
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
112
device = torch.device(f"cuda:{rank}")
113
torch.cuda.set_device(device)
114
init_test_distributed_environment(tp_size, pp_size, rank,
@@ -148,7 +148,7 @@ def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
148
@ray.remote(num_gpus=1, max_calls=1)
149
def send_recv_test_worker(tp_size: int, pp_size: int, rank: int,
150
distributed_init_port: str):
151
- del os.environ["CUDA_VISIBLE_DEVICES"]
151
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
152
device = torch.device(f"cuda:{rank}")
153
torch.cuda.set_device(device)
154
init_test_distributed_environment(tp_size, pp_size, rank,
tests/distributed/test_custom_all_reduce.py
+2
-2
@@ -24,7 +24,7 @@ for i, v in enumerate(test_sizes):
24
25
@ray.remote(num_gpus=1, max_calls=1)
26
def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
27
- del os.environ["CUDA_VISIBLE_DEVICES"]
27
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
28
device = torch.device(f"cuda:{rank}")
29
torch.cuda.set_device(device)
30
init_test_distributed_environment(tp_size, pp_size, rank,
@@ -80,7 +80,7 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
80
81
@ray.remote(num_gpus=1, max_calls=1)
82
def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
83
- del os.environ["CUDA_VISIBLE_DEVICES"]
83
+ os.environ.pop("CUDA_VISIBLE_DEVICES", None)
84
device = torch.device(f"cuda:{rank}")
85
torch.cuda.set_device(device)
86
init_test_distributed_environment(tp_size, pp_size, rank,