@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c4dbe23eb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[BugFix] Pop instead of del CUDA_VISIBLE_DEVICES (#12962)

Signed-off-by: Hollow Man <hollowman@opensuse.org>

ℍ𝕠𝕝𝕝𝕠𝕨 𝕄𝕒𝕟 committed Feb 11, 2025 at 18:21 UTC 6c4dbe23eb85e5d1da00ccaf4923a275d8769a7f
4 files changed +9 -9
examples/offline_inference/rlhf.py
+1 -1
@@ -92,7 +92,7 @@ class MyLLM(LLM):
92 # a hack to make the script work.
93 # stop ray from manipulating CUDA_VISIBLE_DEVICES
94 # at the top-level
95 - del os.environ["CUDA_VISIBLE_DEVICES"]
95 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
96 super().__init__(*args, **kwargs)
97
98
examples/offline_inference/rlhf_colocate.py
+1 -1
@@ -59,7 +59,7 @@ class MyLLM(LLM):
59 # a hack to make the script work.
60 # stop ray from manipulating CUDA_VISIBLE_DEVICES
61 # at the top-level
62 - del os.environ["CUDA_VISIBLE_DEVICES"]
62 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
63 # every worker will use 0.4 GPU, so that we can schedule
64 # 2 instances on the same GPUs.
65 os.environ["VLLM_RAY_PER_WORKER_GPUS"] = "0.4"
tests/distributed/test_comm_ops.py
+5 -5
@@ -22,7 +22,7 @@ def all_reduce_test_worker(tp_size: int, pp_size: int, rank: int,
22 # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
23 # so that each worker can see all the GPUs
24 # they will be able to set the device to the correct GPU
25 - del os.environ["CUDA_VISIBLE_DEVICES"]
25 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
26 device = torch.device(f"cuda:{rank}")
27 torch.cuda.set_device(device)
28 init_test_distributed_environment(tp_size, pp_size, rank,
@@ -44,7 +44,7 @@ def all_gather_test_worker(tp_size: int, pp_size: int, rank: int,
44 # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
45 # so that each worker can see all the GPUs
46 # they will be able to set the device to the correct GPU
47 - del os.environ["CUDA_VISIBLE_DEVICES"]
47 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
48 device = torch.device(f"cuda:{rank}")
49 torch.cuda.set_device(device)
50 init_test_distributed_environment(tp_size, pp_size, rank,
@@ -72,7 +72,7 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
72 # it is important to delete the CUDA_VISIBLE_DEVICES environment variable
73 # so that each worker can see all the GPUs
74 # they will be able to set the device to the correct GPU
75 - del os.environ["CUDA_VISIBLE_DEVICES"]
75 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
76 device = torch.device(f"cuda:{rank}")
77 torch.cuda.set_device(device)
78 init_test_distributed_environment(tp_size, pp_size, rank,
@@ -108,7 +108,7 @@ def broadcast_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
108 @ray.remote(num_gpus=1, max_calls=1)
109 def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
110 distributed_init_port: str):
111 - del os.environ["CUDA_VISIBLE_DEVICES"]
111 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
112 device = torch.device(f"cuda:{rank}")
113 torch.cuda.set_device(device)
114 init_test_distributed_environment(tp_size, pp_size, rank,
@@ -148,7 +148,7 @@ def send_recv_tensor_dict_test_worker(tp_size: int, pp_size: int, rank: int,
148 @ray.remote(num_gpus=1, max_calls=1)
149 def send_recv_test_worker(tp_size: int, pp_size: int, rank: int,
150 distributed_init_port: str):
151 - del os.environ["CUDA_VISIBLE_DEVICES"]
151 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
152 device = torch.device(f"cuda:{rank}")
153 torch.cuda.set_device(device)
154 init_test_distributed_environment(tp_size, pp_size, rank,
tests/distributed/test_custom_all_reduce.py
+2 -2
@@ -24,7 +24,7 @@ for i, v in enumerate(test_sizes):
24
25 @ray.remote(num_gpus=1, max_calls=1)
26 def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
27 - del os.environ["CUDA_VISIBLE_DEVICES"]
27 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
28 device = torch.device(f"cuda:{rank}")
29 torch.cuda.set_device(device)
30 init_test_distributed_environment(tp_size, pp_size, rank,
@@ -80,7 +80,7 @@ def graph_allreduce(tp_size, pp_size, rank, distributed_init_port):
80
81 @ray.remote(num_gpus=1, max_calls=1)
82 def eager_allreduce(tp_size, pp_size, rank, distributed_init_port):
83 - del os.environ["CUDA_VISIBLE_DEVICES"]
83 + os.environ.pop("CUDA_VISIBLE_DEVICES", None)
84 device = torch.device(f"cuda:{rank}")
85 torch.cuda.set_device(device)
86 init_test_distributed_environment(tp_size, pp_size, rank,