A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[misc] fix custom allreduce p2p cache file generation (#7853)
youkaichao committed
Aug 26, 2024 at 15:02 UTC
05826c887b47dce9ca72f6186dcfda394a2e0766
1 file changed
+25
-16
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+25
-16
@@ -4,6 +4,7 @@ import os
4
import pickle
5
import subprocess
6
import sys
7
+import tempfile
8
from itertools import product
9
from typing import Dict, List, Optional, Sequence
10
@@ -211,20 +212,27 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
212
# However, `can_actually_p2p` requires spawn method.
213
# The fix is, we use `subprocess` to call the function,
214
# where we have `if __name__ == "__main__":` in this file.
214
- input_bytes = pickle.dumps((batch_src, batch_tgt))
215
- returned = subprocess.run([sys.executable, __file__],
216
- input=input_bytes,
217
- capture_output=True)
218
- # check if the subprocess is successful
219
- try:
220
- returned.check_returncode()
221
- except Exception as e:
222
- # wrap raised exception to provide more information
223
- raise RuntimeError(
224
- f"Error happened when batch testing "
225
- f"peer-to-peer access from {batch_src} to {batch_tgt}:\n"
226
- f"{returned.stderr.decode()}") from e
227
- result = pickle.loads(returned.stdout)
215
+
216
+ # use a temporary file to store the result
217
+ # we don't use the output of the subprocess directly,
218
+ # because the subprocess might produce logging output
219
+ with tempfile.NamedTemporaryFile() as output_file:
220
+ input_bytes = pickle.dumps(
221
+ (batch_src, batch_tgt, output_file.name))
222
+ returned = subprocess.run([sys.executable, __file__],
223
+ input=input_bytes,
224
+ capture_output=True)
225
+ # check if the subprocess is successful
226
+ try:
227
+ returned.check_returncode()
228
+ except Exception as e:
229
+ # wrap raised exception to provide more information
230
+ raise RuntimeError(
231
+ f"Error happened when batch testing "
232
+ f"peer-to-peer access from {batch_src} to {batch_tgt}:\n"
233
+ f"{returned.stderr.decode()}") from e
234
+ with open(output_file.name, "rb") as f:
235
+ result = pickle.load(f)
236
for _i, _j, r in zip(batch_src, batch_tgt, result):
237
cache[f"{_i}->{_j}"] = r
238
with open(path, "w") as f:
@@ -241,6 +249,7 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
249
__all__ = ["gpu_p2p_access_check"]
250
251
if __name__ == "__main__":
244
- batch_src, batch_tgt = pickle.loads(sys.stdin.buffer.read())
252
+ batch_src, batch_tgt, output_file = pickle.loads(sys.stdin.buffer.read())
253
result = can_actually_p2p(batch_src, batch_tgt)
246
- sys.stdout.buffer.write(pickle.dumps(result))
254
+ with open(output_file, "wb") as f:
255
+ f.write(pickle.dumps(result))