@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05826c887b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[misc] fix custom allreduce p2p cache file generation (#7853)

youkaichao committed Aug 26, 2024 at 15:02 UTC 05826c887b47dce9ca72f6186dcfda394a2e0766
1 file changed +25 -16
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+25 -16
@@ -4,6 +4,7 @@ import os
4 import pickle
5 import subprocess
6 import sys
7 +import tempfile
8 from itertools import product
9 from typing import Dict, List, Optional, Sequence
10
@@ -211,20 +212,27 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
212 # However, `can_actually_p2p` requires spawn method.
213 # The fix is, we use `subprocess` to call the function,
214 # where we have `if __name__ == "__main__":` in this file.
214 - input_bytes = pickle.dumps((batch_src, batch_tgt))
215 - returned = subprocess.run([sys.executable, __file__],
216 - input=input_bytes,
217 - capture_output=True)
218 - # check if the subprocess is successful
219 - try:
220 - returned.check_returncode()
221 - except Exception as e:
222 - # wrap raised exception to provide more information
223 - raise RuntimeError(
224 - f"Error happened when batch testing "
225 - f"peer-to-peer access from {batch_src} to {batch_tgt}:\n"
226 - f"{returned.stderr.decode()}") from e
227 - result = pickle.loads(returned.stdout)
215 +
216 + # use a temporary file to store the result
217 + # we don't use the output of the subprocess directly,
218 + # because the subprocess might produce logging output
219 + with tempfile.NamedTemporaryFile() as output_file:
220 + input_bytes = pickle.dumps(
221 + (batch_src, batch_tgt, output_file.name))
222 + returned = subprocess.run([sys.executable, __file__],
223 + input=input_bytes,
224 + capture_output=True)
225 + # check if the subprocess is successful
226 + try:
227 + returned.check_returncode()
228 + except Exception as e:
229 + # wrap raised exception to provide more information
230 + raise RuntimeError(
231 + f"Error happened when batch testing "
232 + f"peer-to-peer access from {batch_src} to {batch_tgt}:\n"
233 + f"{returned.stderr.decode()}") from e
234 + with open(output_file.name, "rb") as f:
235 + result = pickle.load(f)
236 for _i, _j, r in zip(batch_src, batch_tgt, result):
237 cache[f"{_i}->{_j}"] = r
238 with open(path, "w") as f:
@@ -241,6 +249,7 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
249 __all__ = ["gpu_p2p_access_check"]
250
251 if __name__ == "__main__":
244 - batch_src, batch_tgt = pickle.loads(sys.stdin.buffer.read())
252 + batch_src, batch_tgt, output_file = pickle.loads(sys.stdin.buffer.read())
253 result = can_actually_p2p(batch_src, batch_tgt)
246 - sys.stdout.buffer.write(pickle.dumps(result))
254 + with open(output_file, "wb") as f:
255 + f.write(pickle.dumps(result))