@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eae3d48181
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Use temporary directory in registry (#9721)

Cyrus Leung committed Oct 29, 2024 at 13:08 UTC eae3d48181b1ad27f132f14df18e8cff203f7552
1 file changed +8 -3
vllm/model_executor/models/registry.py
+8 -3
@@ -1,4 +1,5 @@
1 import importlib
2 +import os
3 import pickle
4 import subprocess
5 import sys
@@ -423,9 +424,13 @@ _T = TypeVar("_T")
424
425
426 def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
426 - with tempfile.NamedTemporaryFile() as output_file:
427 + # NOTE: We use a temporary directory instead of a temporary file to avoid
428 + # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
429 + with tempfile.TemporaryDirectory() as tempdir:
430 + output_filepath = os.path.join(tempdir, "registry_output.tmp")
431 +
432 # `cloudpickle` allows pickling lambda functions directly
428 - input_bytes = cloudpickle.dumps((fn, output_file.name))
433 + input_bytes = cloudpickle.dumps((fn, output_filepath))
434
435 # cannot use `sys.executable __file__` here because the script
436 # contains relative imports
@@ -442,7 +447,7 @@ def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
447 raise RuntimeError(f"Error raised in subprocess:\n"
448 f"{returned.stderr.decode()}") from e
449
445 - with open(output_file.name, "rb") as f:
450 + with open(output_filepath, "rb") as f:
451 return pickle.load(f)
452
453