A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Use temporary directory in registry (#9721)
Cyrus Leung committed
Oct 29, 2024 at 13:08 UTC
eae3d48181b1ad27f132f14df18e8cff203f7552
1 file changed
+8
-3
vllm/model_executor/models/registry.py
+8
-3
@@ -1,4 +1,5 @@
1
import importlib
2
+import os
3
import pickle
4
import subprocess
5
import sys
@@ -423,9 +424,13 @@ _T = TypeVar("_T")
424
425
426
def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
426
- with tempfile.NamedTemporaryFile() as output_file:
427
+ # NOTE: We use a temporary directory instead of a temporary file to avoid
428
+ # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file
429
+ with tempfile.TemporaryDirectory() as tempdir:
430
+ output_filepath = os.path.join(tempdir, "registry_output.tmp")
431
+
432
# `cloudpickle` allows pickling lambda functions directly
428
- input_bytes = cloudpickle.dumps((fn, output_file.name))
433
+ input_bytes = cloudpickle.dumps((fn, output_filepath))
434
435
# cannot use `sys.executable __file__` here because the script
436
# contains relative imports
@@ -442,7 +447,7 @@ def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
447
raise RuntimeError(f"Error raised in subprocess:\n"
448
f"{returned.stderr.decode()}") from e
449
445
- with open(output_file.name, "rb") as f:
450
+ with open(output_filepath, "rb") as f:
451
return pickle.load(f)
452
453