@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eaec4b9153
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Add custom Triton cache manager to resolve MoE MP issue (#6140)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Chih-Chieh-Yang <chih.chieh.yang@ibm.com>

Thomas Parnell committed Jul 15, 2024 at 19:12 UTC eaec4b915347d839b0e99bcd57475730a715492c
3 files changed +64
vllm/executor/multiproc_gpu_executor.py
+5
@@ -9,6 +9,7 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
9 ResultHandler, WorkerMonitor)
10 from vllm.logger import init_logger
11 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 +from vllm.triton_utils import maybe_set_triton_cache_manager
13 from vllm.utils import (cuda_device_count_stateless,
14 error_on_invalid_device_count_status,
15 get_distributed_init_method, get_open_port,
@@ -42,6 +43,10 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
43 if "OMP_NUM_THREADS" not in os.environ:
44 os.environ["OMP_NUM_THREADS"] = "1"
45
46 + # workaround for https://github.com/vllm-project/vllm/issues/6103
47 + if world_size > 1:
48 + maybe_set_triton_cache_manager()
49 +
50 assert world_size <= cuda_device_count_stateless(), (
51 "please set tensor_parallel_size to less than max local gpu count")
52
vllm/triton_utils/__init__.py new
+6
@@ -0,0 +1,6 @@
1 +from vllm.triton_utils.custom_cache_manager import (
2 + maybe_set_triton_cache_manager)
3 +
4 +__all__ = [
5 + "maybe_set_triton_cache_manager",
6 +]
vllm/triton_utils/custom_cache_manager.py new
+53
@@ -0,0 +1,53 @@
1 +import os
2 +
3 +from triton.runtime.cache import (FileCacheManager, default_cache_dir,
4 + default_dump_dir, default_override_dir)
5 +
6 +from vllm.logger import init_logger
7 +
8 +logger = init_logger(__name__)
9 +
10 +
11 +def maybe_set_triton_cache_manager() -> None:
12 + """Set environment variable to tell Triton to use a
13 + custom cache manager"""
14 + cache_manger = os.environ.get("TRITON_CACHE_MANAGER", None)
15 + if cache_manger is None:
16 + manager = "vllm.triton_utils.custom_cache_manager:CustomCacheManager"
17 + logger.info("Setting Triton cache manager to: %s", manager)
18 + os.environ["TRITON_CACHE_MANAGER"] = manager
19 +
20 +
21 +class CustomCacheManager(FileCacheManager):
22 + """Re-implements Triton's cache manager, ensuring that a
23 + unique cache directory is created for each process. This is
24 + needed to avoid collisions when running with tp>1 and
25 + using multi-processing as the distributed backend.
26 +
27 + Note this issue was fixed by triton-lang/triton/pull/4295,
28 + but the fix is not yet included in triton==v3.0.0. However,
29 + it should be included in the subsequent version.
30 + """
31 +
32 + def __init__(self, key, override=False, dump=False):
33 + self.key = key
34 + self.lock_path = None
35 + if dump:
36 + self.cache_dir = default_dump_dir()
37 + self.cache_dir = os.path.join(self.cache_dir, self.key)
38 + self.lock_path = os.path.join(self.cache_dir, "lock")
39 + os.makedirs(self.cache_dir, exist_ok=True)
40 + elif override:
41 + self.cache_dir = default_override_dir()
42 + self.cache_dir = os.path.join(self.cache_dir, self.key)
43 + else:
44 + # create cache directory if it doesn't exist
45 + self.cache_dir = os.getenv("TRITON_CACHE_DIR",
46 + "").strip() or default_cache_dir()
47 + if self.cache_dir:
48 + self.cache_dir = f"{self.cache_dir}_{os.getpid()}"
49 + self.cache_dir = os.path.join(self.cache_dir, self.key)
50 + self.lock_path = os.path.join(self.cache_dir, "lock")
51 + os.makedirs(self.cache_dir, exist_ok=True)
52 + else:
53 + raise RuntimeError("Could not create or locate cache dir")