A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[ROCm] Avoid using the default stream on ROCm (#13238)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
Feb 13, 2025 at 20:29 UTC
410886950ac5ce773a57ae115a31cdecfcdd4af1
1 file changed
+6
-1
vllm/utils.py
+6
-1
@@ -942,11 +942,16 @@ def current_stream() -> torch.cuda.Stream:
942
the underlying hypothesis is that we do not call `torch._C._cuda_setStream`
943
from C/C++ code.
944
"""
945
+ from vllm.platforms import current_platform
946
global _current_stream
947
if _current_stream is None:
948
# when this function is called before any stream is set,
949
# we return the default stream.
949
- _current_stream = torch.cuda.current_stream()
950
+ # On ROCm using the default 0 stream in combination with RCCL
951
+ # is hurting performance. Therefore creating a dedicated stream
952
+ # per process
953
+ _current_stream = torch.cuda.Stream() if current_platform.is_rocm(
954
+ ) else torch.cuda.current_stream()
955
return _current_stream
956
957