@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 410886950a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[ROCm] Avoid using the default stream on ROCm (#13238)

Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>

Gregory Shtrasberg committed Feb 13, 2025 at 20:29 UTC 410886950ac5ce773a57ae115a31cdecfcdd4af1
1 file changed +6 -1
vllm/utils.py
+6 -1
@@ -942,11 +942,16 @@ def current_stream() -> torch.cuda.Stream:
942 the underlying hypothesis is that we do not call `torch._C._cuda_setStream`
943 from C/C++ code.
944 """
945 + from vllm.platforms import current_platform
946 global _current_stream
947 if _current_stream is None:
948 # when this function is called before any stream is set,
949 # we return the default stream.
949 - _current_stream = torch.cuda.current_stream()
950 + # On ROCm using the default 0 stream in combination with RCCL
951 + # is hurting performance. Therefore creating a dedicated stream
952 + # per process
953 + _current_stream = torch.cuda.Stream() if current_platform.is_rocm(
954 + ) else torch.cuda.current_stream()
955 return _current_stream
956
957