@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9765b5c406
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[ROCm][Bugfix] Fixed several bugs related to rccl path and attention selector logic (#3699)

Hongxia Yang committed Mar 29, 2024 at 17:52 UTC 9765b5c4061b37c0db417aa6c638b0ca90183be1
4 files changed +5 -5
Dockerfile.rocm
+1 -1
@@ -90,6 +90,6 @@ RUN cd /app \
90 && cd ..
91
92 RUN python3 -m pip install --upgrade pip
93 -RUN python3 -m pip install --no-cache-dir ray[all]
93 +RUN python3 -m pip install --no-cache-dir ray[all]==2.9.3
94
95 CMD ["/bin/bash"]
requirements-rocm.txt
+1 -1
@@ -5,7 +5,7 @@ starlette
5 requests
6 py-cpuinfo
7 psutil
8 -ray >= 2.9
8 +ray == 2.9.3
9 sentencepiece # Required for LLaMA tokenizer.
10 numpy
11 tokenizers>=0.15.0
vllm/attention/backends/xformers.py
+2 -2
@@ -405,8 +405,8 @@ def _check_use_naive_attention() -> bool:
405 if not is_hip():
406 return False
407 # For ROCm, check whether flash attention is installed or not.
408 - has_flash_attn = importlib.util.find_spec("flash_attn") is None
409 - if not has_flash_attn:
408 + use_naive_attention = importlib.util.find_spec("flash_attn") is None
409 + if use_naive_attention:
410 logger.warning("flash_attn is not installed. Using naive attention. "
411 "This will take significantly more GPU memory.")
412 return True
vllm/model_executor/parallel_utils/pynccl.py
+1 -1
@@ -41,7 +41,7 @@ else:
41 if torch.version.cuda is not None:
42 so_file = "libnccl.so.2"
43 elif torch.version.hip is not None:
44 - so_file = "librccl.so.2"
44 + so_file = "librccl.so.1"
45 else:
46 raise ValueError("NCCL only supports CUDA and ROCm backends.")
47 logger.debug(f"Loading nccl from library {so_file}")