@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06ec486794
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Install `flash_attn` in Docker image (#3396)

Thomas Parnell committed Mar 14, 2024 at 18:55 UTC 06ec486794f42db656c3cc16c8c5ed56ce4f696b
1 file changed +24
Dockerfile
+24
@@ -57,6 +57,22 @@ ENV VLLM_INSTALL_PUNICA_KERNELS=1
57 RUN python3 setup.py build_ext --inplace
58 #################### EXTENSION Build IMAGE ####################
59
60 +#################### FLASH_ATTENTION Build IMAGE ####################
61 +FROM dev as flash-attn-builder
62 +# max jobs used for build
63 +ARG max_jobs=2
64 +ENV MAX_JOBS=${max_jobs}
65 +# flash attention version
66 +ARG flash_attn_version=v2.5.6
67 +ENV FLASH_ATTN_VERSION=${flash_attn_version}
68 +
69 +WORKDIR /usr/src/flash-attention-v2
70 +
71 +# Download the wheel or build it if a pre-compiled release doesn't exist
72 +RUN pip --verbose wheel flash-attn==${FLASH_ATTN_VERSION} \
73 + --no-build-isolation --no-deps --no-cache-dir
74 +
75 +#################### FLASH_ATTENTION Build IMAGE ####################
76
77 #################### TEST IMAGE ####################
78 # image to run unit testing suite
@@ -68,6 +84,9 @@ WORKDIR /vllm-workspace
84 # ADD is used to preserve directory structure
85 ADD . /vllm-workspace/
86 COPY --from=build /workspace/vllm/*.so /vllm-workspace/vllm/
87 +# Install flash attention (from pre-built wheel)
88 +RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
89 + pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
90 # ignore build dependencies installation because we are using pre-complied extensions
91 RUN rm pyproject.toml
92 RUN --mount=type=cache,target=/root/.cache/pip VLLM_USE_PRECOMPILED=1 pip install . --verbose
@@ -88,6 +107,11 @@ WORKDIR /workspace
107 COPY requirements.txt requirements.txt
108 RUN --mount=type=cache,target=/root/.cache/pip \
109 pip install -r requirements.txt
110 +
111 +# Install flash attention (from pre-built wheel)
112 +RUN --mount=type=bind,from=flash-attn-builder,src=/usr/src/flash-attention-v2,target=/usr/src/flash-attention-v2 \
113 + pip install /usr/src/flash-attention-v2/*.whl --no-cache-dir
114 +
115 #################### RUNTIME BASE IMAGE ####################
116
117