@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4b62d4641
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[AMD][Build] Porting dockerfiles from the ROCm/vllm fork (#11777)

Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>

Gregory Shtrasberg committed Jan 20, 2025 at 23:22 UTC d4b62d4641377c104baa2de7807f2c61d091cfe2
7 files changed +337 -236
Dockerfile.rocm
+101 -157
@@ -1,174 +1,118 @@
1 -# Default ROCm 6.2 base image
2 -ARG BASE_IMAGE="rocm/pytorch:rocm6.2_ubuntu20.04_py3.9_pytorch_release_2.3.0"
1 +# default base image
2 +ARG REMOTE_VLLM="0"
3 +ARG USE_CYTHON="0"
4 +ARG BUILD_RPD="1"
5 +ARG COMMON_WORKDIR=/app
6 +ARG BASE_IMAGE=rocm/vllm-dev:base
7
4 -# Default ROCm ARCHes to build vLLM for.
5 -ARG PYTORCH_ROCM_ARCH="gfx908;gfx90a;gfx942;gfx1100"
8 +FROM ${BASE_IMAGE} AS base
9
7 -# Whether to install CK-based flash-attention
8 -# If 0, will not install flash-attention
9 -ARG BUILD_FA="1"
10 -ARG FA_GFX_ARCHS="gfx90a;gfx942"
11 -ARG FA_BRANCH="3cea2fb"
12 -
13 -# Whether to build triton on rocm
14 -ARG BUILD_TRITON="1"
15 -ARG TRITON_BRANCH="e192dba"
16 -
17 -### Base image build stage
18 -FROM $BASE_IMAGE AS base
19 -
20 -# Import arg(s) defined before this build stage
21 -ARG PYTORCH_ROCM_ARCH
10 +ARG ARG_PYTORCH_ROCM_ARCH
11 +ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
12
13 # Install some basic utilities
24 -RUN apt-get update && apt-get install python3 python3-pip -y
25 -RUN apt-get update && apt-get install -y \
26 - curl \
27 - ca-certificates \
28 - sudo \
29 - git \
30 - bzip2 \
31 - libx11-6 \
32 - build-essential \
33 - wget \
34 - unzip \
35 - tmux \
36 - ccache \
37 - && rm -rf /var/lib/apt/lists/*
38 -
39 -# When launching the container, mount the code directory to /vllm-workspace
40 -ARG APP_MOUNT=/vllm-workspace
41 -WORKDIR ${APP_MOUNT}
42 -
43 -RUN python3 -m pip install --upgrade pip
44 -# Remove sccache so it doesn't interfere with ccache
45 -# TODO: implement sccache support across components
14 +RUN apt-get update -q -y && apt-get install -q -y \
15 + sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev
16 +# Remove sccache
17 +RUN python3 -m pip install --upgrade pip && pip install setuptools_scm
18 RUN apt-get purge -y sccache; python3 -m pip uninstall -y sccache; rm -f "$(which sccache)"
19 +ARG COMMON_WORKDIR
20 +WORKDIR ${COMMON_WORKDIR}
21 +
22 +
23 +# -----------------------
24 +# vLLM fetch stages
25 +FROM base AS fetch_vllm_0
26 +ONBUILD COPY ./ vllm/
27 +FROM base AS fetch_vllm_1
28 +ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
29 +ARG VLLM_BRANCH="main"
30 +ONBUILD RUN git clone ${VLLM_REPO} \
31 + && cd vllm \
32 + && git checkout ${VLLM_BRANCH}
33 +FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
34 +
35 +# -----------------------
36 +# vLLM build stages
37 +FROM fetch_vllm AS build_vllm
38 +ARG USE_CYTHON
39 +# Build vLLM
40 +RUN cd vllm \
41 + && python3 -m pip install -r requirements-rocm.txt \
42 + && python3 setup.py clean --all \
43 + && if [ ${USE_CYTHON} -eq "1" ]; then python3 setup_cython.py build_ext --inplace; fi \
44 + && python3 setup.py bdist_wheel --dist-dir=dist
45 +FROM scratch AS export_vllm
46 +ARG COMMON_WORKDIR
47 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/dist/*.whl /
48 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements*.txt /
49 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
50 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
51 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
52 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
53 +
54 +# -----------------------
55 +# Test vLLM image
56 +FROM base AS test
57 +
58 +RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/*
59 +
60 +# Install vLLM
61 +RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
62 + cd /install \
63 + && pip install -U -r requirements-rocm.txt \
64 + && pip uninstall -y vllm \
65 + && pip install *.whl
66 +
67 +WORKDIR /vllm-workspace
68 +ARG COMMON_WORKDIR
69 +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace
70
48 -# Install torch == 2.6.0 on ROCm
49 -RUN --mount=type=cache,target=/root/.cache/pip \
50 - case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
51 - *"rocm-6.2"*) \
52 - python3 -m pip uninstall -y torch torchvision \
53 - && python3 -m pip install --pre \
54 - torch \
55 - 'setuptools-scm>=8' \
56 - torchvision \
57 - --extra-index-url https://download.pytorch.org/whl/rocm6.2;; \
58 - *) ;; esac
71 +# install development dependencies (for testing)
72 +RUN cd /vllm-workspace \
73 + && rm -rf vllm \
74 + && python3 -m pip install -e tests/vllm_test_utils \
75 + && python3 -m pip install lm-eval[api]==0.4.4
76
60 -ENV LLVM_SYMBOLIZER_PATH=/opt/rocm/llvm/bin/llvm-symbolizer
61 -ENV PATH=$PATH:/opt/rocm/bin:/libtorch/bin:
62 -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/libtorch/lib:
63 -ENV CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/libtorch/include:/libtorch/include/torch/csrc/api/include/:/opt/rocm/include/:
64 -
65 -ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}
66 -ENV CCACHE_DIR=/root/.cache/ccache
67 -
68 -
69 -### AMD-SMI build stage
70 -FROM base AS build_amdsmi
71 -# Build amdsmi wheel always
72 -RUN cd /opt/rocm/share/amd_smi \
73 - && python3 -m pip wheel . --wheel-dir=/install
74 -
75 -
76 -### Flash-Attention wheel build stage
77 -FROM base AS build_fa
78 -ARG BUILD_FA
79 -ARG FA_GFX_ARCHS
80 -ARG FA_BRANCH
81 -# Build ROCm flash-attention wheel if `BUILD_FA = 1`
82 -RUN --mount=type=cache,target=${CCACHE_DIR} \
83 - if [ "$BUILD_FA" = "1" ]; then \
84 - mkdir -p libs \
85 - && cd libs \
86 - && git clone https://github.com/ROCm/flash-attention.git \
87 - && cd flash-attention \
88 - && git checkout "${FA_BRANCH}" \
89 - && git submodule update --init \
90 - && GPU_ARCHS="${FA_GFX_ARCHS}" python3 setup.py bdist_wheel --dist-dir=/install; \
91 - # Create an empty directory otherwise as later build stages expect one
92 - else mkdir -p /install; \
93 - fi
94 -
95 -
96 -### Triton wheel build stage
97 -FROM base AS build_triton
98 -ARG BUILD_TRITON
99 -ARG TRITON_BRANCH
100 -# Build triton wheel if `BUILD_TRITON = 1`
101 -RUN --mount=type=cache,target=${CCACHE_DIR} \
102 - if [ "$BUILD_TRITON" = "1" ]; then \
103 - mkdir -p libs \
104 - && cd libs \
105 - && python3 -m pip install ninja cmake wheel pybind11 \
106 - && git clone https://github.com/OpenAI/triton.git \
107 - && cd triton \
108 - && git checkout "${TRITON_BRANCH}" \
109 - && cd python \
110 - && python3 setup.py bdist_wheel --dist-dir=/install; \
111 - # Create an empty directory otherwise as later build stages expect one
112 - else mkdir -p /install; \
113 - fi
114 -
115 -
116 -### Final vLLM build stage
77 +# -----------------------
78 +# Final vLLM image
79 FROM base AS final
118 -# Import the vLLM development directory from the build context
119 -COPY . .
120 -ARG GIT_REPO_CHECK=0
121 -RUN --mount=type=bind,source=.git,target=.git \
122 - if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
80
124 -RUN python3 -m pip install --upgrade pip
81 +RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/*
82 +# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt.
83 +# Manually remove it so that later steps of numpy upgrade can continue
84 +RUN case "$(which python3)" in \
85 + *"/opt/conda/envs/py_3.9"*) \
86 + rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/;; \
87 + *) ;; esac
88
126 -# Package upgrades for useful functionality or to avoid dependency issues
127 -RUN --mount=type=cache,target=/root/.cache/pip \
128 - python3 -m pip install --upgrade numba scipy huggingface-hub[cli] pytest-shard
89 +RUN python3 -m pip install --upgrade huggingface-hub[cli]
90 +ARG BUILD_RPD
91 +RUN if [ ${BUILD_RPD} -eq "1" ]; then \
92 + git clone -b nvtx_enabled https://github.com/ROCm/rocmProfileData.git \
93 + && cd rocmProfileData/rpd_tracer \
94 + && pip install -r requirements.txt && cd ../ \
95 + && make && make install \
96 + && cd hipMarker && python3 setup.py install ; fi
97
98 +# Install vLLM
99 +RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
100 + cd /install \
101 + && pip install -U -r requirements-rocm.txt \
102 + && pip uninstall -y vllm \
103 + && pip install *.whl
104 +
105 +ARG COMMON_WORKDIR
106 +
107 +# Copy over the benchmark scripts as well
108 +COPY --from=export_vllm /benchmarks ${COMMON_WORKDIR}/vllm/benchmarks
109 +COPY --from=export_vllm /examples ${COMMON_WORKDIR}/vllm/examples
110
131 -# Workaround for ray >= 2.10.0
111 ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
133 -# Silences the HF Tokenizers warning
112 ENV TOKENIZERS_PARALLELISM=false
113
136 -RUN --mount=type=cache,target=${CCACHE_DIR} \
137 - --mount=type=bind,source=.git,target=.git \
138 - --mount=type=cache,target=/root/.cache/pip \
139 - python3 -m pip install -Ur requirements-rocm.txt \
140 - && python3 setup.py clean --all \
141 - && python3 setup.py develop
142 -
143 -# Copy amdsmi wheel into final image
144 -RUN --mount=type=bind,from=build_amdsmi,src=/install,target=/install \
145 - mkdir -p libs \
146 - && cp /install/*.whl libs \
147 - # Preemptively uninstall to avoid same-version no-installs
148 - && python3 -m pip uninstall -y amdsmi;
149 -
150 -# Copy triton wheel(s) into final image if they were built
151 -RUN --mount=type=bind,from=build_triton,src=/install,target=/install \
152 - mkdir -p libs \
153 - && if ls /install/*.whl; then \
154 - cp /install/*.whl libs \
155 - # Preemptively uninstall to avoid same-version no-installs
156 - && python3 -m pip uninstall -y triton; fi
157 -
158 -# Copy flash-attn wheel(s) into final image if they were built
159 -RUN --mount=type=bind,from=build_fa,src=/install,target=/install \
160 - mkdir -p libs \
161 - && if ls /install/*.whl; then \
162 - cp /install/*.whl libs \
163 - # Preemptively uninstall to avoid same-version no-installs
164 - && python3 -m pip uninstall -y flash-attn; fi
165 -
166 -# Install wheels that were built to the final image
167 -RUN --mount=type=cache,target=/root/.cache/pip \
168 - if ls libs/*.whl; then \
169 - python3 -m pip install libs/*.whl; fi
170 -
171 -# install development dependencies (for testing)
172 -RUN python3 -m pip install -e tests/vllm_test_utils
114 +# Performance environment variable.
115 +ENV HIP_FORCE_DEV_KERNARG=1
116
117 CMD ["/bin/bash"]
118 +
Dockerfile.rocm_base new
+158
@@ -0,0 +1,158 @@
1 +ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:6.3.1-complete
2 +ARG HIPBLASLT_BRANCH="4d40e36"
3 +ARG HIPBLAS_COMMON_BRANCH="7c1566b"
4 +ARG LEGACY_HIPBLASLT_OPTION=
5 +ARG RCCL_BRANCH="648a58d"
6 +ARG RCCL_REPO="https://github.com/ROCm/rccl"
7 +ARG TRITON_BRANCH="e5be006"
8 +ARG TRITON_REPO="https://github.com/triton-lang/triton.git"
9 +ARG PYTORCH_BRANCH="8d4926e"
10 +ARG PYTORCH_VISION_BRANCH="v0.19.1"
11 +ARG PYTORCH_REPO="https://github.com/pytorch/pytorch.git"
12 +ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git"
13 +ARG FA_BRANCH="b7d29fb"
14 +ARG FA_REPO="https://github.com/ROCm/flash-attention.git"
15 +
16 +FROM ${BASE_IMAGE} AS base
17 +
18 +ENV PATH=/opt/rocm/llvm/bin:$PATH
19 +ENV ROCM_PATH=/opt/rocm
20 +ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:
21 +ARG PYTORCH_ROCM_ARCH=gfx90a;gfx942
22 +ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}
23 +
24 +ARG PYTHON_VERSION=3.12
25 +
26 +RUN mkdir -p /app
27 +WORKDIR /app
28 +ENV DEBIAN_FRONTEND=noninteractive
29 +
30 +# Install Python and other dependencies
31 +RUN apt-get update -y \
32 + && apt-get install -y software-properties-common git curl sudo vim less \
33 + && add-apt-repository ppa:deadsnakes/ppa \
34 + && apt-get update -y \
35 + && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
36 + python${PYTHON_VERSION}-lib2to3 python-is-python3 \
37 + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
38 + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
39 + && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
40 + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
41 + && python3 --version && python3 -m pip --version
42 +
43 +RUN pip install -U packaging cmake ninja wheel setuptools pybind11 Cython
44 +
45 +FROM base AS build_hipblaslt
46 +ARG HIPBLASLT_BRANCH
47 +ARG HIPBLAS_COMMON_BRANCH
48 +# Set to "--legacy_hipblas_direct" for ROCm<=6.2
49 +ARG LEGACY_HIPBLASLT_OPTION
50 +RUN git clone https://github.com/ROCm/hipBLAS-common.git
51 +RUN cd hipBLAS-common \
52 + && git checkout ${HIPBLAS_COMMON_BRANCH} \
53 + && mkdir build \
54 + && cd build \
55 + && cmake .. \
56 + && make package \
57 + && dpkg -i ./*.deb
58 +RUN git clone https://github.com/ROCm/hipBLASLt
59 +RUN cd hipBLASLt \
60 + && git checkout ${HIPBLASLT_BRANCH} \
61 + && ./install.sh -d --architecture ${PYTORCH_ROCM_ARCH} ${LEGACY_HIPBLASLT_OPTION} \
62 + && cd build/release \
63 + && make package
64 +RUN mkdir -p /app/install && cp /app/hipBLASLt/build/release/*.deb /app/hipBLAS-common/build/*.deb /app/install
65 +
66 +FROM base AS build_rccl
67 +ARG RCCL_BRANCH
68 +ARG RCCL_REPO
69 +RUN git clone ${RCCL_REPO}
70 +RUN cd rccl \
71 + && git checkout ${RCCL_BRANCH} \
72 + && ./install.sh -p --amdgpu_targets ${PYTORCH_ROCM_ARCH}
73 +RUN mkdir -p /app/install && cp /app/rccl/build/release/*.deb /app/install
74 +
75 +FROM base AS build_triton
76 +ARG TRITON_BRANCH
77 +ARG TRITON_REPO
78 +RUN git clone ${TRITON_REPO}
79 +RUN cd triton \
80 + && git checkout ${TRITON_BRANCH} \
81 + && cd python \
82 + && python3 setup.py bdist_wheel --dist-dir=dist
83 +RUN mkdir -p /app/install && cp /app/triton/python/dist/*.whl /app/install
84 +
85 +FROM base AS build_amdsmi
86 +RUN cd /opt/rocm/share/amd_smi \
87 + && pip wheel . --wheel-dir=dist
88 +RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
89 +
90 +FROM base AS build_pytorch
91 +ARG PYTORCH_BRANCH
92 +ARG PYTORCH_VISION_BRANCH
93 +ARG PYTORCH_REPO
94 +ARG PYTORCH_VISION_REPO
95 +ARG FA_BRANCH
96 +ARG FA_REPO
97 +RUN git clone ${PYTORCH_REPO} pytorch
98 +RUN cd pytorch && git checkout ${PYTORCH_BRANCH} && \
99 + pip install -r requirements.txt && git submodule update --init --recursive \
100 + && python3 tools/amd_build/build_amd.py \
101 + && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
102 + && pip install dist/*.whl
103 +RUN git clone ${PYTORCH_VISION_REPO} vision
104 +RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
105 + && python3 setup.py bdist_wheel --dist-dir=dist \
106 + && pip install dist/*.whl
107 +RUN git clone ${FA_REPO}
108 +RUN cd flash-attention \
109 + && git checkout ${FA_BRANCH} \
110 + && git submodule update --init \
111 + && MAX_JOBS=64 GPU_ARCHS=${PYTORCH_ROCM_ARCH} python3 setup.py bdist_wheel --dist-dir=dist
112 +RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install \
113 + && cp /app/vision/dist/*.whl /app/install \
114 + && cp /app/flash-attention/dist/*.whl /app/install
115 +
116 +FROM base AS final
117 +RUN --mount=type=bind,from=build_hipblaslt,src=/app/install/,target=/install \
118 + dpkg -i /install/*deb \
119 + && sed -i 's/, hipblaslt-dev \(.*\), hipcub-dev/, hipcub-dev/g' /var/lib/dpkg/status \
120 + && sed -i 's/, hipblaslt \(.*\), hipfft/, hipfft/g' /var/lib/dpkg/status
121 +RUN --mount=type=bind,from=build_rccl,src=/app/install/,target=/install \
122 + dpkg -i /install/*deb \
123 + && sed -i 's/, rccl-dev \(.*\), rocalution/, rocalution/g' /var/lib/dpkg/status \
124 + && sed -i 's/, rccl \(.*\), rocalution/, rocalution/g' /var/lib/dpkg/status
125 +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
126 + pip install /install/*.whl
127 +RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
128 + pip install /install/*.whl
129 +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
130 + pip install /install/*.whl
131 +
132 +ARG BASE_IMAGE
133 +ARG HIPBLASLT_BRANCH
134 +ARG LEGACY_HIPBLASLT_OPTION
135 +ARG RCCL_BRANCH
136 +ARG RCCL_REPO
137 +ARG TRITON_BRANCH
138 +ARG TRITON_REPO
139 +ARG PYTORCH_BRANCH
140 +ARG PYTORCH_VISION_BRANCH
141 +ARG PYTORCH_REPO
142 +ARG PYTORCH_VISION_REPO
143 +ARG FA_BRANCH
144 +ARG FA_REPO
145 +RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
146 + && echo "HIPBLAS_COMMON_BRANCH: ${HIPBLAS_COMMON_BRANCH}" >> /app/versions.txt \
147 + && echo "HIPBLASLT_BRANCH: ${HIPBLASLT_BRANCH}" >> /app/versions.txt \
148 + && echo "LEGACY_HIPBLASLT_OPTION: ${LEGACY_HIPBLASLT_OPTION}" >> /app/versions.txt \
149 + && echo "RCCL_BRANCH: ${RCCL_BRANCH}" >> /app/versions.txt \
150 + && echo "RCCL_REPO: ${RCCL_REPO}" >> /app/versions.txt \
151 + && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \
152 + && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \
153 + && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \
154 + && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \
155 + && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \
156 + && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \
157 + && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
158 + && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt
docs/source/getting_started/installation/gpu/rocm.inc.md
+6 -7
@@ -123,11 +123,10 @@ It is important that the user kicks off the docker build using buildkit. Either
123 <gh-file:Dockerfile.rocm> uses ROCm 6.2 by default, but also supports ROCm 5.7, 6.0 and 6.1 in older vLLM branches.
124 It provides flexibility to customize the build of docker image using the following arguments:
125
126 -- `BASE_IMAGE`: specifies the base image used when running `docker build`, specifically the PyTorch on ROCm base image.
127 -- `BUILD_FA`: specifies whether to build CK flash-attention. The default is 1. For [Radeon RX 7900 series (gfx1100)](https://rocm.docs.amd.com/projects/radeon/en/latest/index.html), this should be set to 0 before flash-attention supports this target.
128 -- `FX_GFX_ARCHS`: specifies the GFX architecture that is used to build CK flash-attention, for example, `gfx90a;gfx942` for MI200 and MI300. The default is `gfx90a;gfx942`
129 -- `FA_BRANCH`: specifies the branch used to build the CK flash-attention in [ROCm's flash-attention repo](https://github.com/ROCmSoftwarePlatform/flash-attention). The default is `ae7928c`
130 -- `BUILD_TRITON`: specifies whether to build triton flash-attention. The default value is 1.
126 +- `BASE_IMAGE`: specifies the base image used when running `docker build`. The default value `rocm/vllm-dev:base` is an image published and maintained by AMD. It is being built using <gh-file:Dockerfile.rocm_base>
127 +- `USE_CYTHON`: An option to run cython compilation on a subset of python files upon docker build
128 +- `BUILD_RPD`: Include RocmProfileData profiling tool in the image
129 +- `ARG_PYTORCH_ROCM_ARCH`: Allows to override the gfx architecture values from the base docker image
130
131 Their values can be passed in when running `docker build` with `--build-arg` options.
132
@@ -137,10 +136,10 @@ To build vllm on ROCm 6.2 for MI200 and MI300 series, you can use the default:
136 DOCKER_BUILDKIT=1 docker build -f Dockerfile.rocm -t vllm-rocm .
137 ```
138
140 -To build vllm on ROCm 6.2 for Radeon RX7900 series (gfx1100), you should specify `BUILD_FA` as below:
139 +To build vllm on ROCm 6.2 for Radeon RX7900 series (gfx1100), you should pick the alternative base image:
140
141 ```console
143 -DOCKER_BUILDKIT=1 docker build --build-arg BUILD_FA="0" -f Dockerfile.rocm -t vllm-rocm .
142 +DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE="rocm/vllm-dev:navi_base" -f Dockerfile.rocm -t vllm-rocm .
143 ```
144
145 To run the above docker image `vllm-rocm`, use the below command:
vllm/model_executor/layers/fused_moe/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json
+18 -18
@@ -5,7 +5,7 @@
5 "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 "num_warps": 2,
8 - "num_stages": 0,
8 + "num_stages": 2,
9 "waves_per_eu": 0,
10 "matrix_instr_nonkdim": 16,
11 "kpack": 1
@@ -16,7 +16,7 @@
16 "BLOCK_SIZE_K": 128,
17 "GROUP_SIZE_M": 1,
18 "num_warps": 2,
19 - "num_stages": 0,
19 + "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
22 "kpack": 2
@@ -27,7 +27,7 @@
27 "BLOCK_SIZE_K": 256,
28 "GROUP_SIZE_M": 1,
29 "num_warps": 2,
30 - "num_stages": 0,
30 + "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
33 "kpack": 2
@@ -38,7 +38,7 @@
38 "BLOCK_SIZE_K": 256,
39 "GROUP_SIZE_M": 1,
40 "num_warps": 1,
41 - "num_stages": 0,
41 + "num_stages": 2,
42 "waves_per_eu": 0,
43 "matrix_instr_nonkdim": 16,
44 "kpack": 2
@@ -49,7 +49,7 @@
49 "BLOCK_SIZE_K": 256,
50 "GROUP_SIZE_M": 1,
51 "num_warps": 4,
52 - "num_stages": 0,
52 + "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
55 "kpack": 2
@@ -60,7 +60,7 @@
60 "BLOCK_SIZE_K": 64,
61 "GROUP_SIZE_M": 1,
62 "num_warps": 1,
63 - "num_stages": 0,
63 + "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
66 "kpack": 2
@@ -71,7 +71,7 @@
71 "BLOCK_SIZE_K": 128,
72 "GROUP_SIZE_M": 4,
73 "num_warps": 2,
74 - "num_stages": 0,
74 + "num_stages": 2,
75 "waves_per_eu": 0,
76 "matrix_instr_nonkdim": 16,
77 "kpack": 1
@@ -82,7 +82,7 @@
82 "BLOCK_SIZE_K": 128,
83 "GROUP_SIZE_M": 4,
84 "num_warps": 2,
85 - "num_stages": 0,
85 + "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 "kpack": 2
@@ -93,7 +93,7 @@
93 "BLOCK_SIZE_K": 128,
94 "GROUP_SIZE_M": 4,
95 "num_warps": 8,
96 - "num_stages": 0,
96 + "num_stages": 2,
97 "waves_per_eu": 0,
98 "matrix_instr_nonkdim": 16,
99 "kpack": 2
@@ -104,7 +104,7 @@
104 "BLOCK_SIZE_K": 128,
105 "GROUP_SIZE_M": 4,
106 "num_warps": 4,
107 - "num_stages": 0,
107 + "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
110 "kpack": 2
@@ -115,7 +115,7 @@
115 "BLOCK_SIZE_K": 64,
116 "GROUP_SIZE_M": 4,
117 "num_warps": 8,
118 - "num_stages": 0,
118 + "num_stages": 2,
119 "waves_per_eu": 0,
120 "matrix_instr_nonkdim": 16,
121 "kpack": 2
@@ -126,7 +126,7 @@
126 "BLOCK_SIZE_K": 64,
127 "GROUP_SIZE_M": 4,
128 "num_warps": 8,
129 - "num_stages": 0,
129 + "num_stages": 2,
130 "waves_per_eu": 0,
131 "matrix_instr_nonkdim": 16,
132 "kpack": 1
@@ -137,7 +137,7 @@
137 "BLOCK_SIZE_K": 64,
138 "GROUP_SIZE_M": 4,
139 "num_warps": 8,
140 - "num_stages": 0,
140 + "num_stages": 2,
141 "waves_per_eu": 0,
142 "matrix_instr_nonkdim": 16,
143 "kpack": 2
@@ -148,7 +148,7 @@
148 "BLOCK_SIZE_K": 64,
149 "GROUP_SIZE_M": 1,
150 "num_warps": 8,
151 - "num_stages": 0,
151 + "num_stages": 2,
152 "waves_per_eu": 0,
153 "matrix_instr_nonkdim": 32,
154 "kpack": 2
@@ -159,7 +159,7 @@
159 "BLOCK_SIZE_K": 64,
160 "GROUP_SIZE_M": 1,
161 "num_warps": 8,
162 - "num_stages": 0,
162 + "num_stages": 2,
163 "waves_per_eu": 0,
164 "matrix_instr_nonkdim": 16,
165 "kpack": 2
@@ -170,7 +170,7 @@
170 "BLOCK_SIZE_K": 64,
171 "GROUP_SIZE_M": 1,
172 "num_warps": 8,
173 - "num_stages": 0,
173 + "num_stages": 2,
174 "waves_per_eu": 0,
175 "matrix_instr_nonkdim": 16,
176 "kpack": 2
@@ -181,7 +181,7 @@
181 "BLOCK_SIZE_K": 64,
182 "GROUP_SIZE_M": 1,
183 "num_warps": 8,
184 - "num_stages": 0,
184 + "num_stages": 2,
185 "waves_per_eu": 0,
186 "matrix_instr_nonkdim": 16,
187 "kpack": 1
@@ -192,7 +192,7 @@
192 "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
194 "num_warps": 8,
195 - "num_stages": 0,
195 + "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 "kpack": 1
vllm/model_executor/layers/fused_moe/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json
+18 -18
@@ -5,7 +5,7 @@
5 "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 "num_warps": 2,
8 - "num_stages": 0,
8 + "num_stages": 2,
9 "waves_per_eu": 0,
10 "matrix_instr_nonkdim": 16,
11 "kpack": 2
@@ -16,7 +16,7 @@
16 "BLOCK_SIZE_K": 128,
17 "GROUP_SIZE_M": 1,
18 "num_warps": 4,
19 - "num_stages": 0,
19 + "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
22 "kpack": 1
@@ -27,7 +27,7 @@
27 "BLOCK_SIZE_K": 128,
28 "GROUP_SIZE_M": 1,
29 "num_warps": 4,
30 - "num_stages": 0,
30 + "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
33 "kpack": 2
@@ -38,7 +38,7 @@
38 "BLOCK_SIZE_K": 256,
39 "GROUP_SIZE_M": 1,
40 "num_warps": 2,
41 - "num_stages": 0,
41 + "num_stages": 2,
42 "waves_per_eu": 0,
43 "matrix_instr_nonkdim": 16,
44 "kpack": 2
@@ -49,7 +49,7 @@
49 "BLOCK_SIZE_K": 64,
50 "GROUP_SIZE_M": 1,
51 "num_warps": 8,
52 - "num_stages": 0,
52 + "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
55 "kpack": 1
@@ -60,7 +60,7 @@
60 "BLOCK_SIZE_K": 64,
61 "GROUP_SIZE_M": 1,
62 "num_warps": 4,
63 - "num_stages": 0,
63 + "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
66 "kpack": 1
@@ -71,7 +71,7 @@
71 "BLOCK_SIZE_K": 256,
72 "GROUP_SIZE_M": 4,
73 "num_warps": 2,
74 - "num_stages": 0,
74 + "num_stages": 2,
75 "waves_per_eu": 0,
76 "matrix_instr_nonkdim": 16,
77 "kpack": 2
@@ -82,7 +82,7 @@
82 "BLOCK_SIZE_K": 64,
83 "GROUP_SIZE_M": 1,
84 "num_warps": 8,
85 - "num_stages": 0,
85 + "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 "kpack": 2
@@ -93,7 +93,7 @@
93 "BLOCK_SIZE_K": 64,
94 "GROUP_SIZE_M": 1,
95 "num_warps": 2,
96 - "num_stages": 0,
96 + "num_stages": 2,
97 "waves_per_eu": 0,
98 "matrix_instr_nonkdim": 16,
99 "kpack": 1
@@ -104,7 +104,7 @@
104 "BLOCK_SIZE_K": 256,
105 "GROUP_SIZE_M": 4,
106 "num_warps": 4,
107 - "num_stages": 0,
107 + "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
110 "kpack": 2
@@ -115,7 +115,7 @@
115 "BLOCK_SIZE_K": 64,
116 "GROUP_SIZE_M": 4,
117 "num_warps": 8,
118 - "num_stages": 0,
118 + "num_stages": 2,
119 "waves_per_eu": 0,
120 "matrix_instr_nonkdim": 16,
121 "kpack": 1
@@ -126,7 +126,7 @@
126 "BLOCK_SIZE_K": 64,
127 "GROUP_SIZE_M": 4,
128 "num_warps": 8,
129 - "num_stages": 0,
129 + "num_stages": 2,
130 "waves_per_eu": 0,
131 "matrix_instr_nonkdim": 16,
132 "kpack": 1
@@ -137,7 +137,7 @@
137 "BLOCK_SIZE_K": 128,
138 "GROUP_SIZE_M": 1,
139 "num_warps": 8,
140 - "num_stages": 0,
140 + "num_stages": 2,
141 "waves_per_eu": 0,
142 "matrix_instr_nonkdim": 16,
143 "kpack": 2
@@ -148,7 +148,7 @@
148 "BLOCK_SIZE_K": 64,
149 "GROUP_SIZE_M": 1,
150 "num_warps": 8,
151 - "num_stages": 0,
151 + "num_stages": 2,
152 "waves_per_eu": 0,
153 "matrix_instr_nonkdim": 16,
154 "kpack": 1
@@ -159,7 +159,7 @@
159 "BLOCK_SIZE_K": 64,
160 "GROUP_SIZE_M": 1,
161 "num_warps": 8,
162 - "num_stages": 0,
162 + "num_stages": 2,
163 "waves_per_eu": 0,
164 "matrix_instr_nonkdim": 16,
165 "kpack": 1
@@ -170,7 +170,7 @@
170 "BLOCK_SIZE_K": 64,
171 "GROUP_SIZE_M": 1,
172 "num_warps": 8,
173 - "num_stages": 0,
173 + "num_stages": 2,
174 "waves_per_eu": 0,
175 "matrix_instr_nonkdim": 16,
176 "kpack": 2
@@ -181,7 +181,7 @@
181 "BLOCK_SIZE_K": 64,
182 "GROUP_SIZE_M": 1,
183 "num_warps": 8,
184 - "num_stages": 0,
184 + "num_stages": 2,
185 "waves_per_eu": 0,
186 "matrix_instr_nonkdim": 16,
187 "kpack": 1
@@ -192,7 +192,7 @@
192 "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
194 "num_warps": 8,
195 - "num_stages": 0,
195 + "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 "kpack": 1
vllm/model_executor/layers/fused_moe/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json
+18 -18
@@ -5,7 +5,7 @@
5 "BLOCK_SIZE_K": 128,
6 "GROUP_SIZE_M": 1,
7 "num_warps": 2,
8 - "num_stages": 0,
8 + "num_stages": 2,
9 "waves_per_eu": 0,
10 "matrix_instr_nonkdim": 16,
11 "kpack": 1
@@ -16,7 +16,7 @@
16 "BLOCK_SIZE_K": 64,
17 "GROUP_SIZE_M": 1,
18 "num_warps": 2,
19 - "num_stages": 0,
19 + "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
22 "kpack": 2
@@ -27,7 +27,7 @@
27 "BLOCK_SIZE_K": 256,
28 "GROUP_SIZE_M": 1,
29 "num_warps": 2,
30 - "num_stages": 0,
30 + "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
33 "kpack": 2
@@ -38,7 +38,7 @@
38 "BLOCK_SIZE_K": 256,
39 "GROUP_SIZE_M": 1,
40 "num_warps": 2,
41 - "num_stages": 0,
41 + "num_stages": 2,
42 "waves_per_eu": 0,
43 "matrix_instr_nonkdim": 16,
44 "kpack": 2
@@ -49,7 +49,7 @@
49 "BLOCK_SIZE_K": 256,
50 "GROUP_SIZE_M": 1,
51 "num_warps": 2,
52 - "num_stages": 0,
52 + "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
55 "kpack": 2
@@ -60,7 +60,7 @@
60 "BLOCK_SIZE_K": 64,
61 "GROUP_SIZE_M": 1,
62 "num_warps": 4,
63 - "num_stages": 0,
63 + "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
66 "kpack": 1
@@ -71,7 +71,7 @@
71 "BLOCK_SIZE_K": 256,
72 "GROUP_SIZE_M": 4,
73 "num_warps": 2,
74 - "num_stages": 0,
74 + "num_stages": 2,
75 "waves_per_eu": 0,
76 "matrix_instr_nonkdim": 16,
77 "kpack": 2
@@ -82,7 +82,7 @@
82 "BLOCK_SIZE_K": 256,
83 "GROUP_SIZE_M": 1,
84 "num_warps": 2,
85 - "num_stages": 0,
85 + "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 "kpack": 2
@@ -93,7 +93,7 @@
93 "BLOCK_SIZE_K": 256,
94 "GROUP_SIZE_M": 4,
95 "num_warps": 4,
96 - "num_stages": 0,
96 + "num_stages": 2,
97 "waves_per_eu": 0,
98 "matrix_instr_nonkdim": 16,
99 "kpack": 2
@@ -104,7 +104,7 @@
104 "BLOCK_SIZE_K": 128,
105 "GROUP_SIZE_M": 4,
106 "num_warps": 4,
107 - "num_stages": 0,
107 + "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
110 "kpack": 1
@@ -115,7 +115,7 @@
115 "BLOCK_SIZE_K": 128,
116 "GROUP_SIZE_M": 4,
117 "num_warps": 8,
118 - "num_stages": 0,
118 + "num_stages": 2,
119 "waves_per_eu": 0,
120 "matrix_instr_nonkdim": 16,
121 "kpack": 1
@@ -126,7 +126,7 @@
126 "BLOCK_SIZE_K": 64,
127 "GROUP_SIZE_M": 4,
128 "num_warps": 8,
129 - "num_stages": 0,
129 + "num_stages": 2,
130 "waves_per_eu": 0,
131 "matrix_instr_nonkdim": 16,
132 "kpack": 1
@@ -137,7 +137,7 @@
137 "BLOCK_SIZE_K": 64,
138 "GROUP_SIZE_M": 1,
139 "num_warps": 8,
140 - "num_stages": 0,
140 + "num_stages": 2,
141 "waves_per_eu": 0,
142 "matrix_instr_nonkdim": 32,
143 "kpack": 2
@@ -148,7 +148,7 @@
148 "BLOCK_SIZE_K": 64,
149 "GROUP_SIZE_M": 1,
150 "num_warps": 8,
151 - "num_stages": 0,
151 + "num_stages": 2,
152 "waves_per_eu": 0,
153 "matrix_instr_nonkdim": 16,
154 "kpack": 1
@@ -159,7 +159,7 @@
159 "BLOCK_SIZE_K": 64,
160 "GROUP_SIZE_M": 1,
161 "num_warps": 8,
162 - "num_stages": 0,
162 + "num_stages": 2,
163 "waves_per_eu": 0,
164 "matrix_instr_nonkdim": 16,
165 "kpack": 2
@@ -170,7 +170,7 @@
170 "BLOCK_SIZE_K": 64,
171 "GROUP_SIZE_M": 1,
172 "num_warps": 8,
173 - "num_stages": 0,
173 + "num_stages": 2,
174 "waves_per_eu": 0,
175 "matrix_instr_nonkdim": 16,
176 "kpack": 1
@@ -181,7 +181,7 @@
181 "BLOCK_SIZE_K": 64,
182 "GROUP_SIZE_M": 1,
183 "num_warps": 8,
184 - "num_stages": 0,
184 + "num_stages": 2,
185 "waves_per_eu": 0,
186 "matrix_instr_nonkdim": 16,
187 "kpack": 2
@@ -192,7 +192,7 @@
192 "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
194 "num_warps": 8,
195 - "num_stages": 0,
195 + "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 "kpack": 1
vllm/model_executor/layers/fused_moe/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json
+18 -18
@@ -5,7 +5,7 @@
5 "BLOCK_SIZE_K": 256,
6 "GROUP_SIZE_M": 1,
7 "num_warps": 2,
8 - "num_stages": 0,
8 + "num_stages": 2,
9 "waves_per_eu": 0,
10 "matrix_instr_nonkdim": 16,
11 "kpack": 2
@@ -16,7 +16,7 @@
16 "BLOCK_SIZE_K": 32,
17 "GROUP_SIZE_M": 1,
18 "num_warps": 4,
19 - "num_stages": 0,
19 + "num_stages": 2,
20 "waves_per_eu": 0,
21 "matrix_instr_nonkdim": 16,
22 "kpack": 1
@@ -27,7 +27,7 @@
27 "BLOCK_SIZE_K": 128,
28 "GROUP_SIZE_M": 1,
29 "num_warps": 4,
30 - "num_stages": 0,
30 + "num_stages": 2,
31 "waves_per_eu": 0,
32 "matrix_instr_nonkdim": 16,
33 "kpack": 1
@@ -38,7 +38,7 @@
38 "BLOCK_SIZE_K": 256,
39 "GROUP_SIZE_M": 1,
40 "num_warps": 2,
41 - "num_stages": 0,
41 + "num_stages": 2,
42 "waves_per_eu": 0,
43 "matrix_instr_nonkdim": 16,
44 "kpack": 1
@@ -49,7 +49,7 @@
49 "BLOCK_SIZE_K": 256,
50 "GROUP_SIZE_M": 1,
51 "num_warps": 4,
52 - "num_stages": 0,
52 + "num_stages": 2,
53 "waves_per_eu": 0,
54 "matrix_instr_nonkdim": 16,
55 "kpack": 2
@@ -60,7 +60,7 @@
60 "BLOCK_SIZE_K": 128,
61 "GROUP_SIZE_M": 1,
62 "num_warps": 8,
63 - "num_stages": 0,
63 + "num_stages": 2,
64 "waves_per_eu": 0,
65 "matrix_instr_nonkdim": 16,
66 "kpack": 1
@@ -71,7 +71,7 @@
71 "BLOCK_SIZE_K": 128,
72 "GROUP_SIZE_M": 4,
73 "num_warps": 2,
74 - "num_stages": 0,
74 + "num_stages": 2,
75 "waves_per_eu": 0,
76 "matrix_instr_nonkdim": 16,
77 "kpack": 2
@@ -82,7 +82,7 @@
82 "BLOCK_SIZE_K": 128,
83 "GROUP_SIZE_M": 4,
84 "num_warps": 2,
85 - "num_stages": 0,
85 + "num_stages": 2,
86 "waves_per_eu": 0,
87 "matrix_instr_nonkdim": 16,
88 "kpack": 1
@@ -93,7 +93,7 @@
93 "BLOCK_SIZE_K": 128,
94 "GROUP_SIZE_M": 4,
95 "num_warps": 4,
96 - "num_stages": 0,
96 + "num_stages": 2,
97 "waves_per_eu": 0,
98 "matrix_instr_nonkdim": 16,
99 "kpack": 2
@@ -104,7 +104,7 @@
104 "BLOCK_SIZE_K": 128,
105 "GROUP_SIZE_M": 4,
106 "num_warps": 4,
107 - "num_stages": 0,
107 + "num_stages": 2,
108 "waves_per_eu": 0,
109 "matrix_instr_nonkdim": 16,
110 "kpack": 2
@@ -115,7 +115,7 @@
115 "BLOCK_SIZE_K": 64,
116 "GROUP_SIZE_M": 4,
117 "num_warps": 8,
118 - "num_stages": 0,
118 + "num_stages": 2,
119 "waves_per_eu": 0,
120 "matrix_instr_nonkdim": 16,
121 "kpack": 1
@@ -126,7 +126,7 @@
126 "BLOCK_SIZE_K": 64,
127 "GROUP_SIZE_M": 4,
128 "num_warps": 8,
129 - "num_stages": 0,
129 + "num_stages": 2,
130 "waves_per_eu": 0,
131 "matrix_instr_nonkdim": 32,
132 "kpack": 2
@@ -137,7 +137,7 @@
137 "BLOCK_SIZE_K": 64,
138 "GROUP_SIZE_M": 1,
139 "num_warps": 8,
140 - "num_stages": 0,
140 + "num_stages": 2,
141 "waves_per_eu": 0,
142 "matrix_instr_nonkdim": 16,
143 "kpack": 1
@@ -148,7 +148,7 @@
148 "BLOCK_SIZE_K": 64,
149 "GROUP_SIZE_M": 1,
150 "num_warps": 8,
151 - "num_stages": 0,
151 + "num_stages": 2,
152 "waves_per_eu": 0,
153 "matrix_instr_nonkdim": 16,
154 "kpack": 1
@@ -159,7 +159,7 @@
159 "BLOCK_SIZE_K": 64,
160 "GROUP_SIZE_M": 1,
161 "num_warps": 8,
162 - "num_stages": 0,
162 + "num_stages": 2,
163 "waves_per_eu": 0,
164 "matrix_instr_nonkdim": 16,
165 "kpack": 2
@@ -170,7 +170,7 @@
170 "BLOCK_SIZE_K": 64,
171 "GROUP_SIZE_M": 1,
172 "num_warps": 8,
173 - "num_stages": 0,
173 + "num_stages": 2,
174 "waves_per_eu": 0,
175 "matrix_instr_nonkdim": 16,
176 "kpack": 1
@@ -181,7 +181,7 @@
181 "BLOCK_SIZE_K": 64,
182 "GROUP_SIZE_M": 1,
183 "num_warps": 8,
184 - "num_stages": 0,
184 + "num_stages": 2,
185 "waves_per_eu": 0,
186 "matrix_instr_nonkdim": 16,
187 "kpack": 2
@@ -192,7 +192,7 @@
192 "BLOCK_SIZE_K": 64,
193 "GROUP_SIZE_M": 1,
194 "num_warps": 8,
195 - "num_stages": 0,
195 + "num_stages": 2,
196 "waves_per_eu": 0,
197 "matrix_instr_nonkdim": 16,
198 "kpack": 1