@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd34029e91
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

Refactor TPU requirements file and pin build dependencies (#10010)

Signed-off-by: Richard Liu <ricliu@google.com>

Richard Liu committed Nov 5, 2024 at 08:48 UTC cd34029e91ad2d38a58d190331a65f9096c0b157
3 files changed +23 -61
Dockerfile.tpu
-7
@@ -9,12 +9,6 @@ RUN apt-get update && apt-get install -y \
9 git \
10 ffmpeg libsm6 libxext6 libgl1
11
12 -# Install the TPU and Pallas dependencies.
13 -RUN --mount=type=cache,target=/root/.cache/pip \
14 - python3 -m pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
15 -RUN --mount=type=cache,target=/root/.cache/pip \
16 - python3 -m pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
17 -
12 # Build vLLM.
13 COPY . .
14 ARG GIT_REPO_CHECK=0
@@ -25,7 +19,6 @@ ENV VLLM_TARGET_DEVICE="tpu"
19 RUN --mount=type=cache,target=/root/.cache/pip \
20 --mount=type=bind,source=.git,target=.git \
21 python3 -m pip install \
28 - 'cmake>=3.26' ninja packaging 'setuptools-scm>=8' wheel jinja2 \
22 -r requirements-tpu.txt
23 RUN python3 setup.py develop
24
docs/source/getting_started/tpu-installation.rst
+5 -52
@@ -119,27 +119,19 @@ Uninstall the existing `torch` and `torch_xla` packages:
119
120 pip uninstall torch torch-xla -y
121
122 -Install `torch` and `torch_xla`
122 +Install build dependencies:
123
124 .. code-block:: bash
125
126 - pip install --pre torch==2.6.0.dev20241028+cpu torchvision==0.20.0.dev20241028+cpu --index-url https://download.pytorch.org/whl/nightly/cpu
127 - pip install 'torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev-cp310-cp310-linux_x86_64.whl' -f https://storage.googleapis.com/libtpu-releases/index.html
126 + pip install -r requirements-tpu.txt
127 + sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
128
129 -Install JAX and Pallas:
129 +Run the setup script:
130
131 .. code-block:: bash
132
133 - pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
134 - pip install jaxlib==0.4.32.dev20240829 jax==0.4.32.dev20240829 -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
135 -
136 -Install other build dependencies:
133 + VLLM_TARGET_DEVICE="tpu" python setup.py develop
134
138 -.. code-block:: bash
139 -
140 - pip install -r requirements-tpu.txt
141 - VLLM_TARGET_DEVICE="tpu" python setup.py develop
142 - sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
135
136 Provision Cloud TPUs with GKE
137 -----------------------------
@@ -168,45 +160,6 @@ Run the Docker image with the following command:
160 $ # Make sure to add `--privileged --net host --shm-size=16G`.
161 $ docker run --privileged --net host --shm-size=16G -it vllm-tpu
162
171 -
172 -.. _build_from_source_tpu:
173 -
174 -Build from source
175 ------------------
176 -
177 -You can also build and install the TPU backend from source.
178 -
179 -First, install the dependencies:
180 -
181 -.. code-block:: console
182 -
183 - $ # (Recommended) Create a new conda environment.
184 - $ conda create -n myenv python=3.10 -y
185 - $ conda activate myenv
186 -
187 - $ # Clean up the existing torch and torch-xla packages.
188 - $ pip uninstall torch torch-xla -y
189 -
190 - $ # Install PyTorch and PyTorch XLA.
191 - $ export DATE="20241017"
192 - $ export TORCH_VERSION="2.6.0"
193 - $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
194 - $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
195 -
196 - $ # Install JAX and Pallas.
197 - $ pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
198 - $ pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
199 -
200 - $ # Install other build dependencies.
201 - $ pip install -r requirements-tpu.txt
202 -
203 -
204 -Next, build vLLM from source. This will only take a few seconds:
205 -
206 -.. code-block:: console
207 -
208 - $ VLLM_TARGET_DEVICE="tpu" python setup.py develop
209 -
163 .. note::
164
165 Since TPU relies on XLA which requires static shapes, vLLM bucketizes the possible input shapes and compiles an XLA graph for each different shape.
requirements-tpu.txt
+18 -2
@@ -2,6 +2,22 @@
2 -r requirements-common.txt
3
4 # Dependencies for TPU
5 -# Currently, the TPU backend uses a nightly version of PyTorch XLA.
6 -# You can install the dependencies in Dockerfile.tpu.
5 +cmake>=3.26
6 +ninja
7 +packaging
8 +setuptools-scm>=8
9 +wheel
10 +jinja2
11 ray[default]
12 +
13 +# Install torch_xla
14 +--pre
15 +--extra-index-url https://download.pytorch.org/whl/nightly/cpu
16 +--find-links https://storage.googleapis.com/libtpu-releases/index.html
17 +--find-links https://storage.googleapis.com/jax-releases/jax_nightly_releases.html
18 +--find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
19 +torch==2.6.0.dev20241028+cpu
20 +torchvision==0.20.0.dev20241028+cpu
21 +torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241028-cp310-cp310-linux_x86_64.whl
22 +jaxlib==0.4.32.dev20240829
23 +jax==0.4.32.dev20240829