A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
Refactor TPU requirements file and pin build dependencies (#10010)
Signed-off-by: Richard Liu <ricliu@google.com>
Richard Liu committed
Nov 5, 2024 at 08:48 UTC
cd34029e91ad2d38a58d190331a65f9096c0b157
3 files changed
+23
-61
Dockerfile.tpu
-7
@@ -9,12 +9,6 @@ RUN apt-get update && apt-get install -y \
9
git \
10
ffmpeg libsm6 libxext6 libgl1
11
12
-# Install the TPU and Pallas dependencies.
13
-RUN --mount=type=cache,target=/root/.cache/pip \
14
- python3 -m pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
15
-RUN --mount=type=cache,target=/root/.cache/pip \
16
- python3 -m pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
17
-
12
# Build vLLM.
13
COPY . .
14
ARG GIT_REPO_CHECK=0
@@ -25,7 +19,6 @@ ENV VLLM_TARGET_DEVICE="tpu"
19
RUN --mount=type=cache,target=/root/.cache/pip \
20
--mount=type=bind,source=.git,target=.git \
21
python3 -m pip install \
28
- 'cmake>=3.26' ninja packaging 'setuptools-scm>=8' wheel jinja2 \
22
-r requirements-tpu.txt
23
RUN python3 setup.py develop
24
docs/source/getting_started/tpu-installation.rst
+5
-52
@@ -119,27 +119,19 @@ Uninstall the existing `torch` and `torch_xla` packages:
119
120
pip uninstall torch torch-xla -y
121
122
-Install `torch` and `torch_xla`
122
+Install build dependencies:
123
124
.. code-block:: bash
125
126
- pip install --pre torch==2.6.0.dev20241028+cpu torchvision==0.20.0.dev20241028+cpu --index-url https://download.pytorch.org/whl/nightly/cpu
127
- pip install 'torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev-cp310-cp310-linux_x86_64.whl' -f https://storage.googleapis.com/libtpu-releases/index.html
126
+ pip install -r requirements-tpu.txt
127
+ sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
128
129
-Install JAX and Pallas:
129
+Run the setup script:
130
131
.. code-block:: bash
132
133
- pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
134
- pip install jaxlib==0.4.32.dev20240829 jax==0.4.32.dev20240829 -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
135
-
136
-Install other build dependencies:
133
+ VLLM_TARGET_DEVICE="tpu" python setup.py develop
134
138
-.. code-block:: bash
139
-
140
- pip install -r requirements-tpu.txt
141
- VLLM_TARGET_DEVICE="tpu" python setup.py develop
142
- sudo apt-get install libopenblas-base libopenmpi-dev libomp-dev
135
136
Provision Cloud TPUs with GKE
137
-----------------------------
@@ -168,45 +160,6 @@ Run the Docker image with the following command:
160
$ # Make sure to add `--privileged --net host --shm-size=16G`.
161
$ docker run --privileged --net host --shm-size=16G -it vllm-tpu
162
171
-
172
-.. _build_from_source_tpu:
173
-
174
-Build from source
175
------------------
176
-
177
-You can also build and install the TPU backend from source.
178
-
179
-First, install the dependencies:
180
-
181
-.. code-block:: console
182
-
183
- $ # (Recommended) Create a new conda environment.
184
- $ conda create -n myenv python=3.10 -y
185
- $ conda activate myenv
186
-
187
- $ # Clean up the existing torch and torch-xla packages.
188
- $ pip uninstall torch torch-xla -y
189
-
190
- $ # Install PyTorch and PyTorch XLA.
191
- $ export DATE="20241017"
192
- $ export TORCH_VERSION="2.6.0"
193
- $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
194
- $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-${TORCH_VERSION}.dev${DATE}-cp310-cp310-linux_x86_64.whl
195
-
196
- $ # Install JAX and Pallas.
197
- $ pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
198
- $ pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
199
-
200
- $ # Install other build dependencies.
201
- $ pip install -r requirements-tpu.txt
202
-
203
-
204
-Next, build vLLM from source. This will only take a few seconds:
205
-
206
-.. code-block:: console
207
-
208
- $ VLLM_TARGET_DEVICE="tpu" python setup.py develop
209
-
163
.. note::
164
165
Since TPU relies on XLA which requires static shapes, vLLM bucketizes the possible input shapes and compiles an XLA graph for each different shape.
requirements-tpu.txt
+18
-2
@@ -2,6 +2,22 @@
2
-r requirements-common.txt
3
4
# Dependencies for TPU
5
-# Currently, the TPU backend uses a nightly version of PyTorch XLA.
6
-# You can install the dependencies in Dockerfile.tpu.
5
+cmake>=3.26
6
+ninja
7
+packaging
8
+setuptools-scm>=8
9
+wheel
10
+jinja2
11
ray[default]
12
+
13
+# Install torch_xla
14
+--pre
15
+--extra-index-url https://download.pytorch.org/whl/nightly/cpu
16
+--find-links https://storage.googleapis.com/libtpu-releases/index.html
17
+--find-links https://storage.googleapis.com/jax-releases/jax_nightly_releases.html
18
+--find-links https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
19
+torch==2.6.0.dev20241028+cpu
20
+torchvision==0.20.0.dev20241028+cpu
21
+torch_xla[tpu] @ https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-2.6.0.dev20241028-cp310-cp310-linux_x86_64.whl
22
+jaxlib==0.4.32.dev20240829
23
+jax==0.4.32.dev20240829