A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[CI/Build] use setuptools-scm to set __version__ (#4738)
Co-authored-by: youkaichao <youkaichao@126.com>
Daniele committed
Sep 23, 2024 at 18:44 UTC
ee5f34b1c2c71b2d56054a5ca23fe1c50c1458bb
16 files changed
+94
-95
.gitignore
+3
@@ -196,5 +196,8 @@ _build/
196
*_hip*
197
hip_compat.h
198
199
+# version file generated by setuptools-scm
200
+/vllm/_version.py
201
+
202
# Benchmark dataset
203
benchmarks/*.json
Dockerfile
+2
-3
@@ -79,15 +79,13 @@ ENV MAX_JOBS=${max_jobs}
79
ARG nvcc_threads=8
80
ENV NVCC_THREADS=$nvcc_threads
81
82
-ARG buildkite_commit
83
-ENV BUILDKITE_COMMIT=${buildkite_commit}
84
-
82
ARG USE_SCCACHE
83
ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
84
ARG SCCACHE_REGION_NAME=us-west-2
85
ARG SCCACHE_S3_NO_CREDENTIALS=0
86
# if USE_SCCACHE is set, use sccache to speed up compilation
87
RUN --mount=type=cache,target=/root/.cache/pip \
88
+ --mount=type=bind,source=.git,target=.git \
89
if [ "$USE_SCCACHE" = "1" ]; then \
90
echo "Installing sccache..." \
91
&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \
@@ -107,6 +105,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
105
ENV CCACHE_DIR=/root/.cache/ccache
106
RUN --mount=type=cache,target=/root/.cache/ccache \
107
--mount=type=cache,target=/root/.cache/pip \
108
+ --mount=type=bind,source=.git,target=.git \
109
if [ "$USE_SCCACHE" != "1" ]; then \
110
python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38; \
111
fi
Dockerfile.cpu
+3
-1
@@ -62,8 +62,10 @@ ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
62
63
RUN --mount=type=cache,target=/root/.cache/pip \
64
--mount=type=cache,target=/root/.cache/ccache \
65
+ --mount=type=bind,source=.git,target=.git \
66
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel && \
66
- pip install dist/*.whl
67
+ pip install dist/*.whl && \
68
+ rm -rf dist
69
70
WORKDIR /workspace/
71
Dockerfile.neuron
+13
-10
@@ -6,9 +6,12 @@ FROM $BASE_IMAGE
6
RUN echo "Base image is $BASE_IMAGE"
7
8
# Install some basic utilities
9
-RUN apt-get update \
10
- && apt-get install python3 python3-pip -y \
11
- && apt-get install -y ffmpeg libsm6 libxext6 libgl1
9
+RUN apt-get update && \
10
+ apt-get install -y \
11
+ git \
12
+ python3 \
13
+ python3-pip \
14
+ ffmpeg libsm6 libxext6 libgl1
15
16
### Mount Point ###
17
# When launching the container, mount the code directory to /app
@@ -22,17 +25,17 @@ RUN python3 -m pip install sentencepiece transformers==4.36.2 -U
25
RUN python3 -m pip install transformers-neuronx --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
26
RUN python3 -m pip install --pre neuronx-cc==2.15.* --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
27
25
-COPY ./vllm /app/vllm/vllm
26
-COPY ./setup.py /app/vllm/setup.py
27
-COPY ./requirements-common.txt /app/vllm/requirements-common.txt
28
-COPY ./requirements-neuron.txt /app/vllm/requirements-neuron.txt
28
+COPY . /app/vllm
29
30
RUN cd /app/vllm \
31
- && python3 -m pip install -U -r requirements-neuron.txt
31
+ && python3 -m pip install -U \
32
+ cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
33
+ -r requirements-neuron.txt
34
35
ENV VLLM_TARGET_DEVICE neuron
34
-RUN cd /app/vllm \
35
- && pip install -e . \
36
+RUN --mount=type=bind,source=.git,target=.git \
37
+ cd /app/vllm \
38
+ && pip install --no-build-isolation -v -e . \
39
&& cd ..
40
41
CMD ["/bin/bash"]
Dockerfile.openvino
+3
-2
@@ -4,8 +4,9 @@
4
FROM ubuntu:22.04 AS dev
5
6
RUN apt-get update -y && \
7
- apt-get install -y python3-pip git && \
8
- apt-get install -y ffmpeg libsm6 libxext6 libgl1
7
+ apt-get install -y \
8
+ git python3-pip \
9
+ ffmpeg libsm6 libxext6 libgl1
10
WORKDIR /workspace
11
12
# copy requirements
Dockerfile.ppc64le
+9
-3
@@ -16,9 +16,15 @@ COPY ./ /workspace/vllm
16
WORKDIR /workspace/vllm
17
18
# These packages will be in rocketce eventually
19
-RUN pip install -v cmake xformers torch==2.3.1 uvloop==0.20.0 -r requirements-cpu.txt --prefer-binary --extra-index-url https://repo.fury.io/mgiessing
20
-
21
-RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
19
+RUN --mount=type=cache,target=/root/.cache/pip \
20
+ pip install -v --prefer-binary --extra-index-url https://repo.fury.io/mgiessing \
21
+ cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
22
+ torch==2.3.1 \
23
+ -r requirements-cpu.txt \
24
+ xformers uvloop==0.20.0
25
+
26
+RUN --mount=type=bind,source=.git,target=.git \
27
+ VLLM_TARGET_DEVICE=cpu python3 setup.py install
28
29
WORKDIR /workspace/
30
Dockerfile.rocm
+6
-3
@@ -51,13 +51,15 @@ RUN python3 -m pip install --upgrade pip
51
# TODO: implement sccache support across components
52
RUN apt-get purge -y sccache; python3 -m pip uninstall -y sccache; rm -f "$(which sccache)"
53
# Install torch == 2.5.0 on ROCm
54
-RUN case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
54
+RUN --mount=type=cache,target=/root/.cache/pip \
55
+ case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
56
*"rocm-6.1"*) \
57
python3 -m pip uninstall -y torch torchvision \
57
- && python3 -m pip install --no-cache-dir --pre \
58
+ && python3 -m pip install --pre \
59
torch==2.5.0.dev20240726 \
60
+ cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
61
torchvision==0.20.0.dev20240726 \
60
- --index-url https://download.pytorch.org/whl/nightly/rocm6.1;; \
62
+ --extra-index-url https://download.pytorch.org/whl/nightly/rocm6.1 ;; \
63
*) ;; esac
64
65
ENV LLVM_SYMBOLIZER_PATH=/opt/rocm/llvm/bin/llvm-symbolizer
@@ -138,6 +140,7 @@ ENV RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1
140
ENV TOKENIZERS_PARALLELISM=false
141
142
RUN --mount=type=cache,target=${CCACHE_DIR} \
143
+ --mount=type=bind,source=.git,target=.git \
144
--mount=type=cache,target=/root/.cache/pip \
145
python3 -m pip install -Ur requirements-rocm.txt \
146
&& case "$(ls /opt | grep -Po 'rocm-[0-9]\.[0-9]')" in \
Dockerfile.tpu
+13
-4
@@ -5,16 +5,25 @@ FROM $BASE_IMAGE
5
WORKDIR /workspace
6
7
# Install some basic utilities
8
-RUN apt-get update && apt-get install -y ffmpeg libsm6 libxext6 libgl1
8
+RUN apt-get update && apt-get install -y \
9
+ git \
10
+ ffmpeg libsm6 libxext6 libgl1
11
12
# Install the TPU and Pallas dependencies.
11
-RUN python3 -m pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
12
-RUN python3 -m pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
13
+RUN --mount=type=cache,target=/root/.cache/pip \
14
+ python3 -m pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
15
+RUN --mount=type=cache,target=/root/.cache/pip \
16
+ python3 -m pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
17
18
# Build vLLM.
19
COPY . /workspace/vllm
20
ENV VLLM_TARGET_DEVICE="tpu"
17
-RUN cd /workspace/vllm && python3 -m pip install -r requirements-tpu.txt
21
+RUN --mount=type=cache,target=/root/.cache/pip \
22
+ --mount=type=bind,source=.git,target=.git \
23
+ cd /workspace/vllm && \
24
+ python3 -m pip install \
25
+ cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
26
+ -r requirements-tpu.txt
27
RUN cd /workspace/vllm && python3 setup.py develop
28
29
CMD ["/bin/bash"]
Dockerfile.xpu
+9
-4
@@ -7,15 +7,20 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
7
echo "deb [arch=amd64,i386 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy arc" | tee /etc/apt/sources.list.d/intel.gpu.jammy.list && \
8
chmod 644 /usr/share/keyrings/intel-graphics.gpg
9
10
-RUN apt-get update -y \
11
-&& apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip ffmpeg libsm6 libxext6 libgl1
10
+RUN apt-get update -y && \
11
+ apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip ffmpeg libsm6 libxext6 libgl1
12
13
COPY ./ /workspace/vllm
14
15
WORKDIR /workspace/vllm
16
17
-RUN pip install -v -r requirements-xpu.txt --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
17
+RUN --mount=type=cache,target=/root/.cache/pip \
18
+ pip install -v --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ \
19
+ cmake>=3.26 ninja packaging setuptools-scm>=8 wheel jinja2 \
20
+ -r requirements-xpu.txt
21
19
-RUN VLLM_TARGET_DEVICE=xpu python3 setup.py install
22
+RUN --mount=type=cache,target=/root/.cache/pip \
23
+ --mount=type=bind,source=.git,target=.git \
24
+ VLLM_TARGET_DEVICE=xpu python3 setup.py install
25
26
CMD ["/bin/bash"]
docs/source/getting_started/cpu-installation.rst
+1
-1
@@ -56,7 +56,7 @@ Build from source
56
.. code-block:: console
57
58
$ pip install --upgrade pip
59
- $ pip install wheel packaging ninja "setuptools>=49.4.0" numpy
59
+ $ pip install cmake>=3.26 wheel packaging ninja "setuptools-scm>=8" numpy
60
$ pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
61
62
- Third, build and install oneDNN library from source:
pyproject.toml
+9
-1
@@ -4,7 +4,8 @@ requires = [
4
"cmake>=3.26",
5
"ninja",
6
"packaging",
7
- "setuptools >= 49.4.0",
7
+ "setuptools>=61",
8
+ "setuptools-scm>=8.0",
9
"torch == 2.4.0",
10
"wheel",
11
"jinja2",
@@ -19,6 +20,10 @@ exclude = [
20
"examples/fp8/quantizer/quantize.py"
21
]
22
23
+[tool.ruff.lint.per-file-ignores]
24
+"vllm/version.py" = ["F401"]
25
+"vllm/_version.py" = ["ALL"]
26
+
27
[tool.ruff.lint]
28
select = [
29
# pycodestyle
@@ -46,6 +51,9 @@ ignore = [
51
"UP032",
52
]
53
54
+[tool.setuptools_scm]
55
+version_file = "vllm/_version.py"
56
+
57
[tool.mypy]
58
python_version = "3.8"
59
requirements-build.txt
+2
-1
@@ -2,7 +2,8 @@
2
cmake>=3.26
3
ninja
4
packaging
5
-setuptools>=49.4.0
5
+setuptools>=61
6
+setuptools-scm>=8
7
torch==2.4.0
8
wheel
9
jinja2
setup.py
+10
-50
@@ -5,7 +5,6 @@ import os
5
import re
6
import subprocess
7
import sys
8
-import warnings
8
from pathlib import Path
9
from shutil import which
10
from typing import Dict, List
@@ -14,6 +13,7 @@ import torch
13
from packaging.version import Version, parse
14
from setuptools import Extension, find_packages, setup
15
from setuptools.command.build_ext import build_ext
16
+from setuptools_scm import get_version
17
from torch.utils.cpp_extension import CUDA_HOME
18
19
@@ -28,34 +28,6 @@ def load_module_from_path(module_name, path):
28
ROOT_DIR = os.path.dirname(__file__)
29
logger = logging.getLogger(__name__)
30
31
-
32
-def embed_commit_hash():
33
- try:
34
- if "BUILDKITE_COMMIT" in os.environ:
35
- # ci build
36
- commit_id = os.environ["BUILDKITE_COMMIT"]
37
- else:
38
- commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"],
39
- encoding="utf-8").strip()
40
-
41
- commit_contents = f'__commit__ = "{commit_id}"\n'
42
-
43
- version_file = os.path.join(ROOT_DIR, "vllm", "commit_id.py")
44
- with open(version_file, "w", encoding="utf-8") as f:
45
- f.write(commit_contents)
46
-
47
- except subprocess.CalledProcessError as e:
48
- warnings.warn(f"Failed to get commit hash:\n{e}",
49
- RuntimeWarning,
50
- stacklevel=2)
51
- except Exception as e:
52
- warnings.warn(f"Failed to embed commit hash:\n{e}",
53
- RuntimeWarning,
54
- stacklevel=2)
55
-
56
-
57
-embed_commit_hash()
58
-
31
# cannot import envs directly because it depends on vllm,
32
# which is not installed yet
33
envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))
@@ -381,21 +353,9 @@ def get_path(*filepath) -> str:
353
return os.path.join(ROOT_DIR, *filepath)
354
355
384
-def find_version(filepath: str) -> str:
385
- """Extract version information from the given filepath.
386
-
387
- Adapted from https://github.com/ray-project/ray/blob/0b190ee1160eeca9796bc091e07eaebf4c85b511/python/setup.py
388
- """
389
- with open(filepath) as fp:
390
- version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
391
- fp.read(), re.M)
392
- if version_match:
393
- return version_match.group(1)
394
- raise RuntimeError("Unable to find version string.")
395
-
396
-
356
def get_vllm_version() -> str:
398
- version = find_version(get_path("vllm", "version.py"))
357
+ version = get_version()
358
+ sep = "+" if "+" not in version else "." # dev versions might contain +
359
360
if _no_device():
361
if envs.VLLM_TARGET_DEVICE == "empty":
@@ -406,27 +366,27 @@ def get_vllm_version() -> str:
366
cuda_version_str = cuda_version.replace(".", "")[:3]
367
# skip this for source tarball, required for pypi
368
if "sdist" not in sys.argv:
409
- version += f"+cu{cuda_version_str}"
369
+ version += f"{sep}cu{cuda_version_str}"
370
elif _is_hip():
371
# Get the HIP version
372
hipcc_version = get_hipcc_rocm_version()
373
if hipcc_version != MAIN_CUDA_VERSION:
374
rocm_version_str = hipcc_version.replace(".", "")[:3]
415
- version += f"+rocm{rocm_version_str}"
375
+ version += f"{sep}rocm{rocm_version_str}"
376
elif _is_neuron():
377
# Get the Neuron version
378
neuron_version = str(get_neuronxcc_version())
379
if neuron_version != MAIN_CUDA_VERSION:
380
neuron_version_str = neuron_version.replace(".", "")[:3]
421
- version += f"+neuron{neuron_version_str}"
381
+ version += f"{sep}neuron{neuron_version_str}"
382
elif _is_openvino():
423
- version += "+openvino"
383
+ version += f"{sep}openvino"
384
elif _is_tpu():
425
- version += "+tpu"
385
+ version += f"{sep}tpu"
386
elif _is_cpu():
427
- version += "+cpu"
387
+ version += f"{sep}cpu"
388
elif _is_xpu():
429
- version += "+xpu"
389
+ version += f"{sep}xpu"
390
else:
391
raise RuntimeError("Unknown runtime environment")
392
tests/test_embedded_commit.py
+4
-3
@@ -2,6 +2,7 @@ import vllm
2
3
4
def test_embedded_commit_defined():
5
- assert vllm.__commit__ != "COMMIT_HASH_PLACEHOLDER"
6
- # 7 characters is the length of a short commit hash
7
- assert len(vllm.__commit__) >= 7
5
+ assert hasattr(vllm, "__version__")
6
+ assert hasattr(vllm, "__version_tuple__")
7
+ assert vllm.__version__ != "dev"
8
+ assert vllm.__version_tuple__ != (0, 0, "dev")
vllm/__init__.py
+2
-2
@@ -12,11 +12,11 @@ from vllm.outputs import (CompletionOutput, EmbeddingOutput,
12
from vllm.pooling_params import PoolingParams
13
from vllm.sampling_params import SamplingParams
14
15
-from .version import __commit__, __version__
15
+from .version import __version__, __version_tuple__
16
17
__all__ = [
18
- "__commit__",
18
"__version__",
19
+ "__version_tuple__",
20
"LLM",
21
"ModelRegistry",
22
"PromptType",
vllm/version.py
+5
-7
@@ -1,13 +1,11 @@
1
-import warnings
2
-
1
try:
4
- import vllm.commit_id
5
-
6
- __commit__ = vllm.commit_id.__commit__
2
+ from ._version import __version__, __version_tuple__
3
except Exception as e:
4
+ import warnings
5
+
6
warnings.warn(f"Failed to read commit hash:\n{e}",
7
RuntimeWarning,
8
stacklevel=2)
11
- __commit__ = "COMMIT_HASH_PLACEHOLDER"
9
13
-__version__ = "0.6.1.post2"
10
+ __version__ = "dev"
11
+ __version_tuple__ = (0, 0, __version__)