A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add Dockerfile (#1350)
Stephen Krider committed
Oct 31, 2023 at 12:36 UTC
9cabcb76453e7d8384d1858663f1841cef174c2c
5 files changed
+101
-2
Dockerfile
new
+72
@@ -0,0 +1,72 @@
1
+FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 AS dev
2
+
3
+RUN apt-get update -y \
4
+ && apt-get install -y python3-pip
5
+
6
+WORKDIR /workspace
7
+
8
+# install build and runtime dependencies
9
+COPY requirements.txt requirements.txt
10
+RUN --mount=type=cache,target=/root/.cache/pip \
11
+ pip install -r requirements.txt
12
+
13
+# install development dependencies
14
+COPY requirements-dev.txt requirements-dev.txt
15
+RUN --mount=type=cache,target=/root/.cache/pip \
16
+ pip install -r requirements-dev.txt
17
+
18
+# image to build pytorch extensions
19
+FROM dev AS build
20
+
21
+# copy input files
22
+COPY csrc csrc
23
+COPY setup.py setup.py
24
+COPY requirements.txt requirements.txt
25
+COPY pyproject.toml pyproject.toml
26
+COPY vllm/__init__.py vllm/__init__.py
27
+
28
+# max jobs used by Ninja to build extensions
29
+ENV MAX_JOBS=$max_jobs
30
+RUN python3 setup.py build_ext --inplace
31
+
32
+# image to run unit testing suite
33
+FROM dev AS test
34
+
35
+# copy pytorch extensions separately to avoid having to rebuild
36
+# when python code changes
37
+COPY --from=build /workspace/vllm/*.so /workspace/vllm/
38
+COPY tests tests
39
+COPY vllm vllm
40
+
41
+ENTRYPOINT ["python3", "-m", "pytest", "tests"]
42
+
43
+# use CUDA base as CUDA runtime dependencies are already installed via pip
44
+FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS vllm-base
45
+
46
+# libnccl required for ray
47
+RUN apt-get update -y \
48
+ && apt-get install -y python3-pip
49
+
50
+WORKDIR /workspace
51
+COPY requirements.txt requirements.txt
52
+RUN --mount=type=cache,target=/root/.cache/pip \
53
+ pip install -r requirements.txt
54
+
55
+FROM vllm-base AS vllm
56
+COPY --from=build /workspace/vllm/*.so /workspace/vllm/
57
+COPY vllm vllm
58
+
59
+EXPOSE 8000
60
+ENTRYPOINT ["python3", "-m", "vllm.entrypoints.api_server"]
61
+
62
+# openai api server alternative
63
+FROM vllm-base AS vllm-openai
64
+# install additional dependencies for openai api server
65
+RUN --mount=type=cache,target=/root/.cache/pip \
66
+ pip install accelerate fschat
67
+
68
+COPY --from=build /workspace/vllm/*.so /workspace/vllm/
69
+COPY vllm vllm
70
+
71
+ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
72
+
docs/source/index.rst
+1
@@ -65,6 +65,7 @@ Documentation
65
serving/distributed_serving
66
serving/run_on_sky
67
serving/deploying_with_triton
68
+ serving/deploying_with_docker
69
70
.. toctree::
71
:maxdepth: 1
docs/source/serving/deploying_with_docker.rst
new
+21
@@ -0,0 +1,21 @@
1
+.. _deploying_with_docker:
2
+
3
+Deploying with Docker
4
+============================
5
+
6
+You can build and run vLLM from source via the provided dockerfile. To build vLLM:
7
+
8
+.. code-block:: console
9
+
10
+ $ DOCKER_BUILDKIT=1 docker build . --target vllm --tag vllm --build-arg max_jobs=8
11
+
12
+To run vLLM:
13
+
14
+.. code-block:: console
15
+
16
+ $ docker run --runtime nvidia --gpus all \
17
+ -v ~/.cache/huggingface:/root/.cache/huggingface \
18
+ -p 8000:8000 \
19
+ --env "HUGGING_FACE_HUB_TOKEN=<secret>" \
20
+ vllm <args...>
21
+
requirements-dev.txt
+1
@@ -12,3 +12,4 @@ types-setuptools
12
pytest
13
pytest-forked
14
pytest-asyncio
15
+
setup.py
+6
-2
@@ -239,8 +239,12 @@ def find_version(filepath: str):
239
240
241
def read_readme() -> str:
242
- """Read the README file."""
243
- return io.open(get_path("README.md"), "r", encoding="utf-8").read()
242
+ """Read the README file if present."""
243
+ p = get_path("README.md")
244
+ if os.path.isfile(p):
245
+ return io.open(get_path("README.md"), "r", encoding="utf-8").read()
246
+ else:
247
+ return ""
248
249
250
def get_requirements() -> List[str]: