@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9cabcb7645
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add Dockerfile (#1350)

Stephen Krider committed Oct 31, 2023 at 12:36 UTC 9cabcb76453e7d8384d1858663f1841cef174c2c
5 files changed +101 -2
Dockerfile new
+72
@@ -0,0 +1,72 @@
1 +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 AS dev
2 +
3 +RUN apt-get update -y \
4 + && apt-get install -y python3-pip
5 +
6 +WORKDIR /workspace
7 +
8 +# install build and runtime dependencies
9 +COPY requirements.txt requirements.txt
10 +RUN --mount=type=cache,target=/root/.cache/pip \
11 + pip install -r requirements.txt
12 +
13 +# install development dependencies
14 +COPY requirements-dev.txt requirements-dev.txt
15 +RUN --mount=type=cache,target=/root/.cache/pip \
16 + pip install -r requirements-dev.txt
17 +
18 +# image to build pytorch extensions
19 +FROM dev AS build
20 +
21 +# copy input files
22 +COPY csrc csrc
23 +COPY setup.py setup.py
24 +COPY requirements.txt requirements.txt
25 +COPY pyproject.toml pyproject.toml
26 +COPY vllm/__init__.py vllm/__init__.py
27 +
28 +# max jobs used by Ninja to build extensions
29 +ENV MAX_JOBS=$max_jobs
30 +RUN python3 setup.py build_ext --inplace
31 +
32 +# image to run unit testing suite
33 +FROM dev AS test
34 +
35 +# copy pytorch extensions separately to avoid having to rebuild
36 +# when python code changes
37 +COPY --from=build /workspace/vllm/*.so /workspace/vllm/
38 +COPY tests tests
39 +COPY vllm vllm
40 +
41 +ENTRYPOINT ["python3", "-m", "pytest", "tests"]
42 +
43 +# use CUDA base as CUDA runtime dependencies are already installed via pip
44 +FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS vllm-base
45 +
46 +# libnccl required for ray
47 +RUN apt-get update -y \
48 + && apt-get install -y python3-pip
49 +
50 +WORKDIR /workspace
51 +COPY requirements.txt requirements.txt
52 +RUN --mount=type=cache,target=/root/.cache/pip \
53 + pip install -r requirements.txt
54 +
55 +FROM vllm-base AS vllm
56 +COPY --from=build /workspace/vllm/*.so /workspace/vllm/
57 +COPY vllm vllm
58 +
59 +EXPOSE 8000
60 +ENTRYPOINT ["python3", "-m", "vllm.entrypoints.api_server"]
61 +
62 +# openai api server alternative
63 +FROM vllm-base AS vllm-openai
64 +# install additional dependencies for openai api server
65 +RUN --mount=type=cache,target=/root/.cache/pip \
66 + pip install accelerate fschat
67 +
68 +COPY --from=build /workspace/vllm/*.so /workspace/vllm/
69 +COPY vllm vllm
70 +
71 +ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
72 +
docs/source/index.rst
+1
@@ -65,6 +65,7 @@ Documentation
65 serving/distributed_serving
66 serving/run_on_sky
67 serving/deploying_with_triton
68 + serving/deploying_with_docker
69
70 .. toctree::
71 :maxdepth: 1
docs/source/serving/deploying_with_docker.rst new
+21
@@ -0,0 +1,21 @@
1 +.. _deploying_with_docker:
2 +
3 +Deploying with Docker
4 +============================
5 +
6 +You can build and run vLLM from source via the provided dockerfile. To build vLLM:
7 +
8 +.. code-block:: console
9 +
10 + $ DOCKER_BUILDKIT=1 docker build . --target vllm --tag vllm --build-arg max_jobs=8
11 +
12 +To run vLLM:
13 +
14 +.. code-block:: console
15 +
16 + $ docker run --runtime nvidia --gpus all \
17 + -v ~/.cache/huggingface:/root/.cache/huggingface \
18 + -p 8000:8000 \
19 + --env "HUGGING_FACE_HUB_TOKEN=<secret>" \
20 + vllm <args...>
21 +
requirements-dev.txt
+1
@@ -12,3 +12,4 @@ types-setuptools
12 pytest
13 pytest-forked
14 pytest-asyncio
15 +
setup.py
+6 -2
@@ -239,8 +239,12 @@ def find_version(filepath: str):
239
240
241 def read_readme() -> str:
242 - """Read the README file."""
243 - return io.open(get_path("README.md"), "r", encoding="utf-8").read()
242 + """Read the README file if present."""
243 + p = get_path("README.md")
244 + if os.path.isfile(p):
245 + return io.open(get_path("README.md"), "r", encoding="utf-8").read()
246 + else:
247 + return ""
248
249
250 def get_requirements() -> List[str]: