@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3fefe271ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Update Dockerfile to build Megablocks (#2042)

Simon Mo committed Dec 12, 2023 at 17:34 UTC 3fefe271ecd0bcd03afa8c19416a8f71a7ed40e7
1 file changed +15 -3
Dockerfile
+15 -3
@@ -41,6 +41,14 @@ ENV NVCC_THREADS=$nvcc_threads
41
42 RUN python3 setup.py build_ext --inplace
43
44 +# Build the megablocks library as wheel because it doesn't publish pre-built wheels.
45 +# https://github.com/stanford-futuredata/megablocks/commit/5897cd6f254b7b3edf7a708a3a3314ecb54b6f78
46 +RUN apt-get install -y git && \
47 + git clone https://github.com/stanford-futuredata/megablocks.git && \
48 + cd megablocks && \
49 + git checkout 5897cd6f254b7b3edf7a708a3a3314ecb54b6f78 && \
50 + MAX_JOBS=8 NVCC_THREADS=8 python3 setup.py bdist_wheel
51 +
52 # image to run unit testing suite
53 FROM dev AS test
54
@@ -73,12 +81,16 @@ ENTRYPOINT ["python3", "-m", "vllm.entrypoints.api_server"]
81
82 # openai api server alternative
83 FROM vllm-base AS vllm-openai
76 -# install additional dependencies for openai api server, and mixtral
84 +# install additional dependencies for openai api server
85 RUN --mount=type=cache,target=/root/.cache/pip \
78 - pip install accelerate megablocks
86 + pip install accelerate
87
80 -COPY --from=build /workspace/vllm/*.so /workspace/vllm/
88 COPY vllm vllm
89 +COPY --from=build /workspace/vllm/*.so /workspace/vllm/
90 +COPY --from=build /workspace/megablocks/dist/*.whl /tmp/
91 +RUN --mount=type=cache,target=/root/.cache/pip \
92 + pip install /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl && \
93 + rm /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl
94
95 ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
96