A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Update Dockerfile to build Megablocks (#2042)
Simon Mo committed
Dec 12, 2023 at 17:34 UTC
3fefe271ecd0bcd03afa8c19416a8f71a7ed40e7
1 file changed
+15
-3
Dockerfile
+15
-3
@@ -41,6 +41,14 @@ ENV NVCC_THREADS=$nvcc_threads
41
42
RUN python3 setup.py build_ext --inplace
43
44
+# Build the megablocks library as wheel because it doesn't publish pre-built wheels.
45
+# https://github.com/stanford-futuredata/megablocks/commit/5897cd6f254b7b3edf7a708a3a3314ecb54b6f78
46
+RUN apt-get install -y git && \
47
+ git clone https://github.com/stanford-futuredata/megablocks.git && \
48
+ cd megablocks && \
49
+ git checkout 5897cd6f254b7b3edf7a708a3a3314ecb54b6f78 && \
50
+ MAX_JOBS=8 NVCC_THREADS=8 python3 setup.py bdist_wheel
51
+
52
# image to run unit testing suite
53
FROM dev AS test
54
@@ -73,12 +81,16 @@ ENTRYPOINT ["python3", "-m", "vllm.entrypoints.api_server"]
81
82
# openai api server alternative
83
FROM vllm-base AS vllm-openai
76
-# install additional dependencies for openai api server, and mixtral
84
+# install additional dependencies for openai api server
85
RUN --mount=type=cache,target=/root/.cache/pip \
78
- pip install accelerate megablocks
86
+ pip install accelerate
87
80
-COPY --from=build /workspace/vllm/*.so /workspace/vllm/
88
COPY vllm vllm
89
+COPY --from=build /workspace/vllm/*.so /workspace/vllm/
90
+COPY --from=build /workspace/megablocks/dist/*.whl /tmp/
91
+RUN --mount=type=cache,target=/root/.cache/pip \
92
+ pip install /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl && \
93
+ rm /tmp/megablocks-0.5.0-cp310-cp310-linux_x86_64.whl
94
95
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
96