A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[CI] check size of the wheels (#4319)
Simon Mo committed
May 4, 2024 at 13:44 UTC
021b1a2ab7497769dae8a67ea3467e4bafb474c5
2 files changed
+41
-4
.buildkite/check-wheel-size.py
new
+33
@@ -0,0 +1,33 @@
1
+import os
2
+import zipfile
3
+
4
+MAX_SIZE_MB = 100
5
+
6
+
7
+def print_top_10_largest_files(zip_file):
8
+ with zipfile.ZipFile(zip_file, 'r') as z:
9
+ file_sizes = [(f, z.getinfo(f).file_size) for f in z.namelist()]
10
+ file_sizes.sort(key=lambda x: x[1], reverse=True)
11
+ for f, size in file_sizes[:10]:
12
+ print(f"{f}: {size/(1024*1024)} MBs uncompressed.")
13
+
14
+
15
+def check_wheel_size(directory):
16
+ for root, _, files in os.walk(directory):
17
+ for f in files:
18
+ if f.endswith(".whl"):
19
+ wheel_path = os.path.join(root, f)
20
+ wheel_size = os.path.getsize(wheel_path)
21
+ wheel_size_mb = wheel_size / (1024 * 1024)
22
+ if wheel_size_mb > MAX_SIZE_MB:
23
+ print(
24
+ f"Wheel {wheel_path} is too large ({wheel_size_mb} MB) "
25
+ f"compare to the allowed size ({MAX_SIZE_MB} MB).")
26
+ print_top_10_largest_files(wheel_path)
27
+ return 1
28
+ return 0
29
+
30
+
31
+if __name__ == "__main__":
32
+ import sys
33
+ sys.exit(check_wheel_size(sys.argv[1]))
Dockerfile
+8
-4
@@ -7,7 +7,7 @@
7
8
#################### BASE BUILD IMAGE ####################
9
# prepare basic build environment
10
-FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
10
+FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS dev
11
12
RUN apt-get update -y \
13
&& apt-get install -y python3-pip git
@@ -16,7 +16,7 @@ RUN apt-get update -y \
16
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
17
# this won't be needed for future versions of this docker image
18
# or future versions of triton.
19
-RUN ldconfig /usr/local/cuda-12.1/compat/
19
+RUN ldconfig /usr/local/cuda-12.4/compat/
20
21
WORKDIR /workspace
22
@@ -75,6 +75,10 @@ RUN --mount=type=cache,target=/root/.cache/ccache \
75
--mount=type=cache,target=/root/.cache/pip \
76
python3 setup.py bdist_wheel --dist-dir=dist
77
78
+# check the size of the wheel, we cannot upload wheels larger than 100MB
79
+COPY .buildkite/check-wheel-size.py check-wheel-size.py
80
+RUN python3 check-wheel-size.py dist
81
+
82
# the `vllm_nccl` package must be installed from source distribution
83
# pip is too smart to store a wheel in the cache, and other CI jobs
84
# will directly use the wheel from the cache, which is not what we want.
@@ -102,7 +106,7 @@ RUN pip --verbose wheel flash-attn==${FLASH_ATTN_VERSION} \
106
107
#################### vLLM installation IMAGE ####################
108
# image with vLLM installed
105
-FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS vllm-base
109
+FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS vllm-base
110
WORKDIR /vllm-workspace
111
112
RUN apt-get update -y \
@@ -112,7 +116,7 @@ RUN apt-get update -y \
116
# https://github.com/pytorch/pytorch/issues/107960 -- hopefully
117
# this won't be needed for future versions of this docker image
118
# or future versions of triton.
115
-RUN ldconfig /usr/local/cuda-12.1/compat/
119
+RUN ldconfig /usr/local/cuda-12.4/compat/
120
121
# install vllm wheel first, so that torch etc will be installed
122
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \