@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 021b1a2ab7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI] check size of the wheels (#4319)

Simon Mo committed May 4, 2024 at 13:44 UTC 021b1a2ab7497769dae8a67ea3467e4bafb474c5
2 files changed +41 -4
.buildkite/check-wheel-size.py new
+33
@@ -0,0 +1,33 @@
1 +import os
2 +import zipfile
3 +
4 +MAX_SIZE_MB = 100
5 +
6 +
7 +def print_top_10_largest_files(zip_file):
8 + with zipfile.ZipFile(zip_file, 'r') as z:
9 + file_sizes = [(f, z.getinfo(f).file_size) for f in z.namelist()]
10 + file_sizes.sort(key=lambda x: x[1], reverse=True)
11 + for f, size in file_sizes[:10]:
12 + print(f"{f}: {size/(1024*1024)} MBs uncompressed.")
13 +
14 +
15 +def check_wheel_size(directory):
16 + for root, _, files in os.walk(directory):
17 + for f in files:
18 + if f.endswith(".whl"):
19 + wheel_path = os.path.join(root, f)
20 + wheel_size = os.path.getsize(wheel_path)
21 + wheel_size_mb = wheel_size / (1024 * 1024)
22 + if wheel_size_mb > MAX_SIZE_MB:
23 + print(
24 + f"Wheel {wheel_path} is too large ({wheel_size_mb} MB) "
25 + f"compare to the allowed size ({MAX_SIZE_MB} MB).")
26 + print_top_10_largest_files(wheel_path)
27 + return 1
28 + return 0
29 +
30 +
31 +if __name__ == "__main__":
32 + import sys
33 + sys.exit(check_wheel_size(sys.argv[1]))
Dockerfile
+8 -4
@@ -7,7 +7,7 @@
7
8 #################### BASE BUILD IMAGE ####################
9 # prepare basic build environment
10 -FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS dev
10 +FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS dev
11
12 RUN apt-get update -y \
13 && apt-get install -y python3-pip git
@@ -16,7 +16,7 @@ RUN apt-get update -y \
16 # https://github.com/pytorch/pytorch/issues/107960 -- hopefully
17 # this won't be needed for future versions of this docker image
18 # or future versions of triton.
19 -RUN ldconfig /usr/local/cuda-12.1/compat/
19 +RUN ldconfig /usr/local/cuda-12.4/compat/
20
21 WORKDIR /workspace
22
@@ -75,6 +75,10 @@ RUN --mount=type=cache,target=/root/.cache/ccache \
75 --mount=type=cache,target=/root/.cache/pip \
76 python3 setup.py bdist_wheel --dist-dir=dist
77
78 +# check the size of the wheel, we cannot upload wheels larger than 100MB
79 +COPY .buildkite/check-wheel-size.py check-wheel-size.py
80 +RUN python3 check-wheel-size.py dist
81 +
82 # the `vllm_nccl` package must be installed from source distribution
83 # pip is too smart to store a wheel in the cache, and other CI jobs
84 # will directly use the wheel from the cache, which is not what we want.
@@ -102,7 +106,7 @@ RUN pip --verbose wheel flash-attn==${FLASH_ATTN_VERSION} \
106
107 #################### vLLM installation IMAGE ####################
108 # image with vLLM installed
105 -FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS vllm-base
109 +FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS vllm-base
110 WORKDIR /vllm-workspace
111
112 RUN apt-get update -y \
@@ -112,7 +116,7 @@ RUN apt-get update -y \
116 # https://github.com/pytorch/pytorch/issues/107960 -- hopefully
117 # this won't be needed for future versions of this docker image
118 # or future versions of triton.
115 -RUN ldconfig /usr/local/cuda-12.1/compat/
119 +RUN ldconfig /usr/local/cuda-12.4/compat/
120
121 # install vllm wheel first, so that torch etc will be installed
122 RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \