A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[CI/BUILD] Support non-AVX512 vLLM building and testing (#5574)
Jie Fu (傅杰) committed
Jun 18, 2024 at 02:36 UTC
ab66536dbfedff4ffcbb6dc9f9a21d0a9ac0ec91
3 files changed
+20
-2
.buildkite/run-cpu-test.sh
+4
-1
@@ -4,17 +4,20 @@ set -ex
4
5
# Try building the docker image
6
docker build -t cpu-test -f Dockerfile.cpu .
7
+docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
8
9
# Setup cleanup
9
-remove_docker_container() { docker rm -f cpu-test || true; }
10
+remove_docker_container() { docker rm -f cpu-test cpu-test-avx2 || true; }
11
trap remove_docker_container EXIT
12
remove_docker_container
13
14
# Run the image
15
docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
16
+docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test-avx2 cpu-test-avx2
17
18
# offline inference
19
docker exec cpu-test bash -c "python3 examples/offline_inference.py"
20
+docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
21
22
# Run basic model test
23
docker exec cpu-test bash -c "cd tests;
Dockerfile.cpu
+4
@@ -21,6 +21,10 @@ WORKDIR /workspace/vllm
21
22
RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
23
24
+# Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ...
25
+ARG VLLM_CPU_DISABLE_AVX512
26
+ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
27
+
28
RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
29
30
WORKDIR /workspace/
cmake/cpu_extension.cmake
+12
-1
@@ -33,10 +33,21 @@ function (find_isa CPUINFO TARGET OUT)
33
endif()
34
endfunction()
35
36
+function (is_avx512_disabled OUT)
37
+ set(DISABLE_AVX512 $ENV{VLLM_CPU_DISABLE_AVX512})
38
+ if(DISABLE_AVX512 AND DISABLE_AVX512 STREQUAL "true")
39
+ set(${OUT} ON PARENT_SCOPE)
40
+ else()
41
+ set(${OUT} OFF PARENT_SCOPE)
42
+ endif()
43
+endfunction()
44
+
45
+is_avx512_disabled(AVX512_DISABLED)
46
+
47
find_isa(${CPUINFO} "avx2" AVX2_FOUND)
48
find_isa(${CPUINFO} "avx512f" AVX512_FOUND)
49
39
-if (AVX512_FOUND)
50
+if (AVX512_FOUND AND NOT AVX512_DISABLED)
51
list(APPEND CXX_COMPILE_FLAGS
52
"-mavx512f"
53
"-mavx512vl"