A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[ci][tests] add gh200 tests (#11244)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 16, 2024 at 15:53 UTC
c301616ed23fef433db1a49df332b9d61d3178ad
1 file changed
+25
.buildkite/run-gh200-test.sh
new
+25
@@ -0,0 +1,25 @@
1
+#!/bin/bash
2
+
3
+# This script build the GH200 docker image and run the offline inference inside the container.
4
+# It serves a sanity check for compilation and basic model usage.
5
+set -ex
6
+
7
+# Try building the docker image
8
+DOCKER_BUILDKIT=1 docker build . \
9
+ --target test \
10
+ -platform "linux/arm64" \
11
+ -t gh200-test \
12
+ --build-arg max_jobs=66 \
13
+ --build-arg nvcc_threads=2 \
14
+ --build-arg torch_cuda_arch_list="9.0+PTX" \
15
+ --build-arg vllm_fa_cmake_gpu_arches="90-real"
16
+
17
+# Setup cleanup
18
+remove_docker_container() { docker rm -f gh200-test || true; }
19
+trap remove_docker_container EXIT
20
+remove_docker_container
21
+
22
+# Run the image and test offline inference
23
+docker run --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
24
+ python3 examples/offline_inference.py
25
+'