@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c301616ed2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[ci][tests] add gh200 tests (#11244)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 16, 2024 at 15:53 UTC c301616ed23fef433db1a49df332b9d61d3178ad
1 file changed +25
.buildkite/run-gh200-test.sh new
+25
@@ -0,0 +1,25 @@
1 +#!/bin/bash
2 +
3 +# This script build the GH200 docker image and run the offline inference inside the container.
4 +# It serves a sanity check for compilation and basic model usage.
5 +set -ex
6 +
7 +# Try building the docker image
8 +DOCKER_BUILDKIT=1 docker build . \
9 + --target test \
10 + -platform "linux/arm64" \
11 + -t gh200-test \
12 + --build-arg max_jobs=66 \
13 + --build-arg nvcc_threads=2 \
14 + --build-arg torch_cuda_arch_list="9.0+PTX" \
15 + --build-arg vllm_fa_cmake_gpu_arches="90-real"
16 +
17 +# Setup cleanup
18 +remove_docker_container() { docker rm -f gh200-test || true; }
19 +trap remove_docker_container EXIT
20 +remove_docker_container
21 +
22 +# Run the image and test offline inference
23 +docker run --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
24 + python3 examples/offline_inference.py
25 +'