A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI/Build] Add run-hpu-test.sh script (#10167)
Signed-off-by: Chendi.Xue <chendi.xue@intel.com>
Chendi.Xue committed
Nov 9, 2024 at 00:26 UTC
8e1529dc573c9b4697fca24944918b8d68fd5906
2 files changed
+18
.buildkite/run-hpu-test.sh
new
+16
@@ -0,0 +1,16 @@
1
+#!/bin/bash
2
+
3
+# This script build the CPU docker image and run the offline inference inside the container.
4
+# It serves a sanity check for compilation and basic model usage.
5
+set -ex
6
+
7
+# Try building the docker image
8
+docker build -t hpu-test-env -f Dockerfile.hpu .
9
+
10
+# Setup cleanup
11
+remove_docker_container() { docker rm -f hpu-test || true; }
12
+trap remove_docker_container EXIT
13
+remove_docker_container
14
+
15
+# Run the image and launch offline inference
16
+docker run --runtime=habana --name=hpu-test --network=host -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference.py
\ No newline at end of file
Dockerfile.hpu
+2
@@ -13,4 +13,6 @@ RUN VLLM_TARGET_DEVICE=hpu python3 setup.py install
13
14
WORKDIR /workspace/
15
16
+RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
17
+
18
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]