@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a0550cbc80
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

Add support for multi-node on CI (#5955)

Signed-off-by: kevin <kevin@anyscale.com>

Kevin H. Luu committed Jul 9, 2024 at 12:56 UTC a0550cbc80f504aa2da80b573c22204f686a0389
1 file changed +77
.buildkite/run-multi-node-test.sh new
+77
@@ -0,0 +1,77 @@
1 +#!/bin/bash
2 +
3 +set -euox pipefail
4 +
5 +if [[ $# -lt 3 ]]; then
6 + echo "Please provide the number of nodes and GPU per node."
7 + exit 1
8 +fi
9 +
10 +NUM_NODES=$1
11 +NUM_GPUS=$2
12 +DOCKER_IMAGE=$3
13 +
14 +shift 3
15 +COMMANDS=("$@")
16 +if [ ${#COMMANDS[@]} -ne $NUM_NODES ]; then
17 + echo "The number of commands must be equal to the number of nodes."
18 + echo "Number of nodes: $NUM_NODES"
19 + echo "Number of commands: ${#COMMANDS[@]}"
20 + exit 1
21 +fi
22 +
23 +echo "List of commands"
24 +for command in "${COMMANDS[@]}"; do
25 + echo $command
26 +done
27 +
28 +start_network() {
29 + docker network create --subnet=192.168.10.0/24 docker-net
30 +}
31 +
32 +start_nodes() {
33 + for node in $(seq 0 $(($NUM_NODES-1))); do
34 + GPU_DEVICES='"device='
35 + for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
36 + DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
37 + GPU_DEVICES+=$(($DEVICE_NUM))
38 + if [ $node_gpu -lt $(($NUM_GPUS - 1)) ]; then
39 + GPU_DEVICES+=','
40 + fi
41 + done
42 + GPU_DEVICES+='"'
43 + # echo "Starting node$node with GPU devices: $GPU_DEVICES"
44 + docker run -d --gpus "$GPU_DEVICES" --name node$node --network docker-net --ip 192.168.10.$((10 + $node)) --rm $DOCKER_IMAGE tail -f /dev/null
45 + done
46 +}
47 +
48 +run_nodes() {
49 + for node in $(seq 0 $(($NUM_NODES-1))); do
50 + GPU_DEVICES='"device='
51 + for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
52 + DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
53 + GPU_DEVICES+=$(($DEVICE_NUM))
54 + if [ $node_gpu -lt $(($NUM_GPUS - 1)) ]; then
55 + GPU_DEVICES+=','
56 + fi
57 + done
58 + GPU_DEVICES+='"'
59 + echo "Running node$node with GPU devices: $GPU_DEVICES"
60 + if [ $node -lt $(($NUM_NODES - 1)) ]; then
61 + docker exec -d node$node /bin/bash -c "${COMMANDS[$node]}"
62 + else
63 + docker exec node$node /bin/bash -c "${COMMANDS[$node]}"
64 + fi
65 + done
66 +}
67 +cleanup() {
68 + for node in $(seq 0 $(($NUM_NODES-1))); do
69 + docker stop node$node
70 + done
71 + docker network rm docker-net
72 +}
73 +trap cleanup EXIT
74 +start_network
75 +start_nodes
76 +run_nodes
77 +