| 1 | #!/bin/bash |
| 2 | |
| 3 | set -euo pipefail |
| 4 | |
| 5 | VLLM_LOG="$WORKSPACE/vllm_log.txt" |
| 6 | BM_LOG="$WORKSPACE/bm_log.txt" |
| 7 | |
| 8 | if [ -n "$TARGET_COMMIT" ]; then |
| 9 | head_hash=$(git rev-parse HEAD) |
| 10 | if [ "$TARGET_COMMIT" != "$head_hash" ]; then |
| 11 | echo "Error: target commit $TARGET_COMMIT does not match HEAD: $head_hash" |
| 12 | exit 1 |
| 13 | fi |
| 14 | fi |
| 15 | |
| 16 | echo "model: $MODEL" |
| 17 | echo |
| 18 | |
| 19 | # |
| 20 | # create a log folder |
| 21 | # |
| 22 | mkdir "$WORKSPACE/log" |
| 23 | |
| 24 | # TODO: Move to image building. |
| 25 | pip install pandas |
| 26 | pip install datasets |
| 27 | |
| 28 | # |
| 29 | # create sonnet_4x |
| 30 | # |
| 31 | echo "Create sonnet_4x.txt" |
| 32 | echo "" > benchmarks/sonnet_4x.txt |
| 33 | for _ in {1..4} |
| 34 | do |
| 35 | cat benchmarks/sonnet.txt >> benchmarks/sonnet_4x.txt |
| 36 | done |
| 37 | |
| 38 | # |
| 39 | # start vllm service in backend |
| 40 | # |
| 41 | echo "lanching vllm..." |
| 42 | echo "logging to $VLLM_LOG" |
| 43 | echo |
| 44 | |
| 45 | vllm serve "$MODEL" \ |
| 46 | --seed 42 \ |
| 47 | --max-num-seqs "$MAX_NUM_SEQS" \ |
| 48 | --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ |
| 49 | --tensor-parallel-size "$TENSOR_PARALLEL_SIZE" \ |
| 50 | --no-enable-prefix-caching \ |
| 51 | --download_dir "$DOWNLOAD_DIR" \ |
| 52 | --max-model-len "$MAX_MODEL_LEN" > "$VLLM_LOG" 2>&1 & |
| 53 | |
| 54 | |
| 55 | echo "wait for 20 minutes.." |
| 56 | echo |
| 57 | # sleep 1200 |
| 58 | # wait for 10 minutes... |
| 59 | for _ in {1..120}; do |
| 60 | # TODO: detect other type of errors. |
| 61 | if grep -Fq "raise RuntimeError" "$VLLM_LOG"; then |
| 62 | echo "Detected RuntimeError, exiting." |
| 63 | exit 1 |
| 64 | elif grep -Fq "Application startup complete" "$VLLM_LOG"; then |
| 65 | echo "Application started" |
| 66 | break |
| 67 | else |
| 68 | echo "wait for 10 seconds..." |
| 69 | sleep 10 |
| 70 | fi |
| 71 | done |
| 72 | |
| 73 | # |
| 74 | # run test |
| 75 | # |
| 76 | echo "run benchmark test..." |
| 77 | echo "logging to $BM_LOG" |
| 78 | echo |
| 79 | vllm bench serve \ |
| 80 | --backend vllm \ |
| 81 | --model "$MODEL" \ |
| 82 | --dataset-name sonnet \ |
| 83 | --dataset-path benchmarks/sonnet_4x.txt \ |
| 84 | --sonnet-input-len "$INPUT_LEN" \ |
| 85 | --sonnet-output-len "$OUTPUT_LEN" \ |
| 86 | --ignore-eos > "$BM_LOG" |
| 87 | |
| 88 | echo "completed..." |
| 89 | echo |
| 90 | |
| 91 | throughput=$(grep "Request throughput (req/s):" "$BM_LOG" | sed 's/[^0-9.]//g') |
| 92 | echo "throughput: $throughput" |
| 93 | echo |