| 1 | #!/bin/bash |
| 2 | set -euo pipefail |
| 3 | |
| 4 | MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}" |
| 5 | PORT="${PORT:-8000}" |
| 6 | GPU="${GPU:-0}" |
| 7 | NUM_PROMPTS="${NUM_PROMPTS:-200}" |
| 8 | EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache}" |
| 9 | TIMEOUT="${TIMEOUT:-600}" |
| 10 | |
| 11 | SERVER_PID="" |
| 12 | |
| 13 | cleanup() { |
| 14 | echo "Stopping server..." |
| 15 | if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then |
| 16 | kill "$SERVER_PID" 2>/dev/null || true |
| 17 | wait "$SERVER_PID" 2>/dev/null || true |
| 18 | fi |
| 19 | echo "Done." |
| 20 | } |
| 21 | trap cleanup EXIT INT TERM |
| 22 | |
| 23 | wait_for_server() { |
| 24 | local deadline=$((SECONDS + TIMEOUT)) |
| 25 | echo "Waiting for server on port $PORT..." |
| 26 | while (( SECONDS < deadline )); do |
| 27 | if curl -sf "http://localhost:${PORT}/v1/models" > /dev/null 2>&1; then |
| 28 | echo "Server ready." |
| 29 | return 0 |
| 30 | fi |
| 31 | sleep 2 |
| 32 | done |
| 33 | echo "ERROR: Server did not start within ${TIMEOUT}s" |
| 34 | return 1 |
| 35 | } |
| 36 | |
| 37 | rm -rf "$EC_SHARED_STORAGE_PATH" |
| 38 | mkdir -p "$EC_SHARED_STORAGE_PATH" |
| 39 | |
| 40 | ############################################################################### |
| 41 | # Start server with ec_both |
| 42 | ############################################################################### |
| 43 | CUDA_VISIBLE_DEVICES="$GPU" \ |
| 44 | vllm serve "$MODEL" \ |
| 45 | --port "$PORT" \ |
| 46 | --enforce-eager \ |
| 47 | --ec-transfer-config '{ |
| 48 | "ec_connector": "ECExampleConnector", |
| 49 | "ec_role": "ec_both", |
| 50 | "ec_connector_extra_config": { |
| 51 | "shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'" |
| 52 | } |
| 53 | }' \ |
| 54 | "$@" & |
| 55 | |
| 56 | SERVER_PID=$! |
| 57 | wait_for_server |
| 58 | |
| 59 | ############################################################################### |
| 60 | # Benchmark -- dataset contains duplicate images, exercises cache hits |
| 61 | ############################################################################### |
| 62 | echo "Running benchmark ($NUM_PROMPTS prompts)..." |
| 63 | vllm bench serve \ |
| 64 | --model "$MODEL" \ |
| 65 | --backend openai-chat \ |
| 66 | --endpoint /v1/chat/completions \ |
| 67 | --dataset-name hf \ |
| 68 | --dataset-path lmarena-ai/VisionArena-Chat \ |
| 69 | --seed 0 \ |
| 70 | --num-prompts "$NUM_PROMPTS" \ |
| 71 | --port "$PORT" |
| 72 | |
| 73 | echo "Benchmark complete." |