| 1 | #!/bin/bash |
| 2 | |
| 3 | HOST="0.0.0.0" |
| 4 | PORT=8006 |
| 5 | DATA_PARALLEL_SIZE=4 |
| 6 | REDUNDANT_EXPERTS=0 |
| 7 | LOCAL_MODEL_PATH="/models/models--deepseek-ai--DeepSeek-V2-Lite/snapshots/604d5664dddd88a0433dbae533b7fe9472482de0" |
| 8 | MODEL_NAME="deepseek-ai/DeepSeek-V2-Lite" |
| 9 | |
| 10 | while [[ $# -gt 0 ]]; do |
| 11 | case $1 in |
| 12 | --dp) |
| 13 | DATA_PARALLEL_SIZE="$2" |
| 14 | shift 2 |
| 15 | ;; |
| 16 | --re) |
| 17 | REDUNDANT_EXPERTS="$2" |
| 18 | shift 2 |
| 19 | ;; |
| 20 | --host) |
| 21 | HOST="$2" |
| 22 | shift 2 |
| 23 | ;; |
| 24 | --port) |
| 25 | PORT="$2" |
| 26 | shift 2 |
| 27 | ;; |
| 28 | --model) |
| 29 | MODEL_NAME="$2" |
| 30 | shift 2 |
| 31 | ;; |
| 32 | --local-model) |
| 33 | MODEL_NAME=$LOCAL_MODEL_PATH |
| 34 | shift |
| 35 | ;; |
| 36 | -h|--help) |
| 37 | echo "Usage: $0 [OPTIONS]" |
| 38 | echo "Options:" |
| 39 | echo " --dp SIZE Set data parallel size (default: 4)" |
| 40 | echo " --re SIZE Set redundant experts (default: 0)" |
| 41 | echo " --host HOST Set host address (default: 0.0.0.0)" |
| 42 | echo " --port PORT Set port number (default: 8006)" |
| 43 | echo " --model MODEL_NAME Set model name or path" |
| 44 | echo " -h, --help Show this help message" |
| 45 | exit 0 |
| 46 | ;; |
| 47 | *) |
| 48 | echo "Unknown option: $1" |
| 49 | echo "Use -h or --help for usage information" |
| 50 | exit 1 |
| 51 | ;; |
| 52 | esac |
| 53 | done |
| 54 | |
| 55 | echo "Starting vLLM server for $MODEL_NAME with data parallel size: $DATA_PARALLEL_SIZE and redundant experts: $REDUNDANT_EXPERTS" |
| 56 | |
| 57 | export RAY_DEDUP_LOGS=0 |
| 58 | export VLLM_USE_DEEP_GEMM=1 |
| 59 | |
| 60 | vllm serve "$MODEL_NAME" \ |
| 61 | --data-parallel-size "$DATA_PARALLEL_SIZE" \ |
| 62 | --data-parallel-size-local "$DATA_PARALLEL_SIZE" \ |
| 63 | --data-parallel-backend ray \ |
| 64 | --enforce-eager \ |
| 65 | --enable-expert-parallel \ |
| 66 | --enable-eplb \ |
| 67 | --all2all-backend allgather_reducescatter \ |
| 68 | --num-redundant-experts "$REDUNDANT_EXPERTS" \ |
| 69 | --trust-remote-code \ |
| 70 | --host "$HOST" \ |
| 71 | --port "$PORT" |