| 1 | #!/usr/bin/env bash |
| 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | |
| 5 | set -euo pipefail |
| 6 | |
| 7 | SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" |
| 8 | CONVERTER="${SCRIPT_DIR}/recipe_json_to_vllm_config.py" |
| 9 | OUTPUT_DIR="$(pwd -P)" |
| 10 | CONFIG_PATH="${OUTPUT_DIR}/config.yml" |
| 11 | ENV_PATH="${OUTPUT_DIR}/env.sh" |
| 12 | |
| 13 | args=("$@") |
| 14 | hardware="" |
| 15 | |
| 16 | for ((i = 0; i < ${#args[@]}; i++)); do |
| 17 | case "${args[$i]}" in |
| 18 | --hardware) |
| 19 | if ((i + 1 < ${#args[@]})); then |
| 20 | hardware="${args[$((i + 1))]}" |
| 21 | fi |
| 22 | ;; |
| 23 | --hardware=*) |
| 24 | hardware="${args[$i]#*=}" |
| 25 | ;; |
| 26 | esac |
| 27 | done |
| 28 | |
| 29 | if [[ "${hardware,,}" == "xeon6" ]]; then |
| 30 | args+=(--detect-hardware) |
| 31 | fi |
| 32 | |
| 33 | echo "==> Generating vLLM configuration" |
| 34 | python3 "${CONVERTER}" "${args[@]}" |
| 35 | |
| 36 | echo |
| 37 | echo "==> Loading recipe environment" |
| 38 | # shellcheck disable=SC1090 |
| 39 | source "${ENV_PATH}" |
| 40 | |
| 41 | # Released vLLM images keep recipe-relative assets such as examples/*.jinja |
| 42 | # under /vllm-workspace. Launch from there when available while keeping the |
| 43 | # generated config and environment files in the caller's output directory. |
| 44 | if [[ -d /vllm-workspace/examples ]]; then |
| 45 | cd /vllm-workspace |
| 46 | fi |
| 47 | |
| 48 | echo |
| 49 | echo "==> Starting vLLM" |
| 50 | exec vllm serve --config "${CONFIG_PATH}" |