@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
sh 253 lines 8.3 KB
Raw
1 #!/bin/bash
2 # Run BFCL (Berkeley Function Call Leaderboard) tool-calling correctness
3 # evaluation against a local vLLM server.
4 #
5 # Usage:
6 # # Run with defaults (gpt-oss-20b, multi_turn)
7 # bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
8 #
9 # # Run with gpt-oss-120b and multiple test categories
10 # BFCL_MODEL="openai/gpt-oss-120b" BFCL_TP_SIZE=4 \
11 # BFCL_TEST_CATEGORY="live_simple, multiple, parallel_multiple" \
12 # bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
13 #
14 # # Chain both API types (use BFCL_OUTPUT_DIR to avoid overwriting results)
15 # BFCL_OUTPUT_DIR=./bfcl-chat-completions BFCL_API_TYPE=chat_completions \
16 # bash .buildkite/scripts/tool_call/run-bfcl-eval.sh && \
17 # BFCL_OUTPUT_DIR=./bfcl-responses BFCL_API_TYPE=responses \
18 # bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
19 #
20 # Environment variables (all optional, with defaults):
21 # BFCL_MODEL - HF model name (default: openai/gpt-oss-20b)
22 # BFCL_API_TYPE - API type: "chat_completions" or "responses" (default: chat_completions)
23 # BFCL_OUTPUT_DIR - Directory for BFCL results (default: current working directory)
24 # BFCL_TEST_CATEGORY - BFCL test categories (default: multi_turn)
25 # BFCL_TOOL_CALL_PARSER - Tool call parser name (default: openai)
26 # BFCL_NUM_THREADS - Threads for BFCL generate (default: 8)
27 # BFCL_TP_SIZE - Tensor parallel size (default: 1)
28 # BFCL_MAX_MODEL_LEN - Max model length (default: 4096)
29 # BFCL_PORT - Server port (default: 8000)
30 # BFCL_REASONING_PARSER - Reasoning parser name (default: disabled)
31 # BFCL_TEMPERATURE - Temperature (default: 0.0)
32 # BFCL_EXTRA_ARGS - Additional vLLM server args
33
34 set -euo pipefail
35
36 # ---- Configuration ----
37 MODEL="${BFCL_MODEL:-openai/gpt-oss-20b}"
38 API_TYPE="${BFCL_API_TYPE:-chat_completions}"
39 OUTPUT_DIR="${BFCL_OUTPUT_DIR:-}"
40 TEST_CATEGORY="${BFCL_TEST_CATEGORY:-multi_turn}"
41 TOOL_CALL_PARSER="${BFCL_TOOL_CALL_PARSER:-openai}"
42 NUM_THREADS="${BFCL_NUM_THREADS:-8}"
43 TP_SIZE="${BFCL_TP_SIZE:-1}"
44 MAX_MODEL_LEN="${BFCL_MAX_MODEL_LEN:-4096}"
45 PORT="${BFCL_PORT:-8000}"
46 REASONING_PARSER="${BFCL_REASONING_PARSER:-}"
47 TEMPERATURE="${BFCL_TEMPERATURE:-0.0}"
48 EXTRA_ARGS="${BFCL_EXTRA_ARGS:-}"
49
50 # Set up output directory
51 if [ -n "$OUTPUT_DIR" ]; then
52 mkdir -p "$OUTPUT_DIR"
53 OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)"
54 fi
55
56 echo "============================================"
57 echo "BFCL Tool Call Correctness Evaluation"
58 echo "============================================"
59 echo "Model: $MODEL"
60 echo "Tool parser: $TOOL_CALL_PARSER"
61 echo "API type: $API_TYPE"
62 echo "Output dir: ${OUTPUT_DIR:-<cwd>}"
63 echo "Test category: $TEST_CATEGORY"
64 echo "TP size: $TP_SIZE"
65 echo "Max model len: $MAX_MODEL_LEN"
66 echo "Port: $PORT"
67 echo "Num threads: $NUM_THREADS"
68 echo "============================================"
69
70 # ---- Install bfcl-eval if missing ----
71 if ! python3 -c "import bfcl_eval" 2>/dev/null; then
72 echo "Installing bfcl-eval..."
73 uv pip install "bfcl-eval>=2025.10.20.1,<2026"
74 fi
75
76 # ---- Cleanup handler ----
77 SERVER_PID=""
78 # shellcheck disable=SC2329 # invoked via `trap cleanup EXIT` below
79 cleanup() {
80 if [ -n "$SERVER_PID" ]; then
81 echo "Stopping vLLM server (pid=$SERVER_PID)..."
82 kill "$SERVER_PID" 2>/dev/null || true
83 wait "$SERVER_PID" 2>/dev/null || true
84 fi
85 # Remove BFCL lock files (created by filelock for thread-safe writes)
86 rm -rf .file_locks/
87 if [ -n "${OUTPUT_DIR:-}" ]; then
88 rm -rf "$OUTPUT_DIR/.file_locks/"
89 fi
90 }
91 trap cleanup EXIT
92
93 # ---- Start vLLM server ----
94 echo "Starting vLLM server..."
95
96 SERVE_ARGS=(
97 "$MODEL"
98 --port "$PORT"
99 --enable-auto-tool-choice
100 --tool-call-parser "$TOOL_CALL_PARSER"
101 --tensor-parallel-size "$TP_SIZE"
102 --max-model-len "$MAX_MODEL_LEN"
103 --enforce-eager
104 --enable-prefix-caching
105 )
106
107 # Append reasoning parser if specified
108 if [ -n "$REASONING_PARSER" ]; then
109 SERVE_ARGS+=(--reasoning-parser "$REASONING_PARSER")
110 fi
111
112 # Append any extra args
113 if [ -n "$EXTRA_ARGS" ]; then
114 read -ra EXTRA_ARGS_ARRAY <<< "$EXTRA_ARGS"
115 SERVE_ARGS+=("${EXTRA_ARGS_ARRAY[@]}")
116 fi
117
118 echo "Command: vllm serve ${SERVE_ARGS[*]}"
119 vllm serve "${SERVE_ARGS[@]}" &
120 SERVER_PID=$!
121
122 # ---- Wait for server to be ready ----
123 echo "Waiting for vLLM server to start (timeout: 600s)..."
124 SECONDS_WAITED=0
125 until curl -sf "http://localhost:${PORT}/health" > /dev/null 2>&1; do
126 if [ $SECONDS_WAITED -ge 600 ]; then
127 echo ""
128 echo "ERROR: vLLM server failed to start within 600s"
129 exit 1
130 fi
131 if (( SECONDS_WAITED % 30 == 0 && SECONDS_WAITED > 0 )); then
132 echo " Still waiting... (${SECONDS_WAITED}s elapsed)"
133 fi
134 sleep 2
135 SECONDS_WAITED=$((SECONDS_WAITED + 2))
136 done
137 echo "vLLM server is ready. (started in ${SECONDS_WAITED}s)"
138
139 # ---- Run BFCL evaluation ----
140 # bfcl-eval has no CLI entry point; generate() and evaluate() are Typer
141 # functions that must be called from Python. The MODEL_CONFIG_MAPPING must
142 # be patched in-process so BFCL knows to use the OpenAI-compatible handler
143 # against our local vLLM server.
144 bfcl_exit_code=0
145 python3 - "$MODEL" "$TEST_CATEGORY" "$NUM_THREADS" "$PORT" "$API_TYPE" "$TEMPERATURE" "$OUTPUT_DIR" << 'PYEOF' || bfcl_exit_code=$?
146 import os
147 import sys
148
149 model = sys.argv[1]
150 test_category = sys.argv[2]
151 num_threads = int(sys.argv[3])
152 port = sys.argv[4]
153 api_type = sys.argv[5]
154 temperature = float(sys.argv[6])
155 output_dir = sys.argv[7] if len(sys.argv) > 7 and sys.argv[7] else os.getcwd()
156
157 os.environ["OPENAI_BASE_URL"] = f"http://localhost:{port}/v1"
158 os.environ["OPENAI_API_KEY"] = "dummy"
159 os.environ["BFCL_PROJECT_ROOT"] = output_dir
160
161 import bfcl_eval.constants.model_config as bfcl_model_config
162 from bfcl_eval.constants.model_config import ModelConfig
163 from bfcl_eval.model_handler.api_inference.openai_completion import (
164 OpenAICompletionsHandler,
165 )
166 from bfcl_eval.model_handler.api_inference.openai_response import (
167 OpenAIResponsesHandler,
168 )
169
170 if api_type == "responses":
171 handler = OpenAIResponsesHandler
172 else:
173 handler = OpenAICompletionsHandler
174
175 bfcl_model_config.MODEL_CONFIG_MAPPING[model] = ModelConfig(
176 model_name=model,
177 display_name=f"{model} (FC) (vLLM)",
178 url=f"https://huggingface.co/{model}",
179 org="",
180 license="apache-2.0",
181 model_handler=handler,
182 input_price=None,
183 output_price=None,
184 is_fc_model=True,
185 underscore_to_dot=True,
186 )
187
188 from bfcl_eval.__main__ import evaluate, generate
189 import inspect
190 import typer
191
192
193 def _get_default_kwargs(function):
194 kwargs = {}
195 for k, v in inspect.signature(function).parameters.items():
196 if v.default is not inspect.Parameter.empty:
197 default = v.default
198 if isinstance(default, typer.models.OptionInfo):
199 default = default.default
200 kwargs[k] = default
201 return kwargs
202
203
204 # ---- generate ----
205 print(f"=== BFCL generate: model={model} test_category={test_category} ===")
206 gen_kwargs = _get_default_kwargs(generate)
207 gen_kwargs["model"] = [model]
208 gen_kwargs["test_category"] = [c.strip() for c in test_category.split(",")]
209 gen_kwargs["skip_server_setup"] = True
210 gen_kwargs["num_threads"] = num_threads
211 gen_kwargs["temperature"] = temperature
212 generate(**gen_kwargs)
213
214 # ---- evaluate ----
215 print(f"=== BFCL evaluate: model={model} test_category={test_category} ===")
216 eval_kwargs = _get_default_kwargs(evaluate)
217 eval_kwargs["model"] = [model]
218 eval_kwargs["test_category"] = [c.strip() for c in test_category.split(",")]
219 evaluate(**eval_kwargs)
220
221 print("=== BFCL evaluation completed successfully ===")
222 PYEOF
223
224 # ---- Upload results to buildkite ----
225 if command -v buildkite-agent &>/dev/null; then
226 if [ $bfcl_exit_code -eq 0 ]; then
227 STYLE="success"
228 STATUS="PASSED"
229 else
230 STYLE="error"
231 STATUS="FAILED"
232 fi
233
234 buildkite-agent annotate --style "$STYLE" --context "bfcl-results" <<EOF
235 ### BFCL Tool Call Correctness - ${STATUS}
236 - **Model:** \`${MODEL}\`
237 - **Parser:** \`${TOOL_CALL_PARSER}\`
238 - **API type:** \`${API_TYPE}\`
239 - **Test category:** \`${TEST_CATEGORY}\`
240 EOF
241
242 # BFCL writes results to $BFCL_PROJECT_ROOT/result/ and scores to
243 # $BFCL_PROJECT_ROOT/score/
244 RESULTS_ROOT="${OUTPUT_DIR:-.}"
245 if [ -d "$RESULTS_ROOT/result" ]; then
246 buildkite-agent artifact upload "$RESULTS_ROOT/result/**/*"
247 fi
248 if [ -d "$RESULTS_ROOT/score" ]; then
249 buildkite-agent artifact upload "$RESULTS_ROOT/score/**/*"
250 fi
251 fi
252
253 exit $bfcl_exit_code