A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[CI] Add bfcl tool call correctness eval (#36560)
Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Flora Feng committed
Mar 11, 2026 at 12:27 UTC
741ecf06304097454e4e11a4714918a0ac55e17d
1 file changed
+248
.buildkite/scripts/tool_call/run-bfcl-eval.sh
new
+248
@@ -0,0 +1,248 @@
1
+#!/bin/bash
2
+# Run BFCL (Berkeley Function Call Leaderboard) tool-calling correctness
3
+# evaluation against a local vLLM server.
4
+#
5
+# Usage:
6
+# # Run with defaults (gpt-oss-20b, multi_turn)
7
+# bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
8
+#
9
+# # Run with gpt-oss-120b and multiple test categories
10
+# BFCL_MODEL="openai/gpt-oss-120b" BFCL_TP_SIZE=4 \
11
+# BFCL_TEST_CATEGORY="live_simple, multiple, parallel_multiple" \
12
+# bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
13
+#
14
+# # Chain both API types (use BFCL_OUTPUT_DIR to avoid overwriting results)
15
+# BFCL_OUTPUT_DIR=./bfcl-chat-completions BFCL_API_TYPE=chat_completions \
16
+# bash .buildkite/scripts/tool_call/run-bfcl-eval.sh && \
17
+# BFCL_OUTPUT_DIR=./bfcl-responses BFCL_API_TYPE=responses \
18
+# bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
19
+#
20
+# Environment variables (all optional, with defaults):
21
+# BFCL_MODEL - HF model name (default: openai/gpt-oss-20b)
22
+# BFCL_API_TYPE - API type: "chat_completions" or "responses" (default: chat_completions)
23
+# BFCL_OUTPUT_DIR - Directory for BFCL results (default: current working directory)
24
+# BFCL_TEST_CATEGORY - BFCL test categories (default: multi_turn)
25
+# BFCL_TOOL_CALL_PARSER - Tool call parser name (default: openai)
26
+# BFCL_NUM_THREADS - Threads for BFCL generate (default: 8)
27
+# BFCL_TP_SIZE - Tensor parallel size (default: 1)
28
+# BFCL_MAX_MODEL_LEN - Max model length (default: 4096)
29
+# BFCL_PORT - Server port (default: 8000)
30
+# BFCL_REASONING_PARSER - Reasoning parser name (default: disabled)
31
+# BFCL_EXTRA_ARGS - Additional vLLM server args
32
+
33
+set -euo pipefail
34
+
35
+# ---- Configuration ----
36
+MODEL="${BFCL_MODEL:-openai/gpt-oss-20b}"
37
+API_TYPE="${BFCL_API_TYPE:-chat_completions}"
38
+OUTPUT_DIR="${BFCL_OUTPUT_DIR:-}"
39
+TEST_CATEGORY="${BFCL_TEST_CATEGORY:-multi_turn}"
40
+TOOL_CALL_PARSER="${BFCL_TOOL_CALL_PARSER:-openai}"
41
+NUM_THREADS="${BFCL_NUM_THREADS:-8}"
42
+TP_SIZE="${BFCL_TP_SIZE:-1}"
43
+MAX_MODEL_LEN="${BFCL_MAX_MODEL_LEN:-4096}"
44
+PORT="${BFCL_PORT:-8000}"
45
+REASONING_PARSER="${BFCL_REASONING_PARSER:-}"
46
+EXTRA_ARGS="${BFCL_EXTRA_ARGS:-}"
47
+
48
+# Set up output directory
49
+if [ -n "$OUTPUT_DIR" ]; then
50
+ mkdir -p "$OUTPUT_DIR"
51
+ OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)"
52
+fi
53
+
54
+echo "============================================"
55
+echo "BFCL Tool Call Correctness Evaluation"
56
+echo "============================================"
57
+echo "Model: $MODEL"
58
+echo "Tool parser: $TOOL_CALL_PARSER"
59
+echo "API type: $API_TYPE"
60
+echo "Output dir: ${OUTPUT_DIR:-<cwd>}"
61
+echo "Test category: $TEST_CATEGORY"
62
+echo "TP size: $TP_SIZE"
63
+echo "Max model len: $MAX_MODEL_LEN"
64
+echo "Port: $PORT"
65
+echo "Num threads: $NUM_THREADS"
66
+echo "============================================"
67
+
68
+# ---- Install bfcl-eval if missing ----
69
+if ! python3 -c "import bfcl_eval" 2>/dev/null; then
70
+ echo "Installing bfcl-eval..."
71
+ pip install "bfcl-eval>=2025.10.20.1,<2026"
72
+fi
73
+
74
+# ---- Cleanup handler ----
75
+SERVER_PID=""
76
+cleanup() {
77
+ if [ -n "$SERVER_PID" ]; then
78
+ echo "Stopping vLLM server (pid=$SERVER_PID)..."
79
+ kill "$SERVER_PID" 2>/dev/null || true
80
+ wait "$SERVER_PID" 2>/dev/null || true
81
+ fi
82
+ # Remove BFCL lock files (created by filelock for thread-safe writes)
83
+ rm -rf .file_locks/
84
+ if [ -n "${OUTPUT_DIR:-}" ]; then
85
+ rm -rf "$OUTPUT_DIR/.file_locks/"
86
+ fi
87
+}
88
+trap cleanup EXIT
89
+
90
+# ---- Start vLLM server ----
91
+echo "Starting vLLM server..."
92
+
93
+SERVE_ARGS=(
94
+ "$MODEL"
95
+ --port "$PORT"
96
+ --enable-auto-tool-choice
97
+ --tool-call-parser "$TOOL_CALL_PARSER"
98
+ --tensor-parallel-size "$TP_SIZE"
99
+ --max-model-len "$MAX_MODEL_LEN"
100
+ --enforce-eager
101
+ --no-enable-prefix-caching
102
+)
103
+
104
+# Append reasoning parser if specified
105
+if [ -n "$REASONING_PARSER" ]; then
106
+ SERVE_ARGS+=(--reasoning-parser "$REASONING_PARSER")
107
+fi
108
+
109
+# Append any extra args
110
+if [ -n "$EXTRA_ARGS" ]; then
111
+ read -ra EXTRA_ARGS_ARRAY <<< "$EXTRA_ARGS"
112
+ SERVE_ARGS+=("${EXTRA_ARGS_ARRAY[@]}")
113
+fi
114
+
115
+echo "Command: vllm serve ${SERVE_ARGS[*]}"
116
+vllm serve "${SERVE_ARGS[@]}" &
117
+SERVER_PID=$!
118
+
119
+# ---- Wait for server to be ready ----
120
+echo "Waiting for vLLM server to start (timeout: 600s)..."
121
+SECONDS_WAITED=0
122
+until curl -sf "http://localhost:${PORT}/health" > /dev/null 2>&1; do
123
+ if [ $SECONDS_WAITED -ge 600 ]; then
124
+ echo ""
125
+ echo "ERROR: vLLM server failed to start within 600s"
126
+ exit 1
127
+ fi
128
+ if (( SECONDS_WAITED % 30 == 0 && SECONDS_WAITED > 0 )); then
129
+ echo " Still waiting... (${SECONDS_WAITED}s elapsed)"
130
+ fi
131
+ sleep 2
132
+ SECONDS_WAITED=$((SECONDS_WAITED + 2))
133
+done
134
+echo "vLLM server is ready. (started in ${SECONDS_WAITED}s)"
135
+
136
+# ---- Run BFCL evaluation ----
137
+# bfcl-eval has no CLI entry point; generate() and evaluate() are Typer
138
+# functions that must be called from Python. The MODEL_CONFIG_MAPPING must
139
+# be patched in-process so BFCL knows to use the OpenAI-compatible handler
140
+# against our local vLLM server.
141
+bfcl_exit_code=0
142
+python3 - "$MODEL" "$TEST_CATEGORY" "$NUM_THREADS" "$PORT" "$API_TYPE" "$OUTPUT_DIR" << 'PYEOF' || bfcl_exit_code=$?
143
+import os
144
+import sys
145
+
146
+model = sys.argv[1]
147
+test_category = sys.argv[2]
148
+num_threads = int(sys.argv[3])
149
+port = sys.argv[4]
150
+api_type = sys.argv[5]
151
+output_dir = sys.argv[6] if len(sys.argv) > 6 and sys.argv[6] else os.getcwd()
152
+
153
+os.environ["OPENAI_BASE_URL"] = f"http://localhost:{port}/v1"
154
+os.environ["OPENAI_API_KEY"] = "dummy"
155
+os.environ["BFCL_PROJECT_ROOT"] = output_dir
156
+
157
+import bfcl_eval.constants.model_config as bfcl_model_config
158
+from bfcl_eval.constants.model_config import ModelConfig
159
+from bfcl_eval.model_handler.api_inference.openai_completion import (
160
+ OpenAICompletionsHandler,
161
+)
162
+from bfcl_eval.model_handler.api_inference.openai_response import (
163
+ OpenAIResponsesHandler,
164
+)
165
+
166
+if api_type == "responses":
167
+ handler = OpenAIResponsesHandler
168
+else:
169
+ handler = OpenAICompletionsHandler
170
+
171
+bfcl_model_config.MODEL_CONFIG_MAPPING[model] = ModelConfig(
172
+ model_name=model,
173
+ display_name=f"{model} (FC) (vLLM)",
174
+ url=f"https://huggingface.co/{model}",
175
+ org="",
176
+ license="apache-2.0",
177
+ model_handler=handler,
178
+ input_price=None,
179
+ output_price=None,
180
+ is_fc_model=True,
181
+ underscore_to_dot=True,
182
+)
183
+
184
+from bfcl_eval.__main__ import evaluate, generate
185
+import inspect
186
+import typer
187
+
188
+
189
+def _get_default_kwargs(function):
190
+ kwargs = {}
191
+ for k, v in inspect.signature(function).parameters.items():
192
+ if v.default is not inspect.Parameter.empty:
193
+ default = v.default
194
+ if isinstance(default, typer.models.OptionInfo):
195
+ default = default.default
196
+ kwargs[k] = default
197
+ return kwargs
198
+
199
+
200
+# ---- generate ----
201
+print(f"=== BFCL generate: model={model} test_category={test_category} ===")
202
+gen_kwargs = _get_default_kwargs(generate)
203
+gen_kwargs["model"] = [model]
204
+gen_kwargs["test_category"] = [c.strip() for c in test_category.split(",")]
205
+gen_kwargs["skip_server_setup"] = True
206
+gen_kwargs["num_threads"] = num_threads
207
+generate(**gen_kwargs)
208
+
209
+# ---- evaluate ----
210
+print(f"=== BFCL evaluate: model={model} test_category={test_category} ===")
211
+eval_kwargs = _get_default_kwargs(evaluate)
212
+eval_kwargs["model"] = [model]
213
+eval_kwargs["test_category"] = [c.strip() for c in test_category.split(",")]
214
+evaluate(**eval_kwargs)
215
+
216
+print("=== BFCL evaluation completed successfully ===")
217
+PYEOF
218
+
219
+# ---- Upload results to buildkite ----
220
+if command -v buildkite-agent &>/dev/null; then
221
+ if [ $bfcl_exit_code -eq 0 ]; then
222
+ STYLE="success"
223
+ STATUS="PASSED"
224
+ else
225
+ STYLE="error"
226
+ STATUS="FAILED"
227
+ fi
228
+
229
+ buildkite-agent annotate --style "$STYLE" --context "bfcl-results" <<EOF
230
+### BFCL Tool Call Correctness - ${STATUS}
231
+- **Model:** \`${MODEL}\`
232
+- **Parser:** \`${TOOL_CALL_PARSER}\`
233
+- **API type:** \`${API_TYPE}\`
234
+- **Test category:** \`${TEST_CATEGORY}\`
235
+EOF
236
+
237
+ # BFCL writes results to $BFCL_PROJECT_ROOT/result/ and scores to
238
+ # $BFCL_PROJECT_ROOT/score/
239
+ RESULTS_ROOT="${OUTPUT_DIR:-.}"
240
+ if [ -d "$RESULTS_ROOT/result" ]; then
241
+ buildkite-agent artifact upload "$RESULTS_ROOT/result/**/*"
242
+ fi
243
+ if [ -d "$RESULTS_ROOT/score" ]; then
244
+ buildkite-agent artifact upload "$RESULTS_ROOT/score/**/*"
245
+ fi
246
+fi
247
+
248
+exit $bfcl_exit_code