@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
rs 164 lines 4.86 KB
Raw
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 use std::time::Duration;
5
6 use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main};
7 use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser;
8 use vllm_parser::tool::test_utils::{split_by_chars, test_tools};
9 use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
10
11 mod utils;
12 use utils::{feed_external_parser, feed_parser, openai_tools};
13
14 const CHUNK_CHARS: usize = 7;
15 const LONG_NORMAL_TEXT_REPEATS: usize = 2048;
16 const LONG_TOOL_BODY_REPEATS: usize = 8192;
17
18 fn mixed_fixture() -> String {
19 concat!(
20 "I will check two cities before answering.\n",
21 "<tool_call>\n",
22 "<function=get_weather>\n",
23 "<parameter=location>Hangzhou</parameter>\n",
24 "<parameter=date>2026-04-29</parameter>\n",
25 "<parameter=unit>celsius</parameter>\n",
26 "<parameter=days>3</parameter>\n",
27 "</function>\n",
28 "</tool_call>\n",
29 "<tool_call>\n",
30 "<function=get_weather>\n",
31 "<parameter=location>San Francisco</parameter>\n",
32 "<parameter=date>2026-04-29</parameter>\n",
33 "<parameter=unit>fahrenheit</parameter>\n",
34 "<parameter=days>2</parameter>\n",
35 "</function>\n",
36 "</tool_call>",
37 )
38 .to_string()
39 }
40
41 fn long_normal_text_fixture() -> String {
42 let line = "This is ordinary assistant text with no Qwen Coder tool markers at all.\n";
43 line.repeat(LONG_NORMAL_TEXT_REPEATS)
44 }
45
46 fn long_tool_call_fixture() -> String {
47 let location = "x".repeat(LONG_TOOL_BODY_REPEATS);
48 format!(
49 "<tool_call>\n\
50 <function=get_weather>\n\
51 <parameter=location>{location}</parameter>\n\
52 </function>\n\
53 </tool_call>"
54 )
55 }
56
57 fn native_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
58 Qwen3CoderToolParser::create(tools).expect("Qwen Coder parser should initialize")
59 }
60
61 fn run_stream_group(
62 c: &mut Criterion,
63 name: &str,
64 tools: &[Tool],
65 text: &str,
66 chunk_chars: usize,
67 expected_normal_text: &str,
68 expected_native_calls_len: usize,
69 ) {
70 let chunks = split_by_chars(text, chunk_chars);
71 let openai_tools = openai_tools(tools);
72
73 let mut group = c.benchmark_group(name);
74 group.sample_size(50);
75 group.warm_up_time(Duration::from_millis(300));
76 group.measurement_time(Duration::from_secs(2));
77 group.throughput(Throughput::Bytes(text.len() as u64));
78
79 group.bench_function("native_reuse_parser", |b| {
80 let mut parser = native_parser(tools);
81 b.iter(|| {
82 let result = feed_parser(&mut *parser, black_box(&chunks));
83 debug_assert_eq!(result.0, expected_normal_text);
84 debug_assert_eq!(result.1, expected_native_calls_len);
85 black_box(result);
86 })
87 });
88
89 group.bench_function("native_create_parser", |b| {
90 b.iter_batched(
91 || native_parser(tools),
92 |mut parser| {
93 let result = feed_parser(&mut *parser, black_box(&chunks));
94 debug_assert_eq!(result.0, expected_normal_text);
95 debug_assert_eq!(result.1, expected_native_calls_len);
96 black_box(result);
97 },
98 BatchSize::SmallInput,
99 )
100 });
101
102 group.bench_function("external_reuse_parser", |b| {
103 let mut parser = ExternalQwenCoderParser::new();
104 b.iter(|| {
105 let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
106 debug_assert_eq!(result.0, expected_normal_text);
107 black_box(result);
108 })
109 });
110
111 group.bench_function("external_create_parser", |b| {
112 b.iter_batched(
113 ExternalQwenCoderParser::new,
114 |mut parser| {
115 let result = feed_external_parser(&mut parser, &openai_tools, black_box(&chunks));
116 debug_assert_eq!(result.0, expected_normal_text);
117 black_box(result);
118 },
119 BatchSize::SmallInput,
120 )
121 });
122
123 group.finish();
124 }
125
126 fn bench_qwen3_coder(c: &mut Criterion) {
127 let tools = test_tools();
128 let mixed_text = mixed_fixture();
129 let long_normal_text = long_normal_text_fixture();
130 let long_tool_call = long_tool_call_fixture();
131
132 run_stream_group(
133 c,
134 "qwen3_coder/mixed_text_tool_call",
135 &tools,
136 &mixed_text,
137 CHUNK_CHARS,
138 "I will check two cities before answering.\n",
139 2,
140 );
141
142 run_stream_group(
143 c,
144 "qwen3_coder/long_normal_text",
145 &tools,
146 &long_normal_text,
147 CHUNK_CHARS,
148 &long_normal_text,
149 0,
150 );
151
152 run_stream_group(
153 c,
154 "qwen3_coder/long_tool_call_body",
155 &tools,
156 &long_tool_call,
157 CHUNK_CHARS,
158 "",
159 1,
160 );
161 }
162
163 criterion_group!(benches, bench_qwen3_coder);
164 criterion_main!(benches);