[Rust][Benchmark] Align speed-bench CLI flags with Python and add flag parity test (#51592)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: esmeetu <jasonailu87@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Roy Wang committed
Aug 20, 2026 at 16:50 UTC
963fcfa48c6306cb459655cae0fbaa3a3d1040e7
8 files changed
+406
-16
.buildkite/test_areas/benchmarks.yaml
+1
@@ -9,6 +9,7 @@ steps:
9
source_file_dependencies:
10
- vllm/
11
- "!vllm/distributed/kv_transfer/"
12
+ - rust/src/bench/tests/python_serve_flags.txt
13
- tests/benchmarks/
14
commands:
15
- pytest -v -s benchmarks/
rust/src/bench/README.md
+8
-2
@@ -483,7 +483,7 @@ Run `vllm-bench --help` for the authoritative list. Grouped reference below.
483
| `--model` | Auto-detect | Model name (fetched from `/v1/models` if omitted) |
484
| `--served-model-name` | — | Model name used in API requests |
485
| `--tokenizer` | Same as model | Tokenizer name or path (supports HF, tiktoken, server fallback) |
486
-| `--tokenizer-mode` | `auto` | Tokenizer mode (`auto`, `hf`, `slow`, `mistral`) |
486
+| `--tokenizer-mode` | `auto` | Accepted for Python CLI compatibility but ignored (warns on non-`auto`): resolution always follows the HF `tokenizer.json` → tiktoken → server-side `/tokenize` chain, so `mistral_common` (tekken) tokenizers fall back to server-side tokenization |
487
| `--trust-remote-code` | `false` | Trust remote code for tokenizer |
488
| `--skip-tokenizer-init` | `false` | Skip tokenizer initialization |
489
@@ -521,7 +521,8 @@ Run `vllm-bench --help` for the authoritative list. Grouped reference below.
521
| `--sonnet-output-len` | `150` | Output tokens per request |
522
| `--sonnet-prefix-len` | `200` | Prefix tokens shared across requests |
523
| **SPEED-Bench** | | |
524
-| `--speed-bench-config` | `qualitative` | Split (`qualitative`, `throughput_1k`/`2k`/`8k`/`16k`/`32k`) |
524
+| `--speed-bench-config` | `qualitative` | Split (`qualitative`, `throughput_1k`/`2k`/`8k`/`16k`/`32k`); alias: `--speed-bench-dataset-subset` (Python name) |
525
+| `--speed-bench-output-len` | `4096` | Output tokens per request (matches Python default) |
526
| `--speed-bench-category` | — | Filter by category (`low_entropy`, `high_entropy`, `mixed_entropy`, `coding`, `math`, …) |
527
| `--speed-bench-max-input-len` | — | Truncate prompts to at most N tokens |
528
| **HuggingFace** | | |
@@ -794,6 +795,11 @@ The Rust implementation matches Python `vllm bench serve` in:
795
- Rate control (Gamma distribution, normalization, burstiness, linear/exponential ramp-up)
796
- Metrics (TTFT/TPOT/ITL/E2EL percentiles, peak tokens/sec, peak concurrency, goodput)
797
- Sampling parameters merged into the request body via `extra_body` (same precedence rules)
798
+- CLI flags: a superset of Python's, enforced by `tests/cli_parity.rs` against a
799
+ snapshot of the Python parser (`tests/python_serve_flags.txt`, regenerated by
800
+ `tests/benchmarks/test_rust_bench_cli_parity.py` in the repo root) with an
801
+ explicit allowlist for the Python-only remainder — so `VLLM_USE_RUST_BENCH=1`
802
+ delegation cannot silently reject documented flags
803
804
## Environment Variables
805
rust/src/bench/src/benchmark.rs
+69
-12
@@ -134,6 +134,17 @@ pub(crate) async fn fetch_spec_decode_metrics(
134
Err(_) => return None,
135
};
136
137
+ parse_spec_decode_metrics(&text)
138
+}
139
+
140
+/// Parse spec decode counters from Prometheus text exposition format.
141
+///
142
+/// Matches on the metric name (the part before any labels) rather than the
143
+/// whole line so label values cannot be mistaken for metric names, and only
144
+/// reads `_total` counter samples — skipping e.g. `_created` timestamp series,
145
+/// which would otherwise be summed into the counts (mirrors the Python fix
146
+/// in vllm#41916).
147
+pub(crate) fn parse_spec_decode_metrics(text: &str) -> Option<SpecDecodeMetrics> {
148
let mut num_drafts: u64 = 0;
149
let mut num_draft_tokens: u64 = 0;
150
let mut num_accepted_tokens: u64 = 0;
@@ -145,24 +156,25 @@ pub(crate) async fn fetch_spec_decode_metrics(
156
if line.is_empty() || line.starts_with('#') {
157
continue;
158
}
148
- if !line.starts_with("vllm:spec_decode") {
159
+ let first_token = match line.split_whitespace().next() {
160
+ Some(t) => t,
161
+ None => continue,
162
+ };
163
+ let metric_name = first_token.split('{').next().unwrap_or(first_token);
164
+ if !metric_name.starts_with("vllm:spec_decode") || !metric_name.ends_with("_total") {
165
continue;
166
}
167
found_spec_decode = true;
152
- let parts: Vec<&str> = line.split_whitespace().collect();
153
- if parts.is_empty() {
154
- continue;
155
- }
156
- let val = match parts.last().and_then(|s| s.parse::<f64>().ok()) {
168
+ let val = match line.split_whitespace().last().and_then(|s| s.parse::<f64>().ok()) {
169
Some(v) => v as u64,
170
None => continue,
171
};
172
161
- if line.contains("num_drafts") {
173
+ if metric_name.contains("num_drafts") {
174
num_drafts += val;
163
- } else if line.contains("num_draft_tokens") {
175
+ } else if metric_name.contains("num_draft_tokens") {
176
num_draft_tokens += val;
165
- } else if line.contains("num_accepted_tokens_per_pos") {
177
+ } else if metric_name.contains("num_accepted_tokens_per_pos") {
178
// Parse position label: position="N"
179
if let Some(start) = line.find("position=\"") {
180
let start = start + "position=\"".len();
@@ -172,7 +184,7 @@ pub(crate) async fn fetch_spec_decode_metrics(
184
*accepted_per_pos.entry(pos).or_insert(0) += val;
185
}
186
}
175
- } else if line.contains("num_accepted_tokens") {
187
+ } else if metric_name.contains("num_accepted_tokens") {
188
num_accepted_tokens += val;
189
}
190
}
@@ -502,12 +514,11 @@ pub async fn run_benchmark(config: &BenchConfig) -> Result<serde_json::Value> {
514
downloaded.as_str()
515
}
516
};
505
- let output_len = config.sharegpt_output_len.unwrap_or(config.random_output_len);
517
crate::datasets::speed_bench::load_speed_bench_dataset(
518
tok,
519
path,
520
config.num_prompts,
510
- output_len,
521
+ config.speed_bench_output_len,
522
config.seed,
523
&config.request_id_prefix,
524
config.speed_bench_category.as_deref(),
@@ -1875,6 +1886,52 @@ mod tests {
1886
}
1887
}
1888
1889
+ #[test]
1890
+ fn test_parse_spec_decode_metrics_sums_counters_across_engines() {
1891
+ let text = "\
1892
+# TYPE vllm:spec_decode_num_drafts_total counter
1893
+vllm:spec_decode_num_drafts_total{model_name=\"m\",engine=\"0\"} 100.0
1894
+vllm:spec_decode_num_drafts_total{model_name=\"m\",engine=\"1\"} 50.0
1895
+vllm:spec_decode_num_draft_tokens_total{model_name=\"m\",engine=\"0\"} 700.0
1896
+vllm:spec_decode_num_accepted_tokens_total{model_name=\"m\",engine=\"0\"} 300.0
1897
+vllm:spec_decode_num_accepted_tokens_per_pos_total{position=\"0\",engine=\"0\"} 80.0
1898
+vllm:spec_decode_num_accepted_tokens_per_pos_total{position=\"1\",engine=\"0\"} 40.0
1899
+vllm:num_requests_running{model_name=\"m\"} 0
1900
+";
1901
+ let m = parse_spec_decode_metrics(text).unwrap();
1902
+ assert_eq!(m.num_drafts, 150);
1903
+ assert_eq!(m.num_draft_tokens, 700);
1904
+ assert_eq!(m.num_accepted_tokens, 300);
1905
+ assert_eq!(m.accepted_per_pos, [(0, 80), (1, 40)].into_iter().collect());
1906
+ }
1907
+
1908
+ #[test]
1909
+ fn test_parse_spec_decode_metrics_skips_created_series_and_label_values() {
1910
+ // `_created` timestamps must not be summed into counters, and metric
1911
+ // matching must key off the metric name, not substrings inside label
1912
+ // values (vllm#41916).
1913
+ let text = "\
1914
+vllm:spec_decode_num_drafts_total{model_name=\"m\"} 100.0
1915
+vllm:spec_decode_num_drafts_created{model_name=\"m\"} 1.7863e+09
1916
+vllm:spec_decode_num_draft_tokens_total{model_name=\"m\"} 700.0
1917
+vllm:spec_decode_num_draft_tokens_created{model_name=\"m\"} 1.7863e+09
1918
+vllm:spec_decode_num_accepted_tokens_total{model_name=\"num_drafts\"} 300.0
1919
+";
1920
+ let m = parse_spec_decode_metrics(text).unwrap();
1921
+ assert_eq!(m.num_drafts, 100);
1922
+ assert_eq!(m.num_draft_tokens, 700);
1923
+ assert_eq!(m.num_accepted_tokens, 300);
1924
+ }
1925
+
1926
+ #[test]
1927
+ fn test_parse_spec_decode_metrics_none_without_spec_metrics() {
1928
+ let text = "\
1929
+vllm:num_requests_running{model_name=\"m\"} 0
1930
+vllm:prefix_cache_queries_total{model_name=\"m\"} 0
1931
+";
1932
+ assert!(parse_spec_decode_metrics(text).is_none());
1933
+ }
1934
+
1935
#[test]
1936
fn test_compute_spec_decode_stats_basic() {
1937
let before = make_metrics(100, 300, 200, &[(0, 90), (1, 70), (2, 50)]);
rust/src/bench/src/cli.rs
+11
-2
@@ -176,7 +176,8 @@ pub struct BenchServeArgs {
176
#[arg(long)]
177
pub tokenizer: Option<String>,
178
179
- /// Tokenizer mode (auto, hf, slow, mistral).
179
+ /// Tokenizer mode (auto, hf, slow, mistral). Accepted for Python CLI
180
+ /// compatibility; non-auto values are ignored with a warning.
181
#[arg(long, default_value = "auto")]
182
pub tokenizer_mode: String,
183
@@ -535,9 +536,17 @@ pub struct BenchServeArgs {
536
537
/// SPEED-Bench config/split (qualitative, throughput_1k, throughput_2k, throughput_8k,
538
/// throughput_16k, throughput_32k).
538
- #[arg(long, default_value = "qualitative")]
539
+ #[arg(
540
+ long,
541
+ visible_alias = "speed-bench-dataset-subset",
542
+ default_value = "qualitative"
543
+ )]
544
pub speed_bench_config: SpeedBenchConfig,
545
546
+ /// Number of output tokens per request (SPEED-Bench dataset).
547
+ #[arg(long, default_value_t = 4096)]
548
+ pub speed_bench_output_len: usize,
549
+
550
/// Filter SPEED-Bench by category (e.g. low_entropy, high_entropy, coding, math).
551
#[arg(long)]
552
pub speed_bench_category: Option<String>,
rust/src/bench/src/config.rs
+27
@@ -196,6 +196,7 @@ pub struct BenchConfig {
196
pub speed_bench_config: SpeedBenchConfig,
197
pub speed_bench_category: Option<String>,
198
pub speed_bench_max_input_len: Option<usize>,
199
+ pub speed_bench_output_len: usize,
200
pub hf_split: Option<String>,
201
pub hf_subset: Option<String>,
202
pub hf_output_len: Option<usize>,
@@ -640,6 +641,15 @@ impl BenchConfig {
641
));
642
}
643
644
+ if args.tokenizer_mode != "auto" {
645
+ tracing::warn!(
646
+ mode = %args.tokenizer_mode,
647
+ "--tokenizer-mode is ignored by the Rust client; tokenizer resolution always \
648
+ follows the HF tokenizer.json -> tiktoken -> server-side /tokenize fallback \
649
+ chain (mistral_common tokenizers are not supported locally)"
650
+ );
651
+ }
652
+
653
Ok(BenchConfig {
654
backend: args.backend,
655
base_url,
@@ -727,6 +737,7 @@ impl BenchConfig {
737
speed_bench_config: args.speed_bench_config,
738
speed_bench_category: args.speed_bench_category.clone(),
739
speed_bench_max_input_len: args.speed_bench_max_input_len,
740
+ speed_bench_output_len: args.speed_bench_output_len,
741
hf_split: args.hf_split.clone(),
742
hf_subset: args.hf_subset.clone(),
743
hf_output_len: args.hf_output_len,
@@ -881,6 +892,22 @@ mod tests {
892
]
893
}
894
895
+ #[test]
896
+ fn test_speed_bench_flags_match_python() {
897
+ let args = parse_args(vec![
898
+ "vllm-bench",
899
+ "--model",
900
+ "test-model",
901
+ "--speed-bench-dataset-subset",
902
+ "throughput_8k",
903
+ ]);
904
+ assert!(matches!(
905
+ args.speed_bench_config,
906
+ crate::cli::SpeedBenchConfig::Throughput8k
907
+ ));
908
+ assert_eq!(args.speed_bench_output_len, 4096);
909
+ }
910
+
911
#[test]
912
fn test_prefix_sharing_defaults_to_zero() {
913
let args = base_multi_turn_args();
rust/src/bench/tests/cli_parity.rs
new
+119
@@ -0,0 +1,119 @@
1
+// SPDX-License-Identifier: Apache-2.0
2
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+//! Flag parity with the Python `vllm bench serve` parser.
5
+//!
6
+//! `VLLM_USE_RUST_BENCH=1` makes `vllm bench serve` exec this crate's binary
7
+//! with the raw argv, so every documented Python flag must either parse here
8
+//! or be a deliberate, allowlisted gap. The snapshot is kept current by
9
+//! `tests/benchmarks/test_rust_bench_cli_parity.py` in the repo root.
10
+
11
+use std::collections::HashSet;
12
+
13
+use clap::{CommandFactory, Parser};
14
+use vllm_bench::BenchServeArgs;
15
+
16
+#[derive(Parser)]
17
+struct TestCli {
18
+ #[command(flatten)]
19
+ args: BenchServeArgs,
20
+}
21
+
22
+const PYTHON_FLAGS: &str = include_str!("python_serve_flags.txt");
23
+
24
+/// Python-only flags: `vllm bench serve` features vllm-bench deliberately does
25
+/// not implement. Adding an entry is a decision to let the flag fail under
26
+/// Rust delegation; remove the entry once the flag gains Rust support.
27
+const PYTHON_ONLY: &[&str] = &[
28
+ // asr dataset
29
+ "--asr-max-audio-len-sec",
30
+ "--asr-min-audio-len-sec",
31
+ // bfcl dataset
32
+ "--bfcl-categories",
33
+ // blazedit dataset
34
+ "--blazedit-max-distance",
35
+ "--blazedit-min-distance",
36
+ // spec_bench dataset
37
+ "--spec-bench-category",
38
+ "--spec-bench-output-len",
39
+ // timed-trace dataset
40
+ "--timed-trace-chunk-hash-size",
41
+ "--timed-trace-label-hash-ids",
42
+ "--timed-trace-label-input-length",
43
+ "--timed-trace-label-output-length",
44
+ "--timed-trace-label-timestamp",
45
+ "--timed-trace-sec-multiplier",
46
+ // client-side chat templating / request shaping
47
+ "--chat-template-kwargs",
48
+ "--custom-ensure-client-side-data",
49
+ "--use-beam-search",
50
+ // result post-processing and plotting
51
+ "--plot-dataset-stats",
52
+ "--plot-timeline",
53
+ "--timeline-itl-thresholds",
54
+ // misc python-side controls
55
+ "--hf-name",
56
+ "--no-self-timed",
57
+ "--self-timed",
58
+ "--no-stream",
59
+ "--probe-request-rate",
60
+];
61
+
62
+fn rust_flags() -> HashSet<String> {
63
+ let cmd = TestCli::command();
64
+ let mut flags = HashSet::new();
65
+ for arg in cmd.get_arguments() {
66
+ if let Some(long) = arg.get_long() {
67
+ flags.insert(format!("--{long}"));
68
+ }
69
+ if let Some(aliases) = arg.get_all_aliases() {
70
+ for alias in aliases {
71
+ flags.insert(format!("--{alias}"));
72
+ }
73
+ }
74
+ }
75
+ flags
76
+}
77
+
78
+fn python_flags() -> Vec<&'static str> {
79
+ PYTHON_FLAGS
80
+ .lines()
81
+ .map(str::trim)
82
+ .filter(|line| !line.is_empty() && !line.starts_with('#'))
83
+ .collect()
84
+}
85
+
86
+#[test]
87
+fn python_serve_flags_parse_or_are_allowlisted() {
88
+ let known = rust_flags();
89
+ let missing: Vec<&str> = python_flags()
90
+ .into_iter()
91
+ .filter(|flag| !known.contains(*flag) && !PYTHON_ONLY.contains(flag))
92
+ .collect();
93
+ assert!(
94
+ missing.is_empty(),
95
+ "Python `vllm bench serve` flags unknown to vllm-bench: {missing:?}. \
96
+ Support them in src/cli.rs (a clap alias is enough for renames) or \
97
+ add them to PYTHON_ONLY above as a deliberate gap."
98
+ );
99
+}
100
+
101
+#[test]
102
+fn python_only_allowlist_is_not_stale() {
103
+ let known = rust_flags();
104
+ let stale: Vec<&&str> = PYTHON_ONLY.iter().filter(|f| known.contains(**f)).collect();
105
+ assert!(
106
+ stale.is_empty(),
107
+ "PYTHON_ONLY entries now supported by vllm-bench, remove them: {stale:?}"
108
+ );
109
+}
110
+
111
+#[test]
112
+fn python_only_entries_exist_in_snapshot() {
113
+ let snapshot: HashSet<&str> = python_flags().into_iter().collect();
114
+ let gone: Vec<&&str> = PYTHON_ONLY.iter().filter(|f| !snapshot.contains(**f)).collect();
115
+ assert!(
116
+ gone.is_empty(),
117
+ "PYTHON_ONLY entries no longer exist in the Python parser, remove them: {gone:?}"
118
+ );
119
+}
rust/src/bench/tests/python_serve_flags.txt
new
+110
@@ -0,0 +1,110 @@
1
+# Long CLI flags of the Python `vllm bench serve` parser
2
+# (vllm.benchmarks.serve.add_cli_args), sorted; --help excluded.
3
+# Regenerate: .venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
4
+# Consumed by rust/src/bench/tests/cli_parity.rs, which requires every flag
5
+# below to be a known vllm-bench flag/alias or a PYTHON_ONLY allowlist entry.
6
+--append-result
7
+--asr-max-audio-len-sec
8
+--asr-min-audio-len-sec
9
+--backend
10
+--base-url
11
+--bfcl-categories
12
+--blazedit-max-distance
13
+--blazedit-min-distance
14
+--burstiness
15
+--chat-template-kwargs
16
+--custom-ensure-client-side-data
17
+--custom-output-len
18
+--dataset-name
19
+--dataset-path
20
+--disable-shuffle
21
+--disable-tqdm
22
+--enable-multimodal-chat
23
+--endpoint
24
+--extra-body
25
+--frequency-penalty
26
+--goodput
27
+--header
28
+--hf-name
29
+--hf-output-len
30
+--hf-split
31
+--hf-subset
32
+--host
33
+--ignore-eos
34
+--input-len
35
+--insecure
36
+--label
37
+--logprobs
38
+--lora-assignment
39
+--lora-modules
40
+--max-concurrency
41
+--metadata
42
+--metric-percentiles
43
+--min-p
44
+--model
45
+--no-oversample
46
+--no-reranker
47
+--no-self-timed
48
+--no-stream
49
+--num-prompts
50
+--num-warmups
51
+--output-len
52
+--percentile-metrics
53
+--plot-dataset-stats
54
+--plot-timeline
55
+--port
56
+--prefix-repetition-num-prefixes
57
+--prefix-repetition-output-len
58
+--prefix-repetition-prefix-len
59
+--prefix-repetition-suffix-len
60
+--presence-penalty
61
+--probe-request-rate
62
+--profile
63
+--ramp-up-end-rps
64
+--ramp-up-start-rps
65
+--ramp-up-strategy
66
+--random-batch-size
67
+--random-input-len
68
+--random-mm-base-items-per-request
69
+--random-mm-bucket-config
70
+--random-mm-limit-mm-per-prompt
71
+--random-mm-num-mm-items-range-ratio
72
+--random-output-len
73
+--random-prefix-len
74
+--random-range-ratio
75
+--ready-check-timeout-sec
76
+--repetition-penalty
77
+--request-id-prefix
78
+--request-rate
79
+--result-dir
80
+--result-filename
81
+--save-detailed
82
+--save-result
83
+--seed
84
+--self-timed
85
+--served-model-name
86
+--sharegpt-output-len
87
+--skip-chat-template
88
+--skip-tokenizer-init
89
+--sonnet-input-len
90
+--sonnet-output-len
91
+--sonnet-prefix-len
92
+--spec-bench-category
93
+--spec-bench-output-len
94
+--speed-bench-category
95
+--speed-bench-dataset-subset
96
+--speed-bench-output-len
97
+--temperature
98
+--timed-trace-chunk-hash-size
99
+--timed-trace-label-hash-ids
100
+--timed-trace-label-input-length
101
+--timed-trace-label-output-length
102
+--timed-trace-label-timestamp
103
+--timed-trace-sec-multiplier
104
+--timeline-itl-thresholds
105
+--tokenizer
106
+--tokenizer-mode
107
+--top-k
108
+--top-p
109
+--trust-remote-code
110
+--use-beam-search
tests/benchmarks/test_rust_bench_cli_parity.py
new
+61
@@ -0,0 +1,61 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Keep the `vllm bench serve` flag snapshot in sync for the Rust CLI parity test.
4
+
5
+`rust/src/bench/tests/cli_parity.rs` asserts that every flag in the snapshot is
6
+either a known `vllm-bench` clap flag/alias or explicitly allowlisted as
7
+Python-only, so `VLLM_USE_RUST_BENCH=1` delegation cannot silently reject
8
+documented flags. This test keeps the snapshot itself current.
9
+
10
+Run this file directly to regenerate the snapshot:
11
+
12
+ .venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
13
+"""
14
+
15
+from pathlib import Path
16
+
17
+REPO_ROOT = Path(__file__).resolve().parents[2]
18
+SNAPSHOT = REPO_ROOT / "rust" / "src" / "bench" / "tests" / "python_serve_flags.txt"
19
+HEADER = """\
20
+# Long CLI flags of the Python `vllm bench serve` parser
21
+# (vllm.benchmarks.serve.add_cli_args), sorted; --help excluded.
22
+# Regenerate: .venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py
23
+# Consumed by rust/src/bench/tests/cli_parity.rs, which requires every flag
24
+# below to be a known vllm-bench flag/alias or a PYTHON_ONLY allowlist entry.
25
+"""
26
+
27
+
28
+def _current_flags() -> list[str]:
29
+ from vllm.benchmarks.serve import add_cli_args
30
+ from vllm.utils.argparse_utils import FlexibleArgumentParser
31
+
32
+ parser = FlexibleArgumentParser()
33
+ add_cli_args(parser)
34
+ return sorted(
35
+ {
36
+ opt
37
+ for action in parser._actions
38
+ for opt in action.option_strings
39
+ if opt.startswith("--") and opt != "--help"
40
+ }
41
+ )
42
+
43
+
44
+def _snapshot_flags() -> list[str]:
45
+ lines = SNAPSHOT.read_text().splitlines()
46
+ return [line.strip() for line in lines if line.strip() and not line.startswith("#")]
47
+
48
+
49
+def test_serve_flag_snapshot_is_current():
50
+ assert SNAPSHOT.is_file(), f"missing snapshot {SNAPSHOT}"
51
+ assert _current_flags() == _snapshot_flags(), (
52
+ "`vllm bench serve` flags changed. Regenerate the snapshot with "
53
+ "`.venv/bin/python tests/benchmarks/test_rust_bench_cli_parity.py`, "
54
+ "then for any added flag either support it in rust/src/bench/src/cli.rs "
55
+ "or add it to PYTHON_ONLY in rust/src/bench/tests/cli_parity.rs"
56
+ )
57
+
58
+
59
+if __name__ == "__main__":
60
+ SNAPSHOT.write_text(HEADER + "\n".join(_current_flags()) + "\n")
61
+ print(f"wrote {SNAPSHOT}")