meson: wire up benchmarks
Wire up benchmarks in Meson. The setup is mostly the same as how we wire up our tests. The only difference is that benchmarks get wired up via the `benchmark()` option instead of via `test()`, which gives them a bit of special treatment: - Benchmarks never run in parallel. - Benchmarks aren't run by default when tests are executed. - Meson does not inject the `MALLOC_PERTURB` environment variable. Using benchmarks is quite simple: ``` $ meson setup build # Run all benchmarks. $ meson test -C build --benchmark # Run a specific benchmark. $ meson test -C build --benchmark p0000-* ``` Other than that the usual command line arguments accepted when running tests are also accepted when running benchmarks. Note that the benchmarking target is somewhat limited because it will only run benchmarks for the current build. Other use cases, like running benchmarks against multiple different versions of Git, are not currently supported. Users should continue to use "t/perf/run" for those use cases. The script should get extended at one point in time to support Meson, but this is outside of the scope of this series. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Patrick Steinhardt committed
Apr 28, 2025 at 09:30 UTC
d84eefaeea10cba53fda063f2b68e6c47f250029
3 files changed
+91
-4
meson.build
+12
-4
@@ -70,9 +70,15 @@
70
# # Execute single test interactively such that features like `debug ()` work.
71
# $ meson test -i --test-args='-ix' t1400-update-ref
72
#
73
-# Test execution is parallelized by default and scales with the number of
74
-# processor cores available. You can change the number of processes by passing
75
-# the `-jN` flag to `meson test`.
73
+# # Execute all benchmarks.
74
+# $ meson test -i --benchmark
75
+#
76
+# # Execute single benchmark.
77
+# $ meson test -i --benchmark p0000-*
78
+#
79
+# Test execution (but not benchmark execution) is parallelized by default and
80
+# scales with the number of processor cores available. You can change the
81
+# number of processes by passing the `-jN` flag to `meson test`.
82
#
83
# 4. Install the Git distribution. Again, this can be done via Meson, Ninja or
84
# Samurai:
@@ -235,6 +241,7 @@ git = find_program('git', dirs: program_path, native: true, required: false)
241
sed = find_program('sed', dirs: program_path, native: true)
242
shell = find_program('sh', dirs: program_path, native: true)
243
tar = find_program('tar', dirs: program_path, native: true)
244
+time = find_program('time', dirs: program_path, required: get_option('benchmarks'))
245
246
target_shell = find_program('sh', dirs: program_path, native: false)
247
@@ -836,7 +843,7 @@ endif
843
# features. It is optional if you want to neither execute tests nor use any of
844
# these optional features.
845
perl_required = get_option('perl')
839
-if get_option('gitweb').enabled() or 'netrc' in get_option('credential_helpers') or get_option('docs') != []
846
+if get_option('benchmarks').enabled() or get_option('gitweb').enabled() or 'netrc' in get_option('credential_helpers') or get_option('docs') != []
847
perl_required = true
848
endif
849
@@ -2082,6 +2089,7 @@ meson.add_dist_script(
2089
)
2090
2091
summary({
2092
+ 'benchmarks': get_option('tests') and perl.found() and time.found(),
2093
'curl': curl.found(),
2094
'expat': expat.found(),
2095
'gettext': intl.found(),
meson_options.txt
+2
@@ -101,6 +101,8 @@ option('docs_backend', type: 'combo', choices: ['asciidoc', 'asciidoctor', 'auto
101
description: 'Which backend to use to generate documentation.')
102
103
# Testing.
104
+option('benchmarks', type: 'feature', value: 'auto',
105
+ description: 'Enable benchmarks. This requires Perl and GNU time.')
106
option('coccinelle', type: 'feature', value: 'auto',
107
description: 'Provide a coccicheck target that generates a Coccinelle patch.')
108
option('tests', type: 'boolean', value: true,
t/meson.build
+77
@@ -1097,11 +1097,71 @@ integration_tests = [
1097
't9903-bash-prompt.sh',
1098
]
1099
1100
+benchmarks = [
1101
+ 'perf/p0000-perf-lib-sanity.sh',
1102
+ 'perf/p0001-rev-list.sh',
1103
+ 'perf/p0002-read-cache.sh',
1104
+ 'perf/p0003-delta-base-cache.sh',
1105
+ 'perf/p0004-lazy-init-name-hash.sh',
1106
+ 'perf/p0005-status.sh',
1107
+ 'perf/p0006-read-tree-checkout.sh',
1108
+ 'perf/p0007-write-cache.sh',
1109
+ 'perf/p0008-odb-fsync.sh',
1110
+ 'perf/p0071-sort.sh',
1111
+ 'perf/p0090-cache-tree.sh',
1112
+ 'perf/p0100-globbing.sh',
1113
+ 'perf/p1006-cat-file.sh',
1114
+ 'perf/p1400-update-ref.sh',
1115
+ 'perf/p1450-fsck.sh',
1116
+ 'perf/p1451-fsck-skip-list.sh',
1117
+ 'perf/p1500-graph-walks.sh',
1118
+ 'perf/p2000-sparse-operations.sh',
1119
+ 'perf/p3400-rebase.sh',
1120
+ 'perf/p3404-rebase-interactive.sh',
1121
+ 'perf/p4000-diff-algorithms.sh',
1122
+ 'perf/p4001-diff-no-index.sh',
1123
+ 'perf/p4002-diff-color-moved.sh',
1124
+ 'perf/p4205-log-pretty-formats.sh',
1125
+ 'perf/p4209-pickaxe.sh',
1126
+ 'perf/p4211-line-log.sh',
1127
+ 'perf/p4220-log-grep-engines.sh',
1128
+ 'perf/p4221-log-grep-engines-fixed.sh',
1129
+ 'perf/p5302-pack-index.sh',
1130
+ 'perf/p5303-many-packs.sh',
1131
+ 'perf/p5304-prune.sh',
1132
+ 'perf/p5310-pack-bitmaps.sh',
1133
+ 'perf/p5311-pack-bitmaps-fetch.sh',
1134
+ 'perf/p5312-pack-bitmaps-revs.sh',
1135
+ 'perf/p5313-pack-objects.sh',
1136
+ 'perf/p5314-name-hash.sh',
1137
+ 'perf/p5326-multi-pack-bitmaps.sh',
1138
+ 'perf/p5332-multi-pack-reuse.sh',
1139
+ 'perf/p5333-pseudo-merge-bitmaps.sh',
1140
+ 'perf/p5550-fetch-tags.sh',
1141
+ 'perf/p5551-fetch-rescan.sh',
1142
+ 'perf/p5600-partial-clone.sh',
1143
+ 'perf/p5601-clone-reference.sh',
1144
+ 'perf/p6100-describe.sh',
1145
+ 'perf/p6300-for-each-ref.sh',
1146
+ 'perf/p7000-filter-branch.sh',
1147
+ 'perf/p7102-reset.sh',
1148
+ 'perf/p7300-clean.sh',
1149
+ 'perf/p7519-fsmonitor.sh',
1150
+ 'perf/p7527-builtin-fsmonitor.sh',
1151
+ 'perf/p7810-grep.sh',
1152
+ 'perf/p7820-grep-engines.sh',
1153
+ 'perf/p7821-grep-engines-fixed.sh',
1154
+ 'perf/p7822-grep-perl-character.sh',
1155
+ 'perf/p9210-scalar.sh',
1156
+ 'perf/p9300-fast-import-export.sh',
1157
+]
1158
+
1159
# Sanity check that we are not missing any tests present in 't/'. This check
1160
# only runs once at configure time and is thus best-effort, only. It is
1161
# sufficient to catch missing test suites in our CI though.
1162
foreach glob, tests : {
1163
't[0-9][0-9][0-9][0-9]-*.sh': integration_tests,
1164
+ 'perf/p[0-9][0-9][0-9][0-9]-*.sh': benchmarks,
1165
'unit-tests/t-*.c': unit_test_programs,
1166
'unit-tests/u-*.c': clar_test_suites,
1167
}
@@ -1153,3 +1213,20 @@ foreach integration_test : integration_tests
1213
timeout: 0,
1214
)
1215
endforeach
1216
+
1217
+if perl.found() and time.found()
1218
+ benchmark_environment = test_environment
1219
+ benchmark_environment.set('GTIME', time.full_path())
1220
+
1221
+ foreach benchmark : benchmarks
1222
+ benchmark(fs.stem(benchmark), shell,
1223
+ args: [
1224
+ fs.name(benchmark),
1225
+ ],
1226
+ workdir: meson.current_source_dir() / 'perf',
1227
+ env: benchmark_environment,
1228
+ depends: test_dependencies + bin_wrappers,
1229
+ timeout: 0,
1230
+ )
1231
+ endforeach
1232
+endif