| 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # Process img-bench test templates |
| 4 | # |
| 5 | # Copyright (c) 2021 Virtuozzo International GmbH. |
| 6 | # |
| 7 | # This program is free software; you can redistribute it and/or modify |
| 8 | # it under the terms of the GNU General Public License as published by |
| 9 | # the Free Software Foundation; either version 2 of the License, or |
| 10 | # (at your option) any later version. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | # GNU General Public License for more details. |
| 16 | # |
| 17 | # You should have received a copy of the GNU General Public License |
| 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | # |
| 20 | |
| 21 | |
| 22 | import sys |
| 23 | import subprocess |
| 24 | import re |
| 25 | import json |
| 26 | |
| 27 | import simplebench |
| 28 | from results_to_text import results_to_text |
| 29 | from table_templater import Templater |
| 30 | |
| 31 | |
| 32 | def bench_func(env, case): |
| 33 | test = templater.gen(env['data'], case['data']) |
| 34 | |
| 35 | p = subprocess.run(test, shell=True, stdout=subprocess.PIPE, |
| 36 | stderr=subprocess.STDOUT, universal_newlines=True) |
| 37 | |
| 38 | if p.returncode == 0: |
| 39 | try: |
| 40 | m = re.search(r'Run completed in (\d+.\d+) seconds.', p.stdout) |
| 41 | return {'seconds': float(m.group(1))} |
| 42 | except Exception: |
| 43 | return {'error': f'failed to parse qemu-img output: {p.stdout}'} |
| 44 | else: |
| 45 | return {'error': f'qemu-img failed: {p.returncode}: {p.stdout}'} |
| 46 | |
| 47 | |
| 48 | if __name__ == '__main__': |
| 49 | if len(sys.argv) > 1: |
| 50 | print(""" |
| 51 | Usage: img_bench_templater.py < path/to/test-template.sh |
| 52 | |
| 53 | This script generates performance tests from a test template (example below), |
| 54 | runs them, and displays the results in a table. The template is read from |
| 55 | stdin. It must be written in bash and end with a `qemu-img bench` invocation |
| 56 | (whose result is parsed to get the test instance’s result). |
| 57 | |
| 58 | Use the following syntax in the template to create the various different test |
| 59 | instances: |
| 60 | |
| 61 | column templating: {var1|var2|...} - test will use different values in |
| 62 | different columns. You may use several {} constructions in the test, in this |
| 63 | case product of all choice-sets will be used. |
| 64 | |
| 65 | row templating: [var1|var2|...] - similar thing to define rows (test-cases) |
| 66 | |
| 67 | Test template example: |
| 68 | |
| 69 | Assume you want to compare two qemu-img binaries, called qemu-img-old and |
| 70 | qemu-img-new in your build directory in two test-cases with 4K writes and 64K |
| 71 | writes. The template may look like this: |
| 72 | |
| 73 | qemu_img=/path/to/qemu/build/qemu-img-{old|new} |
| 74 | $qemu_img create -f qcow2 /ssd/x.qcow2 1G |
| 75 | $qemu_img bench -c 100 -d 8 [-s 4K|-s 64K] -w -t none -n /ssd/x.qcow2 |
| 76 | |
| 77 | When passing this to stdin of img_bench_templater.py, the resulting comparison |
| 78 | table will contain two columns (for two binaries) and two rows (for two |
| 79 | test-cases). |
| 80 | |
| 81 | In addition to displaying the results, script also stores results in JSON |
| 82 | format into results.json file in current directory. |
| 83 | """) |
| 84 | sys.exit() |
| 85 | |
| 86 | templater = Templater(sys.stdin.read()) |
| 87 | |
| 88 | envs = [{'id': ' / '.join(x), 'data': x} for x in templater.columns] |
| 89 | cases = [{'id': ' / '.join(x), 'data': x} for x in templater.rows] |
| 90 | |
| 91 | result = simplebench.bench(bench_func, envs, cases, count=5, |
| 92 | initial_run=False) |
| 93 | print(results_to_text(result)) |
| 94 | with open('results.json', 'w') as f: |
| 95 | json.dump(result, f, indent=4) |