Interleave trials in benchmark script
This should be more of a fair A/B test so the timings aren't affected by having different load on your system when testing the two alternatives.
Ben Alpert committed
May 14, 2016 at 12:04 UTC
c0007d56e9d653d42276dcc10dd1896507346bff
2 files changed
+28
-11
scripts/bench/README.md
+1
-2
@@ -3,8 +3,7 @@ Work-in-progress benchmarks.
3
## Running the suite
4
5
```
6
-$ ./measure.py react-a.min.js >a.txt
7
-$ ./measure.py react-b.min.js >b.txt
6
+$ ./measure.py react-a.min.js a.txt react-b.min.js b.txt
7
$ ./analyze.py a.txt b.txt
8
```
9
scripts/bench/measure.py
+27
-9
@@ -9,12 +9,13 @@
9
import functools
10
import json
11
import os
12
+import random
13
import subprocess
14
import sys
15
16
17
def _run_js_in_jsc(jit, js, env):
17
- return subprocess.check_call(
18
+ return subprocess.check_output(
19
['jsc', '-e', """
20
function now() {
21
return preciseTime() * 1000;
@@ -41,7 +42,7 @@ _run_js_in_jsc_nojit = functools.partial(_run_js_in_jsc, False)
42
43
44
def _run_js_in_node(js, env):
44
- return subprocess.check_call(
45
+ return subprocess.check_output(
46
['node', '-e', """
47
function now() {
48
var hrTime = process.hrtime();
@@ -70,7 +71,7 @@ def _run_js_in_node(js, env):
71
72
73
def _measure_ssr_ms(engine, react_path, bench_name, bench_path, measure_warm):
73
- engine(
74
+ return engine(
75
"""
76
var reactCode = readFile(ENV.react_path);
77
var START = now();
@@ -115,36 +116,53 @@ def _measure_ssr_ms(engine, react_path, bench_name, bench_path, measure_warm):
116
117
118
def _main():
118
- if len(sys.argv) != 2:
119
- sys.stderr.write("usage: measure.py react.min.js >out.txt\n")
119
+ if len(sys.argv) < 2 or len(sys.argv) % 2 == 0:
120
+ sys.stderr.write("usage: measure.py react.min.js out.txt react2.min.js out2.txt\n")
121
return 1
121
- react_path = sys.argv[1]
122
+ # [(react_path, out_path)]
123
+ react_paths = sys.argv[1::2]
124
+ files = [open(out_path, 'w') for out_path in sys.argv[2::2]]
125
126
trials = 30
127
sys.stderr.write("Measuring SSR for PE benchmark (%d trials)\n" % trials)
128
+ sys.stderr.write("_" * trials + "\n")
129
for i in range(trials):
130
for engine in [
131
_run_js_in_jsc_jit,
132
_run_js_in_jsc_nojit,
133
_run_js_in_node
134
]:
131
- _measure_ssr_ms(engine, react_path, 'pe', 'bench-pe-es5.js', False)
135
+ engines = range(len(react_paths))
136
+ random.shuffle(engines)
137
+ for i in engines:
138
+ out = _measure_ssr_ms(engine, react_paths[i], 'pe', 'bench-pe-es5.js', False)
139
+ files[i].write(out)
140
sys.stderr.write(".")
141
sys.stderr.flush()
142
sys.stderr.write("\n")
143
+ sys.stderr.flush()
144
136
- trials = 3
145
+ trials = 0
146
sys.stderr.write("Measuring SSR for PE with warm JIT (%d slow trials)\n" % trials)
147
+ sys.stderr.write("_" * trials + "\n")
148
for i in range(trials):
149
for engine in [
150
_run_js_in_jsc_jit,
151
_run_js_in_jsc_nojit,
152
_run_js_in_node
153
]:
144
- _measure_ssr_ms(engine, react_path, 'pe', 'bench-pe-es5.js', True)
154
+ engines = range(len(react_paths))
155
+ random.shuffle(engines)
156
+ for i in engines:
157
+ out = _measure_ssr_ms(engine, react_paths[i], 'pe', 'bench-pe-es5.js', True)
158
+ files[i].write(out)
159
sys.stderr.write(".")
160
sys.stderr.flush()
161
sys.stderr.write("\n")
162
+ sys.stderr.flush()
163
+
164
+ for f in files:
165
+ f.close()
166
167
168
if __name__ == '__main__':