1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
|
#!/usr/bin/env python3
"""
Generate a driver/library benchmark based on DFA/PTA traces.
Usage:
PYTHONPATH=lib bin/generate-dfa-benchmark.py [options] <pta/dfa definition> [output.cc]
generate-dfa-benchmarks reads in a DFA definition and generates runs
(i.e., all words accepted by the DFA up to a configurable length). Each symbol
corresponds to a function call. If arguments are specified in the DFA
definition, each symbol corresponds to a function call with a specific set of
arguments (so all argument combinations are present in the generated runs).
Options:
--accounting=static_state|static_state_immediate|static_statetransition|static_statetransition_immedate
Select accounting method for dummy driver generation
--dummy=<class name>
Generate and use a dummy driver for online energy model overhead evaluation
--depth=<depth> (default: 3)
Maximum number of function calls per run
--repeat=<count> (default: 0)
Repeat benchmark runs <count> times. When 0, benchmark runs are repeated
indefinitely and must be explicitly terminated with Ctrl+C / SIGINT
--instance=<name>
Override the name of the class instance used for benchmarking
--mimosa=[k=v,k=v,...]
Perform energy measurements with MIMOSA. Takes precedence over --timing.
mimosa options are key-value pairs. Possible settings with defaults:
offset = 130 (mysterious 0V offset)
shunt = 330 (measurement shunt in ohms)
voltage = 3.3 (VCC provided to DUT)
--sleep=<ms> (default: 0)
How long to sleep between function calls.
--shrink
Decrease amount of parameter values used in state space exploration
(only use minimum and maximum for numeric values)
--timing
Perform timing measurements using on-chip counters (no external hardware
required)
--trace-filter=<transition,transition,transition,...>[ <transition,transition,transition,...> ...]
Only consider traces whose beginning matches one of the provided transition sequences.
E.g. --trace-filter='init,foo init,bar' will only consider traces with init as first and foo or bar as second transition,
and --trace-filter='init,foo,$ init,bar,$' will only consider the traces init -> foo and init -> bar.
"""
import getopt
import json
import os
import re
import runner
import sys
import tarfile
import time
import io
import yaml
from aspectc import Repo
from automata import PTA
from codegen import *
from harness import OnboardTimerHarness, TransitionHarness
from utils import flatten
opt = dict()
def benchmark_from_runs(pta: PTA, runs: list, harness: OnboardTimerHarness, benchmark_id: int = 0, dummy = False, repeat = 0) -> io.StringIO:
outbuf = io.StringIO()
outbuf.write('#include "arch.h"\n')
if dummy:
outbuf.write('#include "driver/dummy.h"\n')
elif 'includes' in pta.codegen:
for include in pta.codegen['includes']:
outbuf.write('#include "{}"\n'.format(include))
outbuf.write(harness.global_code())
outbuf.write('int main(void)\n')
outbuf.write('{\n')
for driver in ('arch', 'gpio', 'kout'):
outbuf.write('{}.setup();\n'.format(driver))
# There is a race condition between flashing the code and starting the UART log.
# When starting the log before flashing, output from a previous benchmark may cause bogus data to be added.
# When flashing first and then starting the log, the first log lines may be lost.
# To work around this, we flash first, then start the log, and use this delay statement to ensure that no output is lost.
# This is also useful to faciliate MIMOSA calibration after flashing
outbuf.write('arch.delay_ms(12000);\n')
# Output some newlines to ensure the parser can determine the start of the first real output line
outbuf.write('kout << endl << endl;\n')
if 'setup' in pta.codegen:
for call in pta.codegen['setup']:
outbuf.write(call)
if repeat:
outbuf.write('unsigned char i = 0;\n')
outbuf.write('while (i++ < {}) {{\n'.format(repeat))
else:
outbuf.write('while (1) {\n')
outbuf.write(harness.start_benchmark())
class_prefix = ''
if 'instance' in opt:
class_prefix = '{}.'.format(opt['instance'])
elif 'instance' in pta.codegen:
class_prefix = '{}.'.format(pta.codegen['instance'])
num_transitions = 0
num_traces = 0
for run in runs:
outbuf.write(harness.start_run())
harness.start_trace()
param = pta.get_initial_param_dict()
for transition, arguments, parameter in run:
num_transitions += 1
harness.append_transition(transition.name, param, arguments)
harness.append_state(transition.destination.name, parameter.copy())
outbuf.write('// {} -> {}\n'.format(transition.origin.name, transition.destination.name))
if transition.is_interrupt:
outbuf.write('// wait for {} interrupt\n'.format(transition.name))
transition_code = '// TODO add startTransition / stopTransition calls to interrupt routine'
else:
transition_code = '{}{}({});'.format(class_prefix, transition.name, ', '.join(map(str, arguments)))
outbuf.write(harness.pass_transition(pta.get_transition_id(transition), transition_code, transition = transition))
param = parameter
outbuf.write('// current parameters: {}\n'.format(', '.join(map(lambda kv: '{}={}'.format(*kv), param.items()))))
if opt['sleep']:
outbuf.write('arch.delay_ms({:d}); // {}\n'.format(opt['sleep'], transition.destination.name))
outbuf.write(harness.stop_run(num_traces))
if dummy:
outbuf.write('kout << "[Energy] " << {}getEnergy() << endl;\n'.format(class_prefix))
outbuf.write('\n')
num_traces += 1
outbuf.write(harness.stop_benchmark())
outbuf.write('}\n')
# Ensure logging can be terminated after the specified number of measurements
outbuf.write(harness.start_benchmark())
outbuf.write('while(1) { }\n')
outbuf.write('return 0;\n')
outbuf.write('}\n')
return outbuf
def run_benchmark(application_file: str, pta: PTA, runs: list, arch: str, app: str, run_args: list, harness: object, sleep: int = 0, repeat: int = 0, run_offset: int = 0, runs_total: int = 0, dummy = False):
if 'mimosa' in opt:
outbuf = benchmark_from_runs(pta, runs, harness, dummy = dummy, repeat = 1)
else:
outbuf = benchmark_from_runs(pta, runs, harness, dummy = dummy, repeat = repeat)
with open(application_file, 'w') as f:
f.write(outbuf.getvalue())
print('[MAKE] building benchmark with {:d} runs'.format(len(runs)))
# assume an average of 10ms per transition. Mind the 10s start delay.
run_timeout = 10 + num_transitions * (sleep+10) / 1000
if repeat:
run_timeout *= repeat
needs_split = False
if len(runs) > 1000:
needs_split = True
else:
try:
runner.build(arch, app, run_args)
except RuntimeError:
if len(runs) > 50:
# Application is too large -> split up runs
needs_split = True
else:
# Unknown error
raise
# This has been deliberately taken out of the except clause to avoid nested exception handlers
# (they lead to pretty interesting tracebacks which are probably more confusing than helpful)
if needs_split:
print('[MAKE] benchmark code is too large, splitting up')
mid = len(runs) // 2
# Previously prepared trace data is useless
harness.reset()
results = run_benchmark(application_file, pta, runs[:mid], arch, app, run_args, harness.copy(), sleep, repeat, run_offset = run_offset, runs_total = runs_total, dummy = dummy)
results.extend(run_benchmark(application_file, pta, runs[mid:], arch, app, run_args, harness.copy(), sleep, repeat, run_offset = run_offset + mid, runs_total = runs_total, dummy = dummy))
return results
if 'mimosa' in opt:
files = list()
i = 0
while i < opt['repeat']:
runner.flash(arch, app, run_args)
monitor = runner.get_monitor(arch, callback = harness.parser_cb, mimosa = opt['mimosa'])
sync_error = False
try:
slept = 0
while not harness.done:
# possible race condition: if the benchmark completes at this
# exact point, it sets harness.done and unsets harness.synced.
# vvv
if slept > 30 and slept < 40 and not harness.synced and not harness.done:
print('[RUN] has been unsynced for more than 30 seconds, assuming error. Retrying.')
sync_error = True
break
time.sleep(5)
slept += 5
print('[RUN] {:d}/{:d} ({:.0f}%), current benchmark at {:.0f}%'.format(run_offset, runs_total, run_offset * 100 / runs_total, slept * 100 / run_timeout))
except KeyboardInterrupt:
pass
monitor.close()
if sync_error:
for filename in monitor.get_files():
os.remove(filename)
else:
files.extend(monitor.get_files())
i += 1
harness.restart()
return [(runs, harness, monitor, files)]
else:
runner.flash(arch, app, run_args)
monitor = runner.get_monitor(arch, callback = harness.parser_cb)
if arch == 'posix':
print('[RUN] Will run benchmark for {:.0f} seconds'.format(run_timeout))
lines = monitor.run(int(run_timeout))
return [(runs, harness, lines, list())]
try:
slept = 0
while not harness.done:
time.sleep(5)
slept += 5
print('[RUN] {:d}/{:d} ({:.0f}%), current benchmark at {:.0f}%'.format(run_offset, runs_total, run_offset * 100 / runs_total, slept * 100 / run_timeout))
except KeyboardInterrupt:
pass
monitor.close()
return [(runs, harness, monitor, list())]
if __name__ == '__main__':
try:
optspec = (
'accounting= '
'arch= '
'app= '
'depth= '
'dummy= '
'instance= '
'mimosa= '
'repeat= '
'run= '
'sleep= '
'shrink '
'timing '
'timer-pin= '
'trace-filter= '
)
raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' '))
for option, parameter in raw_opts:
optname = re.sub(r'^--', '', option)
opt[optname] = parameter
if 'depth' in opt:
opt['depth'] = int(opt['depth'])
else:
opt['depth'] = 3
if 'repeat' in opt:
opt['repeat'] = int(opt['repeat'])
else:
opt['repeat'] = 0
if 'sleep' in opt:
opt['sleep'] = int(opt['sleep'])
else:
opt['sleep'] = 0
if 'trace-filter' in opt:
trace_filter = list()
for trace in opt['trace-filter'].split():
trace_filter.append(trace.split(','))
opt['trace-filter'] = trace_filter
else:
opt['trace-filter'] = None
if 'mimosa' in opt:
if opt['mimosa'] == '':
opt['mimosa'] = dict()
else:
opt['mimosa'] = dict(map(lambda x: x.split('='), opt['mimosa'].split(',')))
opt.pop('timing', None)
if opt['repeat'] == 0:
opt['repeat'] = 1
except getopt.GetoptError as err:
print(err)
sys.exit(2)
modelfile = args[0]
pta = PTA.from_file(modelfile)
if 'shrink' in opt:
pta.shrink_argument_values()
if 'timer-pin' in opt:
timer_pin = opt['timer-pin']
else:
timer_pin = 'GPIO::p1_0'
if 'dummy' in opt:
enum = dict()
if '.json' not in modelfile:
with open(modelfile, 'r') as f:
driver_definition = yaml.safe_load(f)
if 'dummygen' in driver_definition and 'enum' in driver_definition['dummygen']:
enum = driver_definition['dummygen']['enum']
repo = Repo('/home/derf/var/projects/multipass/build/repo.acp')
pta.set_random_energy_model()
if 'accounting' in opt:
accounting_object = get_accountingmethod(opt['accounting'])(opt['dummy'], pta)
else:
accounting_object = None
drv = MultipassDriver(opt['dummy'], pta, repo.class_by_name[opt['dummy']], enum=enum, accounting=accounting_object)
with open('/home/derf/var/projects/multipass/src/driver/dummy.cc', 'w') as f:
f.write(drv.impl)
with open('/home/derf/var/projects/multipass/include/driver/dummy.h', 'w') as f:
f.write(drv.header)
runs = list(pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter']))
num_transitions = len(runs)
if len(runs) == 0:
print('DFS returned no traces -- perhaps your trace-filter is too restrictive?', file=sys.stderr)
sys.exit(1)
need_return_values = False
if next(filter(lambda x: len(x.return_value_handlers), pta.transitions), None):
need_return_values = True
if 'mimosa' in opt:
harness = TransitionHarness(gpio_pin = timer_pin, pta = pta, log_return_values = need_return_values, repeat = 1, post_transition_delay_us = 20)
elif 'timing' in opt:
harness = OnboardTimerHarness(gpio_pin = timer_pin, pta = pta, counter_limits = runner.get_counter_limits_us(opt['arch']), log_return_values = need_return_values, repeat = opt['repeat'])
if len(args) > 1:
results = run_benchmark(args[1], pta, runs, opt['arch'], opt['app'], opt['run'].split(), harness, opt['sleep'], opt['repeat'], runs_total = len(runs), dummy = 'dummy' in opt)
json_out = {
'opt' : opt,
'pta' : pta.to_json(),
'traces' : list(map(lambda x: x[1].traces, results)),
'raw_output' : list(map(lambda x: x[2].get_lines(), results)),
'files' : list(map(lambda x: x[3], results)),
'configs' : list(map(lambda x: x[2].get_config(), results)),
}
extra_files = flatten(json_out['files'])
if 'instance' in pta.codegen:
output_prefix = time.strftime('/home/derf/var/ess/aemr/data/%Y%m%d-%H%M%S-') + pta.codegen['instance']
else:
output_prefix = time.strftime('/home/derf/var/ess/aemr/data/%Y%m%d-%H%M%S-ptalog')
if len(extra_files):
with open('ptalog.json', 'w') as f:
json.dump(json_out, f)
with tarfile.open('{}.tar'.format(output_prefix), 'w') as tar:
tar.add('ptalog.json')
for extra_file in extra_files:
tar.add(extra_file)
print(' --> {}.tar'.format(output_prefix))
os.remove('ptalog.json')
for extra_file in extra_files:
os.remove(extra_file)
else:
with open('{}.json'.format(output_prefix), 'w') as f:
json.dump(json_out, f)
print(' --> {}.json'.format(output_prefix))
else:
outbuf = benchmark_from_runs(pta, runs, harness, repeat = opt['repeat'])
print(outbuf.getvalue())
sys.exit(0)
|