implement autogeneration and data aggregation of timing benchmarks

1 files changed, 153 insertions, 82 deletions

diff --git a/bin/generate-dfa-benchmark.py b/bin/generate-dfa-benchmark.py
index b9790ed..8e3c227 100755
--- a/bin/generate-dfa-benchmark.py
+++ b/bin/generate-dfa-benchmark.py
@@ -15,6 +15,10 @@ Options:
 --depth=<depth> (default: 3)
     Maximum number of function calls per run
 
+--repeat=<count> (default: 0)
+    Repeat benchmark runs <count> times. When 0, benchmark runs are repeated
+    indefinitely and must be explicitly terminated with Ctrl+C / SIGINT
+
 --instance=<name>
     Override the name of the class instance used for benchmarking
 
@@ -44,6 +48,132 @@ def trace_matches_filter(trace: list, trace_filter: list) -> bool:
             return True
     return False
 
+def benchmark_from_runs(pta: PTA, runs: list, harness: object, benchmark_id: int = 0) -> io.StringIO:
+    outbuf = io.StringIO()
+
+    outbuf.write('#include "arch.h"\n')
+    if 'includes' in pta.codegen:
+        for include in pta.codegen['includes']:
+            outbuf.write('#include "{}"\n'.format(include))
+    outbuf.write(harness.global_code())
+
+    outbuf.write('int main(void)\n')
+    outbuf.write('{\n')
+    for driver in ('arch', 'gpio', 'kout'):
+        outbuf.write('{}.setup();\n'.format(driver))
+    if 'setup' in pta.codegen:
+        for call in pta.codegen['setup']:
+            outbuf.write(call)
+
+    # There is a race condition between flashing the code and starting the UART log.
+    # When starting the log before flashing, output from a previous benchmark may cause bogus data to be added.
+    # When flashing first and then starting the log, the first log lines may be lost.
+    # To work around this, we flash first, then start the log, and use this delay statement to ensure that no output is lost.
+    # This is also useful to faciliate MIMOSA calibration after flashing.
+    outbuf.write('arch.delay_ms(10000);\n')
+
+    outbuf.write('while (1) {\n')
+    outbuf.write(harness.start_benchmark())
+
+    class_prefix = ''
+    if 'instance' in opt:
+        class_prefix = '{}.'.format(opt['instance'])
+    elif 'instance' in pta.codegen:
+        class_prefix = '{}.'.format(pta.codegen['instance'])
+
+    num_transitions = 0
+    num_traces = 0
+    for run in runs:
+        outbuf.write(harness.start_run())
+        harness.start_trace()
+        param = pta.get_initial_param_dict()
+        for transition, arguments, parameter in run:
+            num_transitions += 1
+            harness.append_transition(transition.name, param, arguments)
+            harness.append_state(transition.destination.name, parameter)
+            outbuf.write('// {} -> {}\n'.format(transition.origin.name, transition.destination.name))
+            if transition.is_interrupt:
+                outbuf.write('// wait for {} interrupt\n'.format(transition.name))
+                transition_code = '// TODO add startTransition / stopTransition calls to interrupt routine'
+            else:
+                transition_code = '{}{}({});'.format(class_prefix, transition.name, ', '.join(map(str, arguments)))
+            outbuf.write(harness.pass_transition(pta.get_transition_id(transition), transition_code, transition = transition))
+
+            param = parameter
+
+            outbuf.write('// current parameters: {}\n'.format(', '.join(map(lambda kv: '{}={}'.format(*kv), param.items()))))
+
+            if opt['sleep']:
+                outbuf.write('arch.delay_ms({:d}); // {}\n'.format(opt['sleep'], transition.destination.name))
+
+        outbuf.write(harness.stop_run(num_traces))
+        outbuf.write('\n')
+        num_traces += 1
+
+    outbuf.write(harness.stop_benchmark())
+    outbuf.write('}\n')
+    outbuf.write('return 0;\n')
+    outbuf.write('}\n')
+
+    return outbuf
+
+def run_benchmark(application_file: str, pta: PTA, runs: list, arch: str, app: str, run_args: list, harness: object, sleep: int = 0, repeat: int = 0, run_offset: int = 0, runs_total: int = 0):
+    outbuf = benchmark_from_runs(pta, runs, harness)
+    with open(application_file, 'w') as f:
+        f.write(outbuf.getvalue())
+        print('[MAKE] building benchmark with {:d} runs'.format(len(runs)))
+
+    # assume an average of 10ms per transition. Mind the 10s start delay.
+    run_timeout = 10 + num_transitions * (sleep+10) / 1000
+
+    if repeat:
+        run_timeout *= repeat
+
+    needs_split = False
+    try:
+        runner.build(arch, app, run_args)
+    except RuntimeError:
+        if len(runs) > 50:
+            # Application is too large -> split up runs
+            needs_split = True
+        else:
+            # Unknown error
+            raise
+
+    # This has been deliberately taken out of the except clause to avoid nested exception handlers
+    # (they lead to pretty interesting tracebacks which are probably more confusing than helpful)
+    if needs_split:
+        print('[MAKE] benchmark code is too large, splitting up')
+        mid = len(runs) // 2
+        results = run_benchmark(application_file, pta, runs[:mid], arch, app, run_args, harness, sleep, repeat, run_offset = run_offset, runs_total = runs_total)
+        harness.reset()
+        results.extend(run_benchmark(application_file, pta, runs[mid:], arch, app, run_args, harness, sleep, repeat, run_offset = run_offset + mid, runs_total = runs_total))
+        return results
+
+    runner.flash(arch, app, run_args)
+    monitor = runner.get_monitor(arch, callback = harness.parser_cb)
+
+    if arch == 'posix':
+        print('[RUN] Will run benchmark for {:.0f} seconds'.format(run_timeout))
+        lines = monitor.run(int(run_timeout))
+        return [(runs, harness, lines)]
+
+    # TODO Benchmark bei zu vielen Transitionen in mehrere Programme
+    # aufteilen und diese nacheinander bis zu X % completion (220%)
+    # laufen lassen, zwischendurch jeweils automatisch neu bauen etc.
+    try:
+        slept = 0
+        while repeat == 0 or slept / run_timeout < 1:
+            time.sleep(5)
+            slept += 5
+            print('[RUN] {:d}/{:d} ({:.0f}%), current benchmark at {:.0f}%'.format(run_offset, runs_total, run_offset * 100 / runs_total, slept * 100 / run_timeout))
+    except KeyboardInterrupt:
+        pass
+    lines = monitor.get_lines()
+    monitor.close()
+
+    return [(runs, harness, lines)]
+
 
 if __name__ == '__main__':
 
@@ -53,6 +183,7 @@ if __name__ == '__main__':
             'app= '
             'depth= '
             'instance= '
+            'repeat= '
             'run= '
             'sleep= '
             'timer-pin= '
@@ -69,8 +200,15 @@ if __name__ == '__main__':
         else:
             opt['depth'] = 3
 
+        if 'repeat' in opt:
+            opt['repeat'] = int(opt['repeat'])
+        else:
+            opt['repeat'] = 0
+
         if 'sleep' in opt:
             opt['sleep'] = int(opt['sleep'])
+        else:
+            opt['sleep'] = 0
 
         if 'trace-filter' in opt:
             trace_filter = []
@@ -95,100 +233,33 @@ if __name__ == '__main__':
     else:
         timer_pin = 'GPIO::p1_0'
 
-    harness = OnboardTimerHarness(timer_pin)
-
-    outbuf = io.StringIO()
-
-    outbuf.write('#include "arch.h"\n')
-    if 'includes' in pta.codegen:
-        for include in pta.codegen['includes']:
-            outbuf.write('#include "{}"\n'.format(include))
-    outbuf.write(harness.global_code())
-
-    outbuf.write('int main(void)\n')
-    outbuf.write('{\n')
-    for driver in ('arch', 'gpio', 'kout'):
-        outbuf.write('{}.setup();\n'.format(driver))
-    if 'setup' in pta.codegen:
-        for call in pta.codegen['setup']:
-            outbuf.write(call)
-
-    outbuf.write('while (1) {\n')
-    outbuf.write(harness.start_benchmark())
-
-    class_prefix = ''
-    if 'instance' in opt:
-        class_prefix = '{}.'.format(opt['instance'])
-    elif 'instance' in pta.codegen:
-        class_prefix = '{}.'.format(pta.codegen['instance'])
+    runs = list()
 
-    num_transitions = 0
-    num_traces = 0
     for run in pta.dfs(opt['depth'], with_arguments = True, with_parameters = True):
         if 'trace-filter' in opt and not trace_matches_filter(run, opt['trace-filter']):
             continue
-        outbuf.write(harness.start_run())
-        harness.start_trace()
-        param = pta.get_initial_param_dict()
-        for transition, arguments, parameter in run:
-            num_transitions += 1
-            harness.append_state(transition.origin.name, param)
-            harness.append_transition(transition.name, param, arguments)
-            param = transition.get_params_after_transition(param, arguments)
-            outbuf.write('// {} -> {}\n'.format(transition.origin.name, transition.destination.name))
-            if transition.is_interrupt:
-                outbuf.write('// wait for {} interrupt\n'.format(transition.name))
-                transition_code = '// TODO add startTransition / stopTransition calls to interrupt routine'
-            else:
-                transition_code = '{}{}({});'.format(class_prefix, transition.name, ', '.join(map(str, arguments)))
-            outbuf.write(harness.pass_transition(pta.get_transition_id(transition), transition_code, transition = transition, parameter = parameter))
-
-            param = parameter
-
-            if 'sleep' in opt:
-                outbuf.write('arch.delay_ms({:d});\n'.format(opt['sleep']))
+        runs.append(run)
 
-        outbuf.write(harness.stop_run(num_traces))
-        outbuf.write('\n')
-        num_traces += 1
+    num_transitions = len(runs)
 
-    if num_transitions == 0:
+    if len(runs) == 0:
         print('DFS returned no traces -- perhaps your trace-filter is too restrictive?', file=sys.stderr)
         sys.exit(1)
 
-    outbuf.write(harness.stop_benchmark())
-    outbuf.write('}\n')
-    outbuf.write('return 0;\n')
-    outbuf.write('}\n')
+    harness = OnboardTimerHarness(gpio_pin = timer_pin, pta = pta, counter_limits = runner.get_counter_limits_us(opt['arch']))
 
     if len(args) > 1:
-        with open(args[1], 'w') as f:
-            f.write(outbuf.getvalue())
+        results = run_benchmark(args[1], pta, runs, opt['arch'], opt['app'], opt['run'].split(), harness, opt['sleep'], opt['repeat'], runs_total = len(runs))
+        json_out = {
+            'opt' : opt,
+            'pta' : pta.to_json(),
+            'traces' : harness.traces,
+            'raw_output' : list(map(lambda x: x[2], results)),
+        }
+        with open(time.strftime('ptalog-%Y%m%d-%H%M%S.json'), 'w') as f:
+            json.dump(json_out, f)
     else:
+        outbuf = benchmark_from_runs(pta, runs, harness)
         print(outbuf.getvalue())
 
-    if 'run' in opt:
-        if 'sleep' in opt:
-            run_timeout = num_transitions * opt['sleep'] / 1000
-        else:
-            run_timeout = num_transitions * 10 / 1000
-        monitor = runner.get_monitor(opt['arch'], callback = harness.parser_cb)
-        runner.build(opt['arch'], opt['app'], opt['run'].split())
-        runner.flash(opt['arch'], opt['app'], opt['run'].split())
-        if opt['arch'] != 'posix':
-            try:
-                slept = 0
-                while True:
-                    time.sleep(5)
-                    slept += 5
-                    print('[MON] approx. {:.0f}% done'.format(slept * 100 / run_timeout))
-            except KeyboardInterrupt:
-                pass
-            lines = monitor.get_lines()
-            monitor.close()
-        else:
-            print('[MON] Will run benchmark for {:.0f} seconds'.format(2 * run_timeout))
-            lines = monitor.run(int(2 * run_timeout))
-        print(lines)
-
     sys.exit(0)