diff options
| -rwxr-xr-x | bin/generate-dfa-benchmark.py | 32 | ||||
| -rw-r--r-- | lib/harness.py | 80 | ||||
| -rw-r--r-- | lib/runner.py | 30 |
3 files changed, 109 insertions, 33 deletions
diff --git a/bin/generate-dfa-benchmark.py b/bin/generate-dfa-benchmark.py index b88cf3b..6ccb1ef 100755 --- a/bin/generate-dfa-benchmark.py +++ b/bin/generate-dfa-benchmark.py @@ -40,16 +40,19 @@ Options: import getopt import json +import os import re import runner import sys +import tarfile import time import io import yaml from aspectc import Repo from automata import PTA from codegen import * -from harness import OnboardTimerHarness +from harness import OnboardTimerHarness, TransitionHarness +from utils import flatten opt = dict() @@ -77,7 +80,7 @@ def benchmark_from_runs(pta: PTA, runs: list, harness: OnboardTimerHarness, benc # When flashing first and then starting the log, the first log lines may be lost. # To work around this, we flash first, then start the log, and use this delay statement to ensure that no output is lost. # This is also useful to faciliate MIMOSA calibration after flashing. - outbuf.write('arch.delay_ms(10000);\n') + outbuf.write('arch.delay_ms(12000);\n') outbuf.write('while (1) {\n') outbuf.write(harness.start_benchmark()) @@ -161,7 +164,7 @@ def run_benchmark(application_file: str, pta: PTA, runs: list, arch: str, app: s return results runner.flash(arch, app, run_args) - monitor = runner.get_monitor(arch, callback = harness.parser_cb) + monitor = runner.get_monitor(arch, callback = harness.parser_cb, mimosa = {'shunt': 82}) if arch == 'posix': print('[RUN] Will run benchmark for {:.0f} seconds'.format(run_timeout)) @@ -176,10 +179,9 @@ def run_benchmark(application_file: str, pta: PTA, runs: list, arch: str, app: s print('[RUN] {:d}/{:d} ({:.0f}%), current benchmark at {:.0f}%'.format(run_offset, runs_total, run_offset * 100 / runs_total, slept * 100 / run_timeout)) except KeyboardInterrupt: pass - lines = monitor.get_lines() monitor.close() - return [(runs, harness, lines)] + return [(runs, harness, monitor)] if __name__ == '__main__': @@ -280,6 +282,7 @@ if __name__ == '__main__': need_return_values = True harness = OnboardTimerHarness(gpio_pin = timer_pin, pta = pta, counter_limits = runner.get_counter_limits_us(opt['arch']), log_return_values = need_return_values) + harness = TransitionHarness(gpio_pin = timer_pin, pta = pta, log_return_values = need_return_values) if len(args) > 1: results = run_benchmark(args[1], pta, runs, opt['arch'], opt['app'], opt['run'].split(), harness, opt['sleep'], opt['repeat'], runs_total = len(runs), dummy = 'dummy' in opt) @@ -287,10 +290,25 @@ if __name__ == '__main__': 'opt' : opt, 'pta' : pta.to_json(), 'traces' : list(map(lambda x: x[1].traces, results)), - 'raw_output' : list(map(lambda x: x[2], results)), + 'raw_output' : list(map(lambda x: x[2].get_lines(), results)), + 'files' : list(map(lambda x: x[2].get_files(), results)), + 'configs' : list(map(lambda x: x[2].get_config(), results)), } - with open(time.strftime('ptalog-%Y%m%d-%H%M%S.json'), 'w') as f: + extra_files = flatten(json_out['files']) + output_prefix = time.strftime('/home/derf/var/ess/aemr/data/ptalog-%Y%m%d-%H%M%S') + with open('{}.json'.format(output_prefix), 'w') as f: json.dump(json_out, f) + if len(extra_files): + with tarfile.open('{}.tar'.format(output_prefix), 'w') as tar: + tar.add('{}.json'.format(output_prefix)) + for extra_file in extra_files: + tar.add(extra_file) + print(' --> {}.tar'.format(output_prefix)) + os.remove('{}.json'.format(output_prefix)) + for extra_file in extra_files: + os.remove(extra_file) + else: + print(' --> {}.json'.format(output_prefix)) else: outbuf = benchmark_from_runs(pta, runs, harness) print(outbuf.getvalue()) diff --git a/lib/harness.py b/lib/harness.py index af74977..76b658c 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -96,25 +96,66 @@ class TransitionHarness: def stop_benchmark(self): return '' + def _append_nondeterministic_parameter_value(self, log_data_target, parameter_name, parameter_value): + if log_data_target['parameter'][parameter_name] is None: + log_data_target['parameter'][parameter_name] = list() + log_data_target['parameter'][parameter_name].append(parameter_value) + def parser_cb(self, line): - pass - - def parse_log(self, lines): - sync = False - for line in lines: - print(line) - res = re.fullmatch(r'\[PTA\] (.*=.*)', line) - if re.fullmatch(r'\[PTA\] benchmark start, id=(.*)', line): - print('> got sync') - sync = True - elif not sync: - continue - elif re.fullmatch(r'\[PTA\] trace, count=(.*)', line): - print('> got transition') - pass - elif res: - print(dict(map(lambda x: x.split('='), res.group(1).split()))) - pass + #print('[HARNESS] got line {}'.format(line)) + if re.match(r'\[PTA\] benchmark start, id=(\S+)', line): + self.synced = True + print('[HARNESS] synced') + if self.synced: + res = re.match(r'\[PTA\] trace=(\S+) count=(\S+)', line) + if res: + self.trace_id = int(res.group(1)) + self.trace_length = int(res.group(2)) + self.current_transition_in_trace = 0 + #print('[HARNESS] trace {:d} contains {:d} transitions. Expecting {:d} transitions.'.format(self.trace_id, self.trace_length, len(self.traces[self.trace_id]['trace']) // 2)) + if self.log_return_values: + res = re.match(r'\[PTA\] transition=(\S+) return=(\S+)', line) + else: + res = re.match(r'\[PTA\] transition=(\S+)', line) + if res: + transition_id = int(res.group(1)) + # self.traces contains transitions and states, UART output only contains transitions -> use index * 2 + try: + log_data_target = self.traces[self.trace_id]['trace'][self.current_transition_in_trace * 2] + except IndexError: + transition_name = None + if self.pta: + transition_name = self.pta.transitions[transition_id].name + print('[HARNESS] benchmark id={:d} trace={:d}: transition #{:d} (ID {:d}, name {}) is out of bounds'.format(0, self.trace_id, self.current_transition_in_trace, transition_id, transition_name)) + print(' Offending line: {}'.format(line)) + return + if log_data_target['isa'] != 'transition': + raise RuntimeError('Log mismatch: Expected transition, got {:s}'.format(log_data_target['isa'])) + if self.pta: + transition = self.pta.transitions[transition_id] + if transition.name != log_data_target['name']: + raise RuntimeError('Log mismatch: Expected transition {:s}, got transition {:s}'.format(log_data_target['name'], transition.name)) + if self.log_return_values and len(transition.return_value_handlers): + for handler in transition.return_value_handlers: + if 'parameter' in handler: + parameter_value = return_value = int(res.group(2)) + + if 'return_values' not in log_data_target: + log_data_target['return_values'] = list() + log_data_target['return_values'].append(return_value) + + if 'formula' in handler: + parameter_value = handler['formula'].eval(return_value) + + self._append_nondeterministic_parameter_value(log_data_target, handler['parameter'], parameter_value) + for following_log_data_target in self.traces[self.trace_id]['trace'][(self.current_transition_in_trace * 2 + 1) :]: + self._append_nondeterministic_parameter_value(following_log_data_target, handler['parameter'], parameter_value) + if 'apply_from' in handler and any(map(lambda x: x['name'] == handler['apply_from'], self.traces[self.trace_id]['trace'][: (self.current_transition_in_trace * 2 + 1)])): + for preceding_log_data_target in reversed(self.traces[self.trace_id]['trace'][: (self.current_transition_in_trace * 2)]): + self._append_nondeterministic_parameter_value(preceding_log_data_target, handler['parameter'], parameter_value) + if preceding_log_data_target['name'] == handler['apply_from']: + break + self.current_transition_in_trace += 1 class OnboardTimerHarness(TransitionHarness): """Bar.""" @@ -179,13 +220,12 @@ class OnboardTimerHarness(TransitionHarness): res = re.match(r'\[PTA\] transition=(\S+) cycles=(\S+)/(\S+)', line) if res: transition_id = int(res.group(1)) - # TODO Handle Overflows (requires knowledge of arch-specific max cycle value) cycles = int(res.group(2)) overflow = int(res.group(3)) if overflow >= self.counter_max_overflow: raise RuntimeError('Counter overflow ({:d}/{:d}) in benchmark id={:d} trace={:d}: transition #{:d} (ID {:d})'.format(cycles, overflow, 0, self.trace_id, self.current_transition_in_trace, transition_id)) duration_us = cycles * self.one_cycle_in_us + overflow * self.one_overflow_in_us - # self.traces contains transitions and states, UART output only contains trnasitions -> use index * 2 + # self.traces contains transitions and states, UART output only contains transitions -> use index * 2 try: log_data_target = self.traces[self.trace_id]['trace'][self.current_transition_in_trace * 2] except IndexError: diff --git a/lib/runner.py b/lib/runner.py index e5c75b8..588bd4d 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -114,9 +114,15 @@ class SerialMonitor: time.sleep(timeout) return self.reader.get_lines() - def get_lines(self) ->list: + def get_lines(self) -> list: return self.reader.get_lines() + def get_files(self) -> list: + return list() + + def get_config(self) -> dict: + return dict() + def close(self): """Close serial connection.""" self.worker.stop() @@ -147,13 +153,12 @@ class MIMOSAMonitor(SerialMonitor): raise RuntimeError('{} returned {}'.format(' '.join(cmd), res.returncode)) def _start_mimosa(self): + self._mimosactl('disconnect') self._mimosacmd(['--start']) self._mimosacmd(['--parameter', 'offset', str(self._offset)]) self._mimosacmd(['--parameter', 'shunt', str(self._shunt)]) self._mimosacmd(['--parameter', 'voltage', str(self._voltage)]) self._mimosacmd(['--mimosa-start']) - time.sleep(1) - self._mimosactl('disconnect') time.sleep(2) self._mimosactl('1k') # 987 ohm time.sleep(2) @@ -166,7 +171,11 @@ class MIMOSAMonitor(SerialMonitor): mtime_changed = True mim_file = None time.sleep(1) - for filename in os.listdir(): + # reverse sort ensures that we will get the latest file, which must + # belong to the current measurements. This ensures that older .mim + # files lying around in the directory will not confuse our + # heuristic. + for filename in sorted(os.listdir(), reverse = True): if re.search(r'[.]mim$', filename): mim_file = filename break @@ -180,8 +189,17 @@ class MIMOSAMonitor(SerialMonitor): def close(self): super().close() - mim_file = self._stop_mimosa() - os.remove(mim_file) + self.mim_file = self._stop_mimosa() + + def get_files(self) -> list: + return [self.mim_file] + + def get_config(self) -> dict: + return { + 'offset' : self._offset, + 'shunt' : self._shunt, + 'voltage' : self._voltage, + } class ShellMonitor: """SerialMonitor runs a program and captures its output for a specific amount of time.""" |