diff options
Diffstat (limited to 'bin/eval-online-model-accuracy.py')
| -rwxr-xr-x | bin/eval-online-model-accuracy.py | 95 |
1 files changed, 82 insertions, 13 deletions
diff --git a/bin/eval-online-model-accuracy.py b/bin/eval-online-model-accuracy.py index 3391d89..8ddae65 100755 --- a/bin/eval-online-model-accuracy.py +++ b/bin/eval-online-model-accuracy.py @@ -28,8 +28,8 @@ import runner import sys import time import io +import itertools import yaml -from aspectc import Repo from automata import PTA from codegen import * from harness import OnboardTimerHarness @@ -52,22 +52,49 @@ if __name__ == '__main__': 'sleep= ' 'timer-pin= ' 'trace-filter= ' + 'timer-freq= ' + 'timer-type= ' + 'timestamp-type= ' + 'energy-type= ' + 'power-type= ' + 'timestamp-granularity= ' + 'energy-granularity= ' + 'power-granularity= ' ) raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' ')) + opt_default = { + 'depth' : 3, + 'sleep' : 0, + 'timer-freq' : 1e6, + 'timer-type' : 'uint16_t', + 'timestamp-type' : 'uint16_t', + 'energy-type' : 'uint32_t', + 'power-type' : 'uint16_t', + 'timestamp-granularity' : 1e-6, + 'power-granularity' : 1e-6, + 'energy-granularity' : 1e-12, + } + for option, parameter in raw_opts: optname = re.sub(r'^--', '', option) opt[optname] = parameter - if 'depth' in opt: - opt['depth'] = int(opt['depth']) - else: - opt['depth'] = 3 + for key in 'depth sleep'.split(): + if key in opt: + opt[key] = int(opt[key]) + else: + opt[key] = opt_default[key] - if 'sleep' in opt: - opt['sleep'] = int(opt['sleep']) - else: - opt['sleep'] = 0 + for key in 'timer-freq timestamp-granularity energy-granularity power-granularity'.split(): + if key in opt: + opt[key] = float(opt[key]) + else: + opt[key] = opt_default[key] + + for key in 'timer-type timestamp-type energy-type power-type'.split(): + if key not in opt: + opt[key] = opt_default[key] if 'trace-filter' in opt: trace_filter = [] @@ -96,8 +123,6 @@ if __name__ == '__main__': if 'dummygen' in driver_definition and 'enum' in driver_definition['dummygen']: enum = driver_definition['dummygen']['enum'] - repo = Repo('/home/derf/var/projects/multipass/build/repo.acp') - pta.set_random_energy_model() runs = list(pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = opt['sleep'])) @@ -112,15 +137,59 @@ if __name__ == '__main__': real_durations = list() model_energies = list() for run in runs: - accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, 1e6, 'uint32_t', 'uint32_t', 'uint32_t', 'uint32_t') + accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, opt['timer-freq'], opt['timer-type'], opt['timestamp-type'], + opt['power-type'], opt['energy-type']) real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method) model_energy = accounting_method.get_energy() real_energies.append(real_energy) real_durations.append(real_duration) model_energies.append(model_energy) - print('actual energy {:.0f} µJ, modeled energy {:.0f} µJ'.format(real_energy / 1e6, model_energy / 1e6)) measures = regression_measures(np.array(model_energies), np.array(real_energies)) print('SMAPE {:.0f}%, MAE {}'.format(measures['smape'], measures['mae'])) + + timer_freqs = [1e3, 2e3, 5e3, 1e4, 2e4, 5e4, 1e5, 2e5, 5e5, 1e6, 2e6, 5e6] + timer_types = timestamp_types = power_types = energy_types = 'uint8_t uint16_t uint32_t uint64_t'.split() + + def config_weight(timer_freq, timer_type, ts_type, power_type, energy_type): + base_weight = 0 + for var_type in timer_type, ts_type, power_type, energy_type: + if var_type == 'uint8_t': + base_weight += 1 + elif var_type == 'uint16_t': + base_weight += 2 + elif var_type == 'uint32_t': + base_weight += 4 + elif var_type == 'uint64_t': + base_weight += 8 + return base_weight + + mean_errors = list() + for timer_freq, timer_type, ts_type, power_type, energy_type in itertools.product(timer_freqs, timer_types, timestamp_types, power_types, energy_types): + real_energies = list() + real_durations = list() + model_energies = list() + # duration in µs + for sleep_duration in [1e2, 1e3, 1e4, 1e5, 1e6]: + runs = pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = sleep_duration) + for run in runs: + accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, timer_freq, timer_type, ts_type, power_type, energy_type) + real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method) + model_energy = accounting_method.get_energy() + real_energies.append(real_energy) + real_durations.append(real_duration) + model_energies.append(model_energy) + measures = regression_measures(np.array(model_energies), np.array(real_energies)) + mean_errors.append(((timer_freq, timer_type, ts_type, power_type, energy_type), config_weight(timer_freq, timer_type, ts_type, power_type, energy_type), measures)) + + mean_errors.sort(key = lambda x: x[1]) + mean_errors.sort(key = lambda x: x[2]['mae']) + + for result in mean_errors: + config, weight, measures = result + print('{} -> {:.0f}% / {}'.format( + config, + measures['smape'], measures['mae'])) + sys.exit(0) |