diff options
| -rwxr-xr-x | bin/eval-online-model-accuracy.py | 95 | ||||
| -rw-r--r-- | lib/codegen.py | 33 |
2 files changed, 102 insertions, 26 deletions
diff --git a/bin/eval-online-model-accuracy.py b/bin/eval-online-model-accuracy.py index 3391d89..8ddae65 100755 --- a/bin/eval-online-model-accuracy.py +++ b/bin/eval-online-model-accuracy.py @@ -28,8 +28,8 @@ import runner import sys import time import io +import itertools import yaml -from aspectc import Repo from automata import PTA from codegen import * from harness import OnboardTimerHarness @@ -52,22 +52,49 @@ if __name__ == '__main__': 'sleep= ' 'timer-pin= ' 'trace-filter= ' + 'timer-freq= ' + 'timer-type= ' + 'timestamp-type= ' + 'energy-type= ' + 'power-type= ' + 'timestamp-granularity= ' + 'energy-granularity= ' + 'power-granularity= ' ) raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' ')) + opt_default = { + 'depth' : 3, + 'sleep' : 0, + 'timer-freq' : 1e6, + 'timer-type' : 'uint16_t', + 'timestamp-type' : 'uint16_t', + 'energy-type' : 'uint32_t', + 'power-type' : 'uint16_t', + 'timestamp-granularity' : 1e-6, + 'power-granularity' : 1e-6, + 'energy-granularity' : 1e-12, + } + for option, parameter in raw_opts: optname = re.sub(r'^--', '', option) opt[optname] = parameter - if 'depth' in opt: - opt['depth'] = int(opt['depth']) - else: - opt['depth'] = 3 + for key in 'depth sleep'.split(): + if key in opt: + opt[key] = int(opt[key]) + else: + opt[key] = opt_default[key] - if 'sleep' in opt: - opt['sleep'] = int(opt['sleep']) - else: - opt['sleep'] = 0 + for key in 'timer-freq timestamp-granularity energy-granularity power-granularity'.split(): + if key in opt: + opt[key] = float(opt[key]) + else: + opt[key] = opt_default[key] + + for key in 'timer-type timestamp-type energy-type power-type'.split(): + if key not in opt: + opt[key] = opt_default[key] if 'trace-filter' in opt: trace_filter = [] @@ -96,8 +123,6 @@ if __name__ == '__main__': if 'dummygen' in driver_definition and 'enum' in driver_definition['dummygen']: enum = driver_definition['dummygen']['enum'] - repo = Repo('/home/derf/var/projects/multipass/build/repo.acp') - pta.set_random_energy_model() runs = list(pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = opt['sleep'])) @@ -112,15 +137,59 @@ if __name__ == '__main__': real_durations = list() model_energies = list() for run in runs: - accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, 1e6, 'uint32_t', 'uint32_t', 'uint32_t', 'uint32_t') + accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, opt['timer-freq'], opt['timer-type'], opt['timestamp-type'], + opt['power-type'], opt['energy-type']) real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method) model_energy = accounting_method.get_energy() real_energies.append(real_energy) real_durations.append(real_duration) model_energies.append(model_energy) - print('actual energy {:.0f} µJ, modeled energy {:.0f} µJ'.format(real_energy / 1e6, model_energy / 1e6)) measures = regression_measures(np.array(model_energies), np.array(real_energies)) print('SMAPE {:.0f}%, MAE {}'.format(measures['smape'], measures['mae'])) + + timer_freqs = [1e3, 2e3, 5e3, 1e4, 2e4, 5e4, 1e5, 2e5, 5e5, 1e6, 2e6, 5e6] + timer_types = timestamp_types = power_types = energy_types = 'uint8_t uint16_t uint32_t uint64_t'.split() + + def config_weight(timer_freq, timer_type, ts_type, power_type, energy_type): + base_weight = 0 + for var_type in timer_type, ts_type, power_type, energy_type: + if var_type == 'uint8_t': + base_weight += 1 + elif var_type == 'uint16_t': + base_weight += 2 + elif var_type == 'uint32_t': + base_weight += 4 + elif var_type == 'uint64_t': + base_weight += 8 + return base_weight + + mean_errors = list() + for timer_freq, timer_type, ts_type, power_type, energy_type in itertools.product(timer_freqs, timer_types, timestamp_types, power_types, energy_types): + real_energies = list() + real_durations = list() + model_energies = list() + # duration in µs + for sleep_duration in [1e2, 1e3, 1e4, 1e5, 1e6]: + runs = pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = sleep_duration) + for run in runs: + accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, timer_freq, timer_type, ts_type, power_type, energy_type) + real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method) + model_energy = accounting_method.get_energy() + real_energies.append(real_energy) + real_durations.append(real_duration) + model_energies.append(model_energy) + measures = regression_measures(np.array(model_energies), np.array(real_energies)) + mean_errors.append(((timer_freq, timer_type, ts_type, power_type, energy_type), config_weight(timer_freq, timer_type, ts_type, power_type, energy_type), measures)) + + mean_errors.sort(key = lambda x: x[1]) + mean_errors.sort(key = lambda x: x[2]['mae']) + + for result in mean_errors: + config, weight, measures = result + print('{} -> {:.0f}% / {}'.format( + config, + measures['smape'], measures['mae'])) + sys.exit(0) diff --git a/lib/codegen.py b/lib/codegen.py index 1f4bd6f..98eeafe 100644 --- a/lib/codegen.py +++ b/lib/codegen.py @@ -98,7 +98,7 @@ class SimulatedAccountingMethod: * timer counter size (e.g. a 16-bit timer at 1MHz will overflow after 65us) * variable size for accounting of durations, power and energy values """ - def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type): + def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type, ts_granularity = 1e-6, power_granularity = 1e-6, energy_granularity = 1e-12): """ Simulate Online Accounting for a given PTA. @@ -117,14 +117,18 @@ class SimulatedAccountingMethod: self.energy_class = simulate_int_type(energy_type) self.current_state = pta.state['UNINITIALIZED'] + self.ts_granularity = ts_granularity + self.power_granularity = power_granularity + self.energy_granularity = energy_granularity + self.energy = self.energy_class(0) def _sleep_duration(self, duration_us): - """ - Return the sleep duration a timer with the classes timer frequency would measure. + u""" + Return the sleep duration a timer with the configured timer frequency would measure, in µs I.e., for a 35us sleep with a 50kHz timer (-> one tick per 20us), the OS would likely measure one tick == 20us. - This is based on the assumption that the timer is reset at each transition. + This is based on the assumption that the timer is reset at each transition, so the duration of states may be under-, but not over-estimated """ us_per_tick = 1000000 / self.timer_freq_hz ticks = self.timer_class(int(duration_us // us_per_tick)) @@ -137,18 +141,21 @@ class SimulatedAccountingMethod: self.current_state = transition.destination def get_energy(self): - return self.energy.val + """ + Return total energy in pJ + """ + return self.energy.val * self.energy_granularity * 1e12 class SimulatedStaticStateOnlyAccountingImmediateCalculation(SimulatedAccountingMethod): - def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type): - super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type) + def __init__(self, pta: PTA, *args, **kwargs): + super().__init__(pta, *args, **kwargs) def sleep(self, duration_us): self.energy += self.ts_class(self._sleep_duration(duration_us)) * self.power_class(int(self.current_state.power)) class SimulatedStaticAccountingImmediateCalculation(SimulatedAccountingMethod): - def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type): - super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type) + def __init__(self, pta: PTA, *args, **kwargs): + super().__init__(pta, *args, **kwargs) def sleep(self, duration_us): self.energy += self.ts_class(self._sleep_duration(duration_us)) * self.power_class(int(self.current_state.power)) @@ -158,8 +165,8 @@ class SimulatedStaticAccountingImmediateCalculation(SimulatedAccountingMethod): super().pass_transition(transition) class SimulatedStaticAccounting(SimulatedAccountingMethod): - def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type): - super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type) + def __init__(self, pta: PTA, *args, **kwargs): + super().__init__(pta, *args, **kwargs) self.time_in_state = dict() for state_name in pta.state.keys(): self.time_in_state[state_name] = self.ts_class(0) @@ -185,8 +192,8 @@ class SimulatedStaticAccounting(SimulatedAccountingMethod): class SimulatedStaticStateOnlyAccounting(SimulatedAccountingMethod): - def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type): - super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type) + def __init__(self, pta: PTA, *args, **kwargs): + super().__init__(pta, *args, **kwargs) self.time_in_state = dict() for state_name in pta.state.keys(): self.time_in_state[state_name] = self.ts_class(0) |