evaluate online accounting accuracy for verying variable size

2 files changed, 102 insertions, 26 deletions

diff --git a/bin/eval-online-model-accuracy.py b/bin/eval-online-model-accuracy.py
index 3391d89..8ddae65 100755
--- a/bin/eval-online-model-accuracy.py
+++ b/bin/eval-online-model-accuracy.py
@@ -28,8 +28,8 @@ import runner
 import sys
 import time
 import io
+import itertools
 import yaml
-from aspectc import Repo
 from automata import PTA
 from codegen import *
 from harness import OnboardTimerHarness
@@ -52,22 +52,49 @@ if __name__ == '__main__':
             'sleep= '
             'timer-pin= '
             'trace-filter= '
+            'timer-freq= '
+            'timer-type= '
+            'timestamp-type= '
+            'energy-type= '
+            'power-type= '
+            'timestamp-granularity= '
+            'energy-granularity= '
+            'power-granularity= '
         )
         raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' '))
 
+        opt_default = {
+            'depth' : 3,
+            'sleep' : 0,
+            'timer-freq' : 1e6,
+            'timer-type' : 'uint16_t',
+            'timestamp-type' : 'uint16_t',
+            'energy-type' : 'uint32_t',
+            'power-type' : 'uint16_t',
+            'timestamp-granularity' : 1e-6,
+            'power-granularity' : 1e-6,
+            'energy-granularity' : 1e-12,
+        }
+
         for option, parameter in raw_opts:
             optname = re.sub(r'^--', '', option)
             opt[optname] = parameter
 
-        if 'depth' in opt:
-            opt['depth'] = int(opt['depth'])
-        else:
-            opt['depth'] = 3
+        for key in 'depth sleep'.split():
+            if key in opt:
+                opt[key] = int(opt[key])
+            else:
+                opt[key] = opt_default[key]
 
-        if 'sleep' in opt:
-            opt['sleep'] = int(opt['sleep'])
-        else:
-            opt['sleep'] = 0
+        for key in 'timer-freq timestamp-granularity energy-granularity power-granularity'.split():
+            if key in opt:
+                opt[key] = float(opt[key])
+            else:
+                opt[key] = opt_default[key]
+
+        for key in 'timer-type timestamp-type energy-type power-type'.split():
+            if key not in opt:
+                opt[key] = opt_default[key]
 
         if 'trace-filter' in opt:
             trace_filter = []
@@ -96,8 +123,6 @@ if __name__ == '__main__':
         if 'dummygen' in driver_definition and 'enum' in driver_definition['dummygen']:
             enum = driver_definition['dummygen']['enum']
 
-    repo = Repo('/home/derf/var/projects/multipass/build/repo.acp')
-
     pta.set_random_energy_model()
 
     runs = list(pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = opt['sleep']))
@@ -112,15 +137,59 @@ if __name__ == '__main__':
     real_durations = list()
     model_energies = list()
     for run in runs:
-        accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, 1e6, 'uint32_t', 'uint32_t', 'uint32_t', 'uint32_t')
+        accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, opt['timer-freq'], opt['timer-type'], opt['timestamp-type'],
+            opt['power-type'], opt['energy-type'])
         real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method)
         model_energy = accounting_method.get_energy()
         real_energies.append(real_energy)
         real_durations.append(real_duration)
         model_energies.append(model_energy)
-        print('actual energy {:.0f} µJ, modeled energy {:.0f} µJ'.format(real_energy / 1e6, model_energy / 1e6))
 
     measures = regression_measures(np.array(model_energies), np.array(real_energies))
     print('SMAPE {:.0f}%, MAE {}'.format(measures['smape'], measures['mae']))
 
+
+    timer_freqs = [1e3, 2e3, 5e3, 1e4, 2e4, 5e4, 1e5, 2e5, 5e5, 1e6, 2e6, 5e6]
+    timer_types = timestamp_types = power_types = energy_types = 'uint8_t uint16_t uint32_t uint64_t'.split()
+
+    def config_weight(timer_freq, timer_type, ts_type, power_type, energy_type):
+        base_weight = 0
+        for var_type in timer_type, ts_type, power_type, energy_type:
+            if var_type == 'uint8_t':
+                base_weight += 1
+            elif var_type == 'uint16_t':
+                base_weight += 2
+            elif var_type == 'uint32_t':
+                base_weight += 4
+            elif var_type == 'uint64_t':
+                base_weight += 8
+        return base_weight
+
+    mean_errors = list()
+    for timer_freq, timer_type, ts_type, power_type, energy_type in itertools.product(timer_freqs, timer_types, timestamp_types, power_types, energy_types):
+        real_energies = list()
+        real_durations = list()
+        model_energies = list()
+        # duration in µs
+        for sleep_duration in [1e2, 1e3, 1e4, 1e5, 1e6]:
+            runs = pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = sleep_duration)
+            for run in runs:
+                accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, timer_freq, timer_type, ts_type, power_type, energy_type)
+                real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method)
+                model_energy = accounting_method.get_energy()
+                real_energies.append(real_energy)
+                real_durations.append(real_duration)
+                model_energies.append(model_energy)
+        measures = regression_measures(np.array(model_energies), np.array(real_energies))
+        mean_errors.append(((timer_freq, timer_type, ts_type, power_type, energy_type), config_weight(timer_freq, timer_type, ts_type, power_type, energy_type), measures))
+
+    mean_errors.sort(key = lambda x: x[1])
+    mean_errors.sort(key = lambda x: x[2]['mae'])
+
+    for result in mean_errors:
+        config, weight, measures = result
+        print('{}  -> {:.0f}% / {}'.format(
+            config,
+            measures['smape'], measures['mae']))
+
     sys.exit(0)
diff --git a/lib/codegen.py b/lib/codegen.py
index 1f4bd6f..98eeafe 100644
--- a/lib/codegen.py
+++ b/lib/codegen.py
@@ -98,7 +98,7 @@ class SimulatedAccountingMethod:
     * timer counter size (e.g. a 16-bit timer at 1MHz will overflow after 65us)
     * variable size for accounting of durations, power and energy values
     """
-    def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type):
+    def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type, ts_granularity = 1e-6, power_granularity = 1e-6, energy_granularity = 1e-12):
         """
         Simulate Online Accounting for a given PTA.
         
@@ -117,14 +117,18 @@ class SimulatedAccountingMethod:
         self.energy_class = simulate_int_type(energy_type)
         self.current_state = pta.state['UNINITIALIZED']
 
+        self.ts_granularity = ts_granularity
+        self.power_granularity = power_granularity
+        self.energy_granularity = energy_granularity
+
         self.energy = self.energy_class(0)
 
     def _sleep_duration(self, duration_us):
-        """
-        Return the sleep duration a timer with the classes timer frequency would measure.
+        u"""
+        Return the sleep duration a timer with the configured timer frequency would measure, in µs
 
         I.e., for a 35us sleep with a 50kHz timer (-> one tick per 20us), the OS would likely measure one tick == 20us.
-        This is based on the assumption that the timer is reset at each transition.
+        This is based on the assumption that the timer is reset at each transition, so the duration of states may be under-, but not over-estimated
         """
         us_per_tick = 1000000 / self.timer_freq_hz
         ticks = self.timer_class(int(duration_us // us_per_tick))
@@ -137,18 +141,21 @@ class SimulatedAccountingMethod:
         self.current_state = transition.destination
 
     def get_energy(self):
-        return self.energy.val
+        """
+        Return total energy in pJ
+        """
+        return self.energy.val * self.energy_granularity * 1e12
 
 class SimulatedStaticStateOnlyAccountingImmediateCalculation(SimulatedAccountingMethod):
-    def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type):
-        super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type)
+    def __init__(self, pta: PTA, *args, **kwargs):
+        super().__init__(pta, *args, **kwargs)
 
     def sleep(self, duration_us):
         self.energy += self.ts_class(self._sleep_duration(duration_us)) * self.power_class(int(self.current_state.power))
 
 class SimulatedStaticAccountingImmediateCalculation(SimulatedAccountingMethod):
-    def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type):
-        super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type)
+    def __init__(self, pta: PTA, *args, **kwargs):
+        super().__init__(pta, *args, **kwargs)
 
     def sleep(self, duration_us):
         self.energy += self.ts_class(self._sleep_duration(duration_us)) * self.power_class(int(self.current_state.power))
@@ -158,8 +165,8 @@ class SimulatedStaticAccountingImmediateCalculation(SimulatedAccountingMethod):
         super().pass_transition(transition)
 
 class SimulatedStaticAccounting(SimulatedAccountingMethod):
-    def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type):
-        super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type)
+    def __init__(self, pta: PTA, *args, **kwargs):
+        super().__init__(pta, *args, **kwargs)
         self.time_in_state = dict()
         for state_name in pta.state.keys():
             self.time_in_state[state_name] = self.ts_class(0)
@@ -185,8 +192,8 @@ class SimulatedStaticAccounting(SimulatedAccountingMethod):
 
 
 class SimulatedStaticStateOnlyAccounting(SimulatedAccountingMethod):
-    def __init__(self, pta: PTA, timer_freq_hz, timer_type, ts_type, power_type, energy_type):
-        super().__init__(pta, timer_freq_hz, timer_type, ts_type, power_type, energy_type)
+    def __init__(self, pta: PTA, *args, **kwargs):
+        super().__init__(pta, *args, **kwargs)
         self.time_in_state = dict()
         for state_name in pta.state.keys():
             self.time_in_state[state_name] = self.ts_class(0)