evaluate online accounting accuracy for verying variable size

1 files changed, 82 insertions, 13 deletions

diff --git a/bin/eval-online-model-accuracy.py b/bin/eval-online-model-accuracy.py
index 3391d89..8ddae65 100755
--- a/bin/eval-online-model-accuracy.py
+++ b/bin/eval-online-model-accuracy.py
@@ -28,8 +28,8 @@ import runner
 import sys
 import time
 import io
+import itertools
 import yaml
-from aspectc import Repo
 from automata import PTA
 from codegen import *
 from harness import OnboardTimerHarness
@@ -52,22 +52,49 @@ if __name__ == '__main__':
             'sleep= '
             'timer-pin= '
             'trace-filter= '
+            'timer-freq= '
+            'timer-type= '
+            'timestamp-type= '
+            'energy-type= '
+            'power-type= '
+            'timestamp-granularity= '
+            'energy-granularity= '
+            'power-granularity= '
         )
         raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' '))
 
+        opt_default = {
+            'depth' : 3,
+            'sleep' : 0,
+            'timer-freq' : 1e6,
+            'timer-type' : 'uint16_t',
+            'timestamp-type' : 'uint16_t',
+            'energy-type' : 'uint32_t',
+            'power-type' : 'uint16_t',
+            'timestamp-granularity' : 1e-6,
+            'power-granularity' : 1e-6,
+            'energy-granularity' : 1e-12,
+        }
+
         for option, parameter in raw_opts:
             optname = re.sub(r'^--', '', option)
             opt[optname] = parameter
 
-        if 'depth' in opt:
-            opt['depth'] = int(opt['depth'])
-        else:
-            opt['depth'] = 3
+        for key in 'depth sleep'.split():
+            if key in opt:
+                opt[key] = int(opt[key])
+            else:
+                opt[key] = opt_default[key]
 
-        if 'sleep' in opt:
-            opt['sleep'] = int(opt['sleep'])
-        else:
-            opt['sleep'] = 0
+        for key in 'timer-freq timestamp-granularity energy-granularity power-granularity'.split():
+            if key in opt:
+                opt[key] = float(opt[key])
+            else:
+                opt[key] = opt_default[key]
+
+        for key in 'timer-type timestamp-type energy-type power-type'.split():
+            if key not in opt:
+                opt[key] = opt_default[key]
 
         if 'trace-filter' in opt:
             trace_filter = []
@@ -96,8 +123,6 @@ if __name__ == '__main__':
         if 'dummygen' in driver_definition and 'enum' in driver_definition['dummygen']:
             enum = driver_definition['dummygen']['enum']
 
-    repo = Repo('/home/derf/var/projects/multipass/build/repo.acp')
-
     pta.set_random_energy_model()
 
     runs = list(pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = opt['sleep']))
@@ -112,15 +137,59 @@ if __name__ == '__main__':
     real_durations = list()
     model_energies = list()
     for run in runs:
-        accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, 1e6, 'uint32_t', 'uint32_t', 'uint32_t', 'uint32_t')
+        accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, opt['timer-freq'], opt['timer-type'], opt['timestamp-type'],
+            opt['power-type'], opt['energy-type'])
         real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method)
         model_energy = accounting_method.get_energy()
         real_energies.append(real_energy)
         real_durations.append(real_duration)
         model_energies.append(model_energy)
-        print('actual energy {:.0f} µJ, modeled energy {:.0f} µJ'.format(real_energy / 1e6, model_energy / 1e6))
 
     measures = regression_measures(np.array(model_energies), np.array(real_energies))
     print('SMAPE {:.0f}%, MAE {}'.format(measures['smape'], measures['mae']))
 
+
+    timer_freqs = [1e3, 2e3, 5e3, 1e4, 2e4, 5e4, 1e5, 2e5, 5e5, 1e6, 2e6, 5e6]
+    timer_types = timestamp_types = power_types = energy_types = 'uint8_t uint16_t uint32_t uint64_t'.split()
+
+    def config_weight(timer_freq, timer_type, ts_type, power_type, energy_type):
+        base_weight = 0
+        for var_type in timer_type, ts_type, power_type, energy_type:
+            if var_type == 'uint8_t':
+                base_weight += 1
+            elif var_type == 'uint16_t':
+                base_weight += 2
+            elif var_type == 'uint32_t':
+                base_weight += 4
+            elif var_type == 'uint64_t':
+                base_weight += 8
+        return base_weight
+
+    mean_errors = list()
+    for timer_freq, timer_type, ts_type, power_type, energy_type in itertools.product(timer_freqs, timer_types, timestamp_types, power_types, energy_types):
+        real_energies = list()
+        real_durations = list()
+        model_energies = list()
+        # duration in µs
+        for sleep_duration in [1e2, 1e3, 1e4, 1e5, 1e6]:
+            runs = pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = sleep_duration)
+            for run in runs:
+                accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, timer_freq, timer_type, ts_type, power_type, energy_type)
+                real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method)
+                model_energy = accounting_method.get_energy()
+                real_energies.append(real_energy)
+                real_durations.append(real_duration)
+                model_energies.append(model_energy)
+        measures = regression_measures(np.array(model_energies), np.array(real_energies))
+        mean_errors.append(((timer_freq, timer_type, ts_type, power_type, energy_type), config_weight(timer_freq, timer_type, ts_type, power_type, energy_type), measures))
+
+    mean_errors.sort(key = lambda x: x[1])
+    mean_errors.sort(key = lambda x: x[2]['mae'])
+
+    for result in mean_errors:
+        config, weight, measures = result
+        print('{}  -> {:.0f}% / {}'.format(
+            config,
+            measures['smape'], measures['mae']))
+
     sys.exit(0)