1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
#!/usr/bin/env python3
"""
Evaluate accuracy of online model for DFA/PTA traces.
Usage:
PYTHONPATH=lib bin/eval-online-model-accuracy.py [options] <pta/dfa definition>
Options:
--accounting=static_state|static_state_immediate|static_statetransition|static_statetransition_immedate
Select accounting method
--depth=<depth> (default: 3)
Maximum number of function calls per run
--sleep=<ms> (default: 0)
How long to sleep between simulated function calls.
--trace-filter=<transition,transition,transition,...>[ <transition,transition,transition,...> ...]
Only consider traces whose beginning matches one of the provided transition sequences.
E.g. --trace-filter='init,foo init,bar' will only consider traces with init as first and foo or bar as second transition,
and --trace-filter='init,foo,$ init,bar,$' will only consider the traces init -> foo and init -> bar.
"""
import getopt
import json
import re
import runner
import sys
import time
import io
import itertools
import yaml
from automata import PTA
from codegen import *
from harness import OnboardTimerHarness
from dfatool import regression_measures
opt = dict()
if __name__ == '__main__':
try:
optspec = (
'accounting= '
'arch= '
'app= '
'depth= '
'dummy= '
'instance= '
'repeat= '
'run= '
'sleep= '
'timer-pin= '
'trace-filter= '
'timer-freq= '
'timer-type= '
'timestamp-type= '
'energy-type= '
'power-type= '
'timestamp-granularity= '
'energy-granularity= '
'power-granularity= '
)
raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' '))
opt_default = {
'depth' : 3,
'sleep' : 0,
'timer-freq' : 1e6,
'timer-type' : 'uint16_t',
'timestamp-type' : 'uint16_t',
'energy-type' : 'uint32_t',
'power-type' : 'uint16_t',
'timestamp-granularity' : 1e-6,
'power-granularity' : 1e-6,
'energy-granularity' : 1e-12,
}
for option, parameter in raw_opts:
optname = re.sub(r'^--', '', option)
opt[optname] = parameter
for key in 'depth sleep'.split():
if key in opt:
opt[key] = int(opt[key])
else:
opt[key] = opt_default[key]
for key in 'timer-freq timestamp-granularity energy-granularity power-granularity'.split():
if key in opt:
opt[key] = float(opt[key])
else:
opt[key] = opt_default[key]
for key in 'timer-type timestamp-type energy-type power-type'.split():
if key not in opt:
opt[key] = opt_default[key]
if 'trace-filter' in opt:
trace_filter = []
for trace in opt['trace-filter'].split():
trace_filter.append(trace.split(','))
opt['trace-filter'] = trace_filter
else:
opt['trace-filter'] = None
except getopt.GetoptError as err:
print(err)
sys.exit(2)
modelfile = args[0]
pta = PTA.from_file(modelfile)
enum = dict()
if '.json' not in modelfile:
with open(modelfile, 'r') as f:
driver_definition = yaml.safe_load(f)
if 'dummygen' in driver_definition and 'enum' in driver_definition['dummygen']:
enum = driver_definition['dummygen']['enum']
pta.set_random_energy_model()
runs = list(pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = opt['sleep']))
num_transitions = len(runs)
if len(runs) == 0:
print('DFS returned no traces -- perhaps your trace-filter is too restrictive?', file=sys.stderr)
sys.exit(1)
real_energies = list()
real_durations = list()
model_energies = list()
for run in runs:
accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, opt['timer-freq'], opt['timer-type'], opt['timestamp-type'],
opt['power-type'], opt['energy-type'])
real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method)
model_energy = accounting_method.get_energy()
real_energies.append(real_energy)
real_durations.append(real_duration)
model_energies.append(model_energy)
measures = regression_measures(np.array(model_energies), np.array(real_energies))
print('SMAPE {:.0f}%, MAE {}'.format(measures['smape'], measures['mae']))
timer_freqs = [1e3, 2e3, 5e3, 1e4, 2e4, 5e4, 1e5, 2e5, 5e5, 1e6, 2e6, 5e6]
timer_types = timestamp_types = power_types = energy_types = 'uint8_t uint16_t uint32_t uint64_t'.split()
def config_weight(timer_freq, timer_type, ts_type, power_type, energy_type):
base_weight = 0
for var_type in timer_type, ts_type, power_type, energy_type:
if var_type == 'uint8_t':
base_weight += 1
elif var_type == 'uint16_t':
base_weight += 2
elif var_type == 'uint32_t':
base_weight += 4
elif var_type == 'uint64_t':
base_weight += 8
return base_weight
#sys.exit(0)
mean_errors = list()
for timer_freq, timer_type, ts_type, power_type, energy_type in itertools.product(timer_freqs, timer_types, timestamp_types, power_types, energy_types):
real_energies = list()
real_durations = list()
model_energies = list()
# duration in µs
# Bei kurzer Dauer (z.B. nur [1e2]) performt auc uint32_t für Energie gut, sonst nicht so (weil overflow)
for sleep_duration in [1e2, 1e3, 1e4, 1e5, 1e6]:
runs = pta.dfs(opt['depth'], with_arguments = True, with_parameters = True, trace_filter = opt['trace-filter'], sleep = sleep_duration)
for run in runs:
accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, timer_freq, timer_type, ts_type, power_type, energy_type)
real_energy, real_duration, _, _ = pta.simulate(run, accounting = accounting_method)
model_energy = accounting_method.get_energy()
real_energies.append(real_energy)
real_durations.append(real_duration)
model_energies.append(model_energy)
measures = regression_measures(np.array(model_energies), np.array(real_energies))
mean_errors.append(((timer_freq, timer_type, ts_type, power_type, energy_type), config_weight(timer_freq, timer_type, ts_type, power_type, energy_type), measures))
mean_errors.sort(key = lambda x: x[1])
mean_errors.sort(key = lambda x: x[2]['mae'])
for result in mean_errors:
config, weight, measures = result
print('{} -> {:.0f}% / {}'.format(
config,
measures['smape'], measures['mae']))
sys.exit(0)
|
