summaryrefslogtreecommitdiff
path: root/bin/eval-online-model-accuracy.py
diff options
context:
space:
mode:
authorDaniel Friesel <daniel.friesel@uos.de>2020-05-28 12:04:37 +0200
committerDaniel Friesel <daniel.friesel@uos.de>2020-05-28 12:04:37 +0200
commitc69331e4d925658b2bf26dcb387981f6530d7b9e (patch)
treed19c7f9b0bf51f68c104057e013630e009835268 /bin/eval-online-model-accuracy.py
parent23927051ac3e64cabbaa6c30e8356dfe90ebfa6c (diff)
use black(1) for uniform code formatting
Diffstat (limited to 'bin/eval-online-model-accuracy.py')
-rwxr-xr-xbin/eval-online-model-accuracy.py171
1 files changed, 106 insertions, 65 deletions
diff --git a/bin/eval-online-model-accuracy.py b/bin/eval-online-model-accuracy.py
index 21e7a1e..202ac28 100755
--- a/bin/eval-online-model-accuracy.py
+++ b/bin/eval-online-model-accuracy.py
@@ -33,72 +33,74 @@ import numpy as np
opt = dict()
-if __name__ == '__main__':
+if __name__ == "__main__":
try:
optspec = (
- 'accounting= '
- 'arch= '
- 'app= '
- 'depth= '
- 'dummy= '
- 'instance= '
- 'repeat= '
- 'run= '
- 'sleep= '
- 'timer-pin= '
- 'trace-filter= '
- 'timer-freq= '
- 'timer-type= '
- 'timestamp-type= '
- 'energy-type= '
- 'power-type= '
- 'timestamp-granularity= '
- 'energy-granularity= '
- 'power-granularity= '
+ "accounting= "
+ "arch= "
+ "app= "
+ "depth= "
+ "dummy= "
+ "instance= "
+ "repeat= "
+ "run= "
+ "sleep= "
+ "timer-pin= "
+ "trace-filter= "
+ "timer-freq= "
+ "timer-type= "
+ "timestamp-type= "
+ "energy-type= "
+ "power-type= "
+ "timestamp-granularity= "
+ "energy-granularity= "
+ "power-granularity= "
)
- raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' '))
+ raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(" "))
opt_default = {
- 'depth': 3,
- 'sleep': 0,
- 'timer-freq': 1e6,
- 'timer-type': 'uint16_t',
- 'timestamp-type': 'uint16_t',
- 'energy-type': 'uint32_t',
- 'power-type': 'uint16_t',
- 'timestamp-granularity': 1e-6,
- 'power-granularity': 1e-6,
- 'energy-granularity': 1e-12,
+ "depth": 3,
+ "sleep": 0,
+ "timer-freq": 1e6,
+ "timer-type": "uint16_t",
+ "timestamp-type": "uint16_t",
+ "energy-type": "uint32_t",
+ "power-type": "uint16_t",
+ "timestamp-granularity": 1e-6,
+ "power-granularity": 1e-6,
+ "energy-granularity": 1e-12,
}
for option, parameter in raw_opts:
- optname = re.sub(r'^--', '', option)
+ optname = re.sub(r"^--", "", option)
opt[optname] = parameter
- for key in 'depth sleep'.split():
+ for key in "depth sleep".split():
if key in opt:
opt[key] = int(opt[key])
else:
opt[key] = opt_default[key]
- for key in 'timer-freq timestamp-granularity energy-granularity power-granularity'.split():
+ for (
+ key
+ ) in "timer-freq timestamp-granularity energy-granularity power-granularity".split():
if key in opt:
opt[key] = float(opt[key])
else:
opt[key] = opt_default[key]
- for key in 'timer-type timestamp-type energy-type power-type'.split():
+ for key in "timer-type timestamp-type energy-type power-type".split():
if key not in opt:
opt[key] = opt_default[key]
- if 'trace-filter' in opt:
+ if "trace-filter" in opt:
trace_filter = []
- for trace in opt['trace-filter'].split():
- trace_filter.append(trace.split(','))
- opt['trace-filter'] = trace_filter
+ for trace in opt["trace-filter"].split():
+ trace_filter.append(trace.split(","))
+ opt["trace-filter"] = trace_filter
else:
- opt['trace-filter'] = None
+ opt["trace-filter"] = None
except getopt.GetoptError as err:
print(err)
@@ -109,81 +111,120 @@ if __name__ == '__main__':
pta = PTA.from_file(modelfile)
enum = dict()
- if '.json' not in modelfile:
- with open(modelfile, 'r') as f:
+ if ".json" not in modelfile:
+ with open(modelfile, "r") as f:
driver_definition = yaml.safe_load(f)
- if 'dummygen' in driver_definition and 'enum' in driver_definition['dummygen']:
- enum = driver_definition['dummygen']['enum']
+ if "dummygen" in driver_definition and "enum" in driver_definition["dummygen"]:
+ enum = driver_definition["dummygen"]["enum"]
pta.set_random_energy_model()
- runs = list(pta.dfs(opt['depth'], with_arguments=True, with_parameters=True, trace_filter=opt['trace-filter'], sleep=opt['sleep']))
+ runs = list(
+ pta.dfs(
+ opt["depth"],
+ with_arguments=True,
+ with_parameters=True,
+ trace_filter=opt["trace-filter"],
+ sleep=opt["sleep"],
+ )
+ )
num_transitions = len(runs)
if len(runs) == 0:
- print('DFS returned no traces -- perhaps your trace-filter is too restrictive?', file=sys.stderr)
+ print(
+ "DFS returned no traces -- perhaps your trace-filter is too restrictive?",
+ file=sys.stderr,
+ )
sys.exit(1)
real_energies = list()
real_durations = list()
model_energies = list()
for run in runs:
- accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, opt['timer-freq'], opt['timer-type'], opt['timestamp-type'],
- opt['power-type'], opt['energy-type'])
- real_energy, real_duration, _, _ = pta.simulate(run, accounting=accounting_method)
+ accounting_method = get_simulated_accountingmethod(opt["accounting"])(
+ pta,
+ opt["timer-freq"],
+ opt["timer-type"],
+ opt["timestamp-type"],
+ opt["power-type"],
+ opt["energy-type"],
+ )
+ real_energy, real_duration, _, _ = pta.simulate(
+ run, accounting=accounting_method
+ )
model_energy = accounting_method.get_energy()
real_energies.append(real_energy)
real_durations.append(real_duration)
model_energies.append(model_energy)
measures = regression_measures(np.array(model_energies), np.array(real_energies))
- print('SMAPE {:.0f}%, MAE {}'.format(measures['smape'], measures['mae']))
+ print("SMAPE {:.0f}%, MAE {}".format(measures["smape"], measures["mae"]))
timer_freqs = [1e3, 2e3, 5e3, 1e4, 2e4, 5e4, 1e5, 2e5, 5e5, 1e6, 2e6, 5e6]
- timer_types = timestamp_types = power_types = energy_types = 'uint8_t uint16_t uint32_t uint64_t'.split()
+ timer_types = (
+ timestamp_types
+ ) = power_types = energy_types = "uint8_t uint16_t uint32_t uint64_t".split()
def config_weight(timer_freq, timer_type, ts_type, power_type, energy_type):
base_weight = 0
for var_type in timer_type, ts_type, power_type, energy_type:
- if var_type == 'uint8_t':
+ if var_type == "uint8_t":
base_weight += 1
- elif var_type == 'uint16_t':
+ elif var_type == "uint16_t":
base_weight += 2
- elif var_type == 'uint32_t':
+ elif var_type == "uint32_t":
base_weight += 4
- elif var_type == 'uint64_t':
+ elif var_type == "uint64_t":
base_weight += 8
return base_weight
# sys.exit(0)
mean_errors = list()
- for timer_freq, timer_type, ts_type, power_type, energy_type in itertools.product(timer_freqs, timer_types, timestamp_types, power_types, energy_types):
+ for timer_freq, timer_type, ts_type, power_type, energy_type in itertools.product(
+ timer_freqs, timer_types, timestamp_types, power_types, energy_types
+ ):
real_energies = list()
real_durations = list()
model_energies = list()
# duration in µs
# Bei kurzer Dauer (z.B. nur [1e2]) performt auch uint32_t für Energie gut, sonst nicht so (weil overflow)
for sleep_duration in [1e2, 1e3, 1e4, 1e5, 1e6]:
- runs = pta.dfs(opt['depth'], with_arguments=True, with_parameters=True, trace_filter=opt['trace-filter'], sleep=sleep_duration)
+ runs = pta.dfs(
+ opt["depth"],
+ with_arguments=True,
+ with_parameters=True,
+ trace_filter=opt["trace-filter"],
+ sleep=sleep_duration,
+ )
for run in runs:
- accounting_method = get_simulated_accountingmethod(opt['accounting'])(pta, timer_freq, timer_type, ts_type, power_type, energy_type)
- real_energy, real_duration, _, _ = pta.simulate(run, accounting=accounting_method)
+ accounting_method = get_simulated_accountingmethod(opt["accounting"])(
+ pta, timer_freq, timer_type, ts_type, power_type, energy_type
+ )
+ real_energy, real_duration, _, _ = pta.simulate(
+ run, accounting=accounting_method
+ )
model_energy = accounting_method.get_energy()
real_energies.append(real_energy)
real_durations.append(real_duration)
model_energies.append(model_energy)
- measures = regression_measures(np.array(model_energies), np.array(real_energies))
- mean_errors.append(((timer_freq, timer_type, ts_type, power_type, energy_type), config_weight(timer_freq, timer_type, ts_type, power_type, energy_type), measures))
+ measures = regression_measures(
+ np.array(model_energies), np.array(real_energies)
+ )
+ mean_errors.append(
+ (
+ (timer_freq, timer_type, ts_type, power_type, energy_type),
+ config_weight(timer_freq, timer_type, ts_type, power_type, energy_type),
+ measures,
+ )
+ )
mean_errors.sort(key=lambda x: x[1])
- mean_errors.sort(key=lambda x: x[2]['mae'])
+ mean_errors.sort(key=lambda x: x[2]["mae"])
for result in mean_errors:
config, weight, measures = result
- print('{} -> {:.0f}% / {}'.format(
- config,
- measures['smape'], measures['mae']))
+ print("{} -> {:.0f}% / {}".format(config, measures["smape"], measures["mae"]))
sys.exit(0)