diff options
| -rwxr-xr-x | bin/analyze-archive.py | 7 | ||||
| -rwxr-xr-x | bin/eval-outlier-removal.py | 152 | ||||
| -rwxr-xr-x | bin/eval-rel-energy.py | 103 | ||||
| -rwxr-xr-x | bin/test.py | 21 | ||||
| -rwxr-xr-x | lib/dfatool.py | 140 |
5 files changed, 318 insertions, 105 deletions
diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index 307cbd2..276daf5 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -5,7 +5,7 @@ import json import plotter import re import sys -from dfatool import PTAModel, RawData +from dfatool import PTAModel, RawData, pta_trace_to_aggregate from dfatool import soft_cast_int, is_numeric, gplearn_to_function opts = {} @@ -147,8 +147,9 @@ if __name__ == '__main__': raw_data = RawData(args) preprocessed_data = raw_data.get_preprocessed_data() - model = PTAModel(preprocessed_data, - ignore_trace_indexes = ignored_trace_indexes, + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data, ignored_trace_indexes) + model = PTAModel(by_name, parameters, arg_count, + traces = preprocessed_data, discard_outliers = discard_outliers, function_override = function_override, hwmodel = hwmodel) diff --git a/bin/eval-outlier-removal.py b/bin/eval-outlier-removal.py new file mode 100755 index 0000000..b6e8733 --- /dev/null +++ b/bin/eval-outlier-removal.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +import getopt +import plotter +import re +import sys +from dfatool import PTAModel, RawData, soft_cast_int, pta_trace_to_aggregate + +opts = {} + +def model_quality_table(result_lists, info_list): + for state_or_tran in result_lists[0].keys(): + for key in result_lists[0][state_or_tran].keys(): + buf = '{:20s} {:15s}'.format(state_or_tran, key) + for i, results in enumerate(result_lists): + info = info_list[i] + buf += ' ||| ' + if info == None or info(state_or_tran, key): + result = results[state_or_tran][key] + if 'smape' in result: + buf += '{:6.2f}% / {:9.0f}'.format(result['smape'], result['mae']) + else: + buf += '{:6} {:9.0f}'.format('', result['mae']) + else: + buf += '{:6}----{:9}'.format('', '') + print(buf) + +def combo_model_quality_table(result_lists, info_list): + for state_or_tran in result_lists[0][0].keys(): + for key in result_lists[0][0][state_or_tran].keys(): + for sub_result_lists in result_lists: + buf = '{:20s} {:15s}'.format(state_or_tran, key) + for i, results in enumerate(sub_result_lists): + info = info_list[i] + buf += ' ||| ' + if info == None or info(state_or_tran, key): + result = results[state_or_tran][key] + if 'smape' in result: + buf += '{:6.2f}% / {:9.0f}'.format(result['smape'], result['mae']) + else: + buf += '{:6} {:9.0f}'.format('', result['mae']) + else: + buf += '{:6}----{:9}'.format('', '') + print(buf) + +if __name__ == '__main__': + + ignored_trace_indexes = [] + discard_outliers = None + + try: + raw_opts, args = getopt.getopt(sys.argv[1:], "", + 'plot ignored-trace-indexes= discard-outliers='.split(' ')) + + for option, parameter in raw_opts: + optname = re.sub(r'^--', '', option) + opts[optname] = parameter + + if 'ignored-trace-indexes' in opts: + ignored_trace_indexes = list(map(int, opts['ignored-trace-indexes'].split(','))) + if 0 in ignored_trace_indexes: + print('[E] arguments to --ignored-trace-indexes start from 1') + + if 'discard-outliers' in opts: + discard_outliers = float(opts['discard-outliers']) + + except getopt.GetoptError as err: + print(err) + sys.exit(2) + + raw_data = RawData(args) + + preprocessed_data = raw_data.get_preprocessed_data() + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data, ignored_trace_indexes) + m1 = PTAModel(by_name, parameters, arg_count, + traces = preprocessed_data, + ignore_trace_indexes = ignored_trace_indexes) + m2 = PTAModel(by_name, parameters, arg_count, + traces = preprocessed_data, + ignore_trace_indexes = ignored_trace_indexes, + discard_outliers = discard_outliers) + + print('--- simple static model ---') + static_m1 = m1.get_static() + static_m2 = m2.get_static() + #for state in model.states(): + # print('{:10s}: {:.0f} µW ({:.2f})'.format( + # state, + # static_model(state, 'power'), + # model.generic_param_dependence_ratio(state, 'power'))) + # for param in model.parameters(): + # print('{:10s} dependence on {:15s}: {:.2f}'.format( + # '', + # param, + # model.param_dependence_ratio(state, 'power', param))) + #for trans in model.transitions(): + # print('{:10s}: {:.0f} / {:.0f} / {:.0f} pJ ({:.2f} / {:.2f} / {:.2f})'.format( + # trans, static_model(trans, 'energy'), + # static_model(trans, 'rel_energy_prev'), + # static_model(trans, 'rel_energy_next'), + # model.generic_param_dependence_ratio(trans, 'energy'), + # model.generic_param_dependence_ratio(trans, 'rel_energy_prev'), + # model.generic_param_dependence_ratio(trans, 'rel_energy_next'))) + # print('{:10s}: {:.0f} µs'.format(trans, static_model(trans, 'duration'))) + static_q1 = m1.assess(static_m1) + static_q2 = m2.assess(static_m2) + static_q12 = m1.assess(static_m2) + + print('--- LUT ---') + lut_m1 = m1.get_param_lut() + lut_m2 = m2.get_param_lut() + lut_q1 = m1.assess(lut_m1) + lut_q2 = m2.assess(lut_m2) + lut_q12 = m1.assess(lut_m2) + + print('--- param model ---') + param_m1, param_i1 = m1.get_fitted() + for state in m1.states(): + for attribute in ['power']: + if param_i1(state, attribute): + print('{:10s}: {}'.format(state, param_i1(state, attribute)['function']._model_str)) + print('{:10s} {}'.format('', param_i1(state, attribute)['function']._regression_args)) + for trans in m1.transitions(): + for attribute in ['energy', 'rel_energy_prev', 'rel_energy_next', 'duration', 'timeout']: + if param_i1(trans, attribute): + print('{:10s}: {:10s}: {}'.format(trans, attribute, param_i1(trans, attribute)['function']._model_str)) + print('{:10s} {:10s} {}'.format('', '', param_i1(trans, attribute)['function']._regression_args)) + param_m2, param_i2 = m2.get_fitted() + for state in m2.states(): + for attribute in ['power']: + if param_i2(state, attribute): + print('{:10s}: {}'.format(state, param_i2(state, attribute)['function']._model_str)) + print('{:10s} {}'.format('', param_i2(state, attribute)['function']._regression_args)) + for trans in m2.transitions(): + for attribute in ['energy', 'rel_energy_prev', 'rel_energy_next', 'duration', 'timeout']: + if param_i2(trans, attribute): + print('{:10s}: {:10s}: {}'.format(trans, attribute, param_i2(trans, attribute)['function']._model_str)) + print('{:10s} {:10s} {}'.format('', '', param_i2(trans, attribute)['function']._regression_args)) + + analytic_q1 = m1.assess(param_m1) + analytic_q2 = m2.assess(param_m2) + analytic_q12 = m1.assess(param_m2) + model_quality_table([static_q1, analytic_q1, lut_q1], [None, param_i1, None]) + model_quality_table([static_q2, analytic_q2, lut_q2], [None, param_i2, None]) + model_quality_table([static_q12, analytic_q12, lut_q12], [None, param_i2, None]) + combo_model_quality_table([ + [static_q1, analytic_q1, lut_q1], + [static_q2, analytic_q2, lut_q2], + [static_q12, analytic_q12, lut_q12]], + [None, param_i1, None]) + + sys.exit(0) diff --git a/bin/eval-rel-energy.py b/bin/eval-rel-energy.py new file mode 100755 index 0000000..ea7a226 --- /dev/null +++ b/bin/eval-rel-energy.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +import getopt +import re +import sys +from dfatool import PTAModel, RawData, pta_trace_to_aggregate + +opts = {} + +def get_file_groups(args): + groups = [] + index_low = 0 + while ':' in args[index_low : ]: + index_high = args[index_low : ].index(':') + index_low + groups.append(args[index_low : index_high]) + index_low = index_high + 1 + groups.append(args[index_low : ]) + return groups + +if __name__ == '__main__': + + ignored_trace_indexes = [] + discard_outliers = None + safe_functions_enabled = False + function_override = {} + show_models = [] + show_quality = [] + + try: + optspec = ( + 'plot-unparam= plot-param= show-models= show-quality= ' + 'ignored-trace-indexes= discard-outliers= function-override= ' + 'with-safe-functions' + ) + raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' ')) + + for option, parameter in raw_opts: + optname = re.sub(r'^--', '', option) + opts[optname] = parameter + + if 'ignored-trace-indexes' in opts: + ignored_trace_indexes = list(map(int, opts['ignored-trace-indexes'].split(','))) + if 0 in ignored_trace_indexes: + print('[E] arguments to --ignored-trace-indexes start from 1') + + if 'discard-outliers' in opts: + discard_outliers = float(opts['discard-outliers']) + + if 'function-override' in opts: + for function_desc in opts['function-override'].split(';'): + state_or_tran, attribute, *function_str = function_desc.split(' ') + function_override[(state_or_tran, attribute)] = ' '.join(function_str) + + if 'show-models' in opts: + show_models = opts['show-models'].split(',') + + if 'show-quality' in opts: + show_quality = opts['show-quality'].split(',') + + if 'with-safe-functions' in opts: + safe_functions_enabled = True + + except getopt.GetoptError as err: + print(err) + sys.exit(2) + + score_absolute = 0 + score_relative = 0 + + for file_group in get_file_groups(args): + print('') + print('{}:'.format(' '.join(file_group))) + raw_data = RawData(file_group) + + preprocessed_data = raw_data.get_preprocessed_data(verbose = False) + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data, ignored_trace_indexes) + model = PTAModel(by_name, parameters, arg_count, + traces = preprocessed_data, + ignore_trace_indexes = ignored_trace_indexes, + discard_outliers = discard_outliers, + function_override = function_override, + verbose = False) + + lut_quality = model.assess(model.get_param_lut()) + + for trans in model.transitions(): + absolute_quality = lut_quality['by_dfa_component'][trans]['energy'] + relative_quality = lut_quality['by_dfa_component'][trans]['rel_energy_prev'] + if absolute_quality['mae'] < relative_quality['mae']: + best = 'absolute' + score_absolute += 1 + else: + best = 'relative' + score_relative += 1 + + print('{:20s}: {:s} (diff {:.0f} / {:.2f}%, abs {:.0f} / {:.2f}%, rel {:.0f} / {:.2f}%)'.format( + trans, best, + abs(absolute_quality['mae'] - relative_quality['mae']), + abs(absolute_quality['mae'] - relative_quality['mae']) * 100 / max(absolute_quality['mae'], relative_quality['mae']), + absolute_quality['mae'], absolute_quality['smape'], + relative_quality['mae'], relative_quality['smape'])) + + sys.exit(0) diff --git a/bin/test.py b/bin/test.py index ae72df2..f53a9ef 100755 --- a/bin/test.py +++ b/bin/test.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -from dfatool import PTAModel, RawData, analytic +from dfatool import PTAModel, RawData, pta_trace_to_aggregate import unittest class TestStaticModel(unittest.TestCase): def test_model_singlefile_rf24(self): raw_data = RawData(['../data/20170220_164723_RF24_int_A.tar']) preprocessed_data = raw_data.get_preprocessed_data(verbose = False) - model = PTAModel(preprocessed_data, verbose = False) + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data) + model = PTAModel(by_name, parameters, arg_count, verbose = False) self.assertEqual(model.states(), 'POWERDOWN RX STANDBY1 TX'.split(' ')) self.assertEqual(model.transitions(), 'begin epilogue powerDown powerUp setDataRate_num setPALevel_num startListening stopListening write_nb'.split(' ')) static_model = model.get_static() @@ -72,7 +73,8 @@ class TestStaticModel(unittest.TestCase): def test_model_singlefile_mmparam(self): raw_data = RawData(['../data/20161221_123347_mmparam.tar']) preprocessed_data = raw_data.get_preprocessed_data(verbose = False) - model = PTAModel(preprocessed_data, verbose = False) + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data) + model = PTAModel(by_name, parameters, arg_count, verbose = False) self.assertEqual(model.states(), 'OFF ON'.split(' ')) self.assertEqual(model.transitions(), 'off setBrightness'.split(' ')) static_model = model.get_static() @@ -100,7 +102,8 @@ class TestStaticModel(unittest.TestCase): ] raw_data = RawData(testfiles) preprocessed_data = raw_data.get_preprocessed_data(verbose = False) - model = PTAModel(preprocessed_data, verbose = False) + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data) + model = PTAModel(by_name, parameters, arg_count, verbose = False) self.assertEqual(model.states(), 'ACTIVE POWEROFF'.split(' ')) self.assertEqual(model.transitions(), 'getTemp setHyst setOS shutdown start'.split(' ')) static_model = model.get_static() @@ -129,7 +132,8 @@ class TestStaticModel(unittest.TestCase): ] raw_data = RawData(testfiles) preprocessed_data = raw_data.get_preprocessed_data(verbose = False) - model = PTAModel(preprocessed_data, verbose = False) + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data) + model = PTAModel(by_name, parameters, arg_count, verbose = False) self.assertEqual(model.states(), 'DISABLED ENABLED'.split(' ')) self.assertEqual(model.transitions(), 'clear disable enable ioInit sendLine toggleVCOM'.split(' ')) static_model = model.get_static() @@ -161,7 +165,8 @@ class TestStaticModel(unittest.TestCase): ] raw_data = RawData(testfiles) preprocessed_data = raw_data.get_preprocessed_data(verbose = False) - model = PTAModel(preprocessed_data, verbose = False) + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data) + model = PTAModel(by_name, parameters, arg_count, verbose = False) self.assertEqual(model.states(), 'B G OFF R'.split(' ')) self.assertEqual(model.transitions(), 'blue green off red'.split(' ')) static_model = model.get_static() @@ -193,7 +198,8 @@ class TestStaticModel(unittest.TestCase): ] raw_data = RawData(testfiles) preprocessed_data = raw_data.get_preprocessed_data(verbose = False) - model = PTAModel(preprocessed_data, verbose = False) + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data) + model = PTAModel(by_name, parameters, arg_count, verbose = False) self.assertEqual(model.states(), 'IDLE RX SLEEP SLEEP_EWOR SYNTH_ON TX XOFF'.split(' ')) self.assertEqual(model.transitions(), 'crystal_off eWOR idle init prepare_xmit receive send setSymbolRate setTxPower sleep txDone'.split(' ')) static_model = model.get_static() @@ -229,5 +235,4 @@ class TestStaticModel(unittest.TestCase): self.assertAlmostEqual(param_info('RX', 'power')['function']._regression_args[1], 206, places=0) if __name__ == '__main__': - analytic.safe_function_enabled = False unittest.main() diff --git a/lib/dfatool.py b/lib/dfatool.py index e9af2af..634424b 100755 --- a/lib/dfatool.py +++ b/lib/dfatool.py @@ -1014,6 +1014,36 @@ class AnalyticModel: return detailed_results +def _add_trace_data_to_aggregate(aggregate, key, element): + if not key in aggregate: + aggregate[key] = { + 'isa' : element['isa'] + } + for datakey in element['offline_aggregates'].keys(): + aggregate[key][datakey] = [] + if element['isa'] == 'state': + aggregate[key]['attributes'] = ['power'] + else: + aggregate[key]['attributes'] = ['duration', 'energy', 'rel_energy_prev', 'rel_energy_next'] + if element['plan']['level'] == 'epilogue': + aggregate[key]['attributes'].insert(0, 'timeout') + for datakey, dataval in element['offline_aggregates'].items(): + aggregate[key][datakey].extend(dataval) + +def pta_trace_to_aggregate(traces, ignore_trace_indexes = []): + arg_count = dict() + by_name = dict() + parameter_names = sorted(traces[0]['trace'][0]['parameter'].keys()) + for run in traces: + if run['id'] not in ignore_trace_indexes: + for elem in run['trace']: + if elem['isa'] == 'transition' and not elem['name'] in arg_count and 'args' in elem: + arg_count[elem['name']] = len(elem['args']) + if elem['name'] != 'UNINITIALIZED': + _add_trace_data_to_aggregate(by_name, elem['name'], elem) + return by_name, parameter_names, arg_count + + class PTAModel: u""" Parameter-aware PTA-based energy model. @@ -1046,7 +1076,7 @@ class PTAModel: - rel_energy_next: transition energy relative to next state mean power in pJ """ - def __init__(self, preprocessed_data, ignore_trace_indexes = [], discard_outliers = None, function_override = {}, verbose = True, use_corrcoef = False, hwmodel = None): + def __init__(self, by_name, parameters, arg_count, traces = [], ignore_trace_indexes = [], discard_outliers = None, function_override = {}, verbose = True, use_corrcoef = False, hwmodel = None): """ Prepare a new PTA energy model. @@ -1135,29 +1165,19 @@ class PTAModel: ] ] """ - self.traces = preprocessed_data - self.by_name = {} - self.by_param = {} - self.by_trace = {} + self.by_name = by_name + self.by_param = by_name_to_by_param(by_name) + self._parameter_names = sorted(parameters) + self._num_args = arg_count + self.traces = traces self.cache = {} np.seterr('raise') - self._parameter_names = sorted(self.traces[0]['trace'][0]['parameter'].keys()) - self._num_args = {} self._outlier_threshold = discard_outliers self._use_corrcoef = use_corrcoef self.function_override = function_override self.verbose = verbose self.hwmodel = hwmodel self.ignore_trace_indexes = ignore_trace_indexes - if discard_outliers != None: - self._compute_outlier_stats(ignore_trace_indexes, discard_outliers) - for run in self.traces: - if run['id'] not in ignore_trace_indexes: - for i, elem in enumerate(run['trace']): - if elem['name'] != 'UNINITIALIZED': - self._load_run_elem(i, elem) - if elem['isa'] == 'transition' and not elem['name'] in self._num_args and 'args' in elem: - self._num_args[elem['name']] = len(elem['args']) self._aggregate_to_ndarray(self.by_name) self._compute_all_param_statistics() @@ -1166,86 +1186,14 @@ class PTAModel: param_values = map(lambda x: x[param_index], self.by_name[state_or_tran]['param']) return sorted(set(param_values)) - def _compute_outlier_stats(self, ignore_trace_indexes, threshold): - tmp_by_param = {} - self.median_by_param = {} - for run in self.traces: - if run['id'] not in ignore_trace_indexes: - for i, elem in enumerate(run['trace']): - key = (elem['name'], tuple(_elem_param_and_arg_list(elem))) - if not key in tmp_by_param: - tmp_by_param[key] = {} - for attribute in elem['offline_attributes']: - tmp_by_param[key][attribute] = [] - for attribute in elem['offline_attributes']: - tmp_by_param[key][attribute].extend(elem['offline_aggregates'][attribute]) - for key, elem in tmp_by_param.items(): - if not key in self.median_by_param: - self.median_by_param[key] = {} - for attribute in tmp_by_param[key].keys(): - self.median_by_param[key][attribute] = np.median(tmp_by_param[key][attribute]) - - def _compute_all_param_statistics(self): self.stats = ParamStats(self.by_name, self.by_param, self._parameter_names, self._num_args, self._use_corrcoef) - @classmethod - def from_model(self, model_data, parameter_names): - self.by_name = {} - self.by_param = {} - np.seterr('raise') - self._parameter_names = parameter_names - for state_or_tran, values in model_data.items(): - for elem in values: - self._load_agg_elem(state_or_tran, elem) - #if elem['isa'] == 'transition' and not state_or_tran in self._num_args and 'args' in elem: - # self._num_args = len(elem['args']) - self._aggregate_to_ndarray(self.by_name) - self._compute_all_param_statistics() - def _aggregate_to_ndarray(self, aggregate): for elem in aggregate.values(): for key in elem['attributes']: elem[key] = np.array(elem[key]) - def _prune_outliers(self, key, attribute, data): - if self._outlier_threshold == None: - return data - median = self.median_by_param[key][attribute] - if np.median(np.abs(data - median)) == 0: - return data - pruned_data = list(filter(lambda x: np.abs(0.6745 * (x - median) / np.median(np.abs(data - median))) > self._outlier_threshold, data )) - if len(pruned_data): - vprint(self.verbose, '[I] Pruned outliers from ({}) {}: {}'.format(key, attribute, pruned_data)) - data = list(filter(lambda x: np.abs(0.6745 * (x - median) / np.median(np.abs(data - median))) <= self._outlier_threshold, data )) - return data - - def _add_data_to_aggregate(self, aggregate, key, element): - if not key in aggregate: - aggregate[key] = { - 'isa' : element['isa'] - } - for datakey in element['offline_aggregates'].keys(): - aggregate[key][datakey] = [] - if element['isa'] == 'state': - aggregate[key]['attributes'] = ['power'] - else: - aggregate[key]['attributes'] = ['duration', 'energy', 'rel_energy_prev', 'rel_energy_next'] - if element['plan']['level'] == 'epilogue': - aggregate[key]['attributes'].insert(0, 'timeout') - for datakey, dataval in element['offline_aggregates'].items(): - if datakey in element['offline_attributes']: - dataval = self._prune_outliers((element['name'], tuple(_elem_param_and_arg_list(element))), datakey, dataval) - aggregate[key][datakey].extend(dataval) - - def _load_agg_elem(self, name, elem): - self._add_data_to_aggregate(self.by_name, name, elem) - self._add_data_to_aggregate(self.by_param, (name, tuple(elem['param'])), elem) - - def _load_run_elem(self, i, elem): - self._add_data_to_aggregate(self.by_name, elem['name'], elem) - self._add_data_to_aggregate(self.by_param, (elem['name'], tuple(_elem_param_and_arg_list(elem))), elem) - # This heuristic is very similar to the "function is not much better than # median" checks in get_fitted. So far, doing it here as well is mostly # a performance and not an algorithm quality decision. @@ -1475,13 +1423,17 @@ class PTAModel: real_timeout_list.append(real_timeout) model_timeout_list.append(model_timeout) + if len(self.traces): + return { + 'by_dfa_component' : detailed_results, + 'duration_by_trace' : regression_measures(np.array(model_duration_list), np.array(real_duration_list)), + 'energy_by_trace' : regression_measures(np.array(model_energy_list), np.array(real_energy_list)), + 'timeout_by_trace' : regression_measures(np.array(model_timeout_list), np.array(real_timeout_list)), + 'rel_energy_by_trace' : regression_measures(np.array(model_rel_energy_list), np.array(real_energy_list)), + 'state_energy_by_trace' : regression_measures(np.array(model_state_energy_list), np.array(real_energy_list)), + } return { - 'by_dfa_component' : detailed_results, - 'duration_by_trace' : regression_measures(np.array(model_duration_list), np.array(real_duration_list)), - 'energy_by_trace' : regression_measures(np.array(model_energy_list), np.array(real_energy_list)), - 'timeout_by_trace' : regression_measures(np.array(model_timeout_list), np.array(real_timeout_list)), - 'rel_energy_by_trace' : regression_measures(np.array(model_rel_energy_list), np.array(real_energy_list)), - 'state_energy_by_trace' : regression_measures(np.array(model_state_energy_list), np.array(real_energy_list)), + 'by_dfa_component' : detailed_results } |