diff options
| -rwxr-xr-x | bin/analyze-archive.py | 109 | ||||
| -rwxr-xr-x | lib/dfatool.py | 79 |
2 files changed, 143 insertions, 45 deletions
diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index df3ecbe..1597155 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -20,24 +20,47 @@ def print_model_quality(results): print('{:20s} {:15s} {:.0f}'.format( state_or_tran, key, result['mae'])) +def format_quality_measures(result): + if 'smape' in result: + return '{:6.2f}% / {:9.0f}'.format(result['smape'], result['mae']) + else: + return '{:6} {:9.0f}'.format('', result['mae']) def model_quality_table(result_lists, info_list): - for state_or_tran in result_lists[0].keys(): - for key in result_lists[0][state_or_tran].keys(): + for state_or_tran in result_lists[0]['by_dfa_component'].keys(): + for key in result_lists[0]['by_dfa_component'][state_or_tran].keys(): buf = '{:20s} {:15s}'.format(state_or_tran, key) for i, results in enumerate(result_lists): info = info_list[i] buf += ' ||| ' if info == None or info(state_or_tran, key): - result = results[state_or_tran][key] - if 'smape' in result: - buf += '{:6.2f}% / {:9.0f}'.format(result['smape'], result['mae']) - else: - buf += '{:6} {:9.0f}'.format('', result['mae']) + result = results['by_dfa_component'][state_or_tran][key] + buf += format_quality_measures(result) else: buf += '{:6}----{:9}'.format('', '') print(buf) +def model_summary_table(result_list): + buf = 'transition duration' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['duration_by_trace']) + print(buf) + buf = 'total energy ' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['energy_by_trace']) + print(buf) + buf = 'transition timeout ' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['timeout_by_trace']) + print(buf) + + def print_text_model_data(model, pm, pq, lm, lq, am, ai, aq): print('') print(r'key attribute $1 - \frac{\sigma_X}{...}$') @@ -59,14 +82,15 @@ if __name__ == '__main__': ignored_trace_indexes = None discard_outliers = None - tex_output = False safe_functions_enabled = False function_override = {} + show_models = [] + show_quality = [] try: optspec = ( - 'plot-unparam= plot-param= ' - 'ignored-trace-indexes= discard-outliers= function-override= tex-output ' + 'plot-unparam= plot-param= show-models= show-quality= ' + 'ignored-trace-indexes= discard-outliers= function-override= ' 'with-safe-functions' ) raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' ')) @@ -88,8 +112,11 @@ if __name__ == '__main__': state_or_tran, attribute, *function_str = function_desc.split(' ') function_override[(state_or_tran, attribute)] = ' '.join(function_str) - if 'tex-output' in opts: - tex_output = True + if 'show-models' in opts: + show_models = opts['show-models'].split(',') + + if 'show-quality' in opts: + show_quality = opts['show-quality'].split(',') if 'with-safe-functions' in opts: safe_functions_enabled = True @@ -112,36 +139,40 @@ if __name__ == '__main__': state_or_trans, attribute = kv.split(' ') plotter.plot_y(model.by_name[state_or_trans][attribute]) - print('--- simple static model ---') + if len(show_models): + print('--- simple static model ---') static_model = model.get_static() - #for state in model.states(): - # print('{:10s}: {:.0f} µW ({:.2f})'.format( - # state, - # static_model(state, 'power'), - # model.generic_param_dependence_ratio(state, 'power'))) - # for param in model.parameters(): - # print('{:10s} dependence on {:15s}: {:.2f}'.format( - # '', - # param, - # model.param_dependence_ratio(state, 'power', param))) - #for trans in model.transitions(): - # print('{:10s}: {:.0f} / {:.0f} / {:.0f} pJ ({:.2f} / {:.2f} / {:.2f})'.format( - # trans, static_model(trans, 'energy'), - # static_model(trans, 'rel_energy_prev'), - # static_model(trans, 'rel_energy_next'), - # model.generic_param_dependence_ratio(trans, 'energy'), - # model.generic_param_dependence_ratio(trans, 'rel_energy_prev'), - # model.generic_param_dependence_ratio(trans, 'rel_energy_next'))) - # print('{:10s}: {:.0f} µs'.format(trans, static_model(trans, 'duration'))) + if 'static' in show_models or 'all' in show_models: + for state in model.states(): + print('{:10s}: {:.0f} µW ({:.2f})'.format( + state, + static_model(state, 'power'), + model.generic_param_dependence_ratio(state, 'power'))) + for param in model.parameters(): + print('{:10s} dependence on {:15s}: {:.2f}'.format( + '', + param, + model.param_dependence_ratio(state, 'power', param))) + for trans in model.transitions(): + print('{:10s}: {:.0f} / {:.0f} / {:.0f} pJ ({:.2f} / {:.2f} / {:.2f})'.format( + trans, static_model(trans, 'energy'), + static_model(trans, 'rel_energy_prev'), + static_model(trans, 'rel_energy_next'), + model.generic_param_dependence_ratio(trans, 'energy'), + model.generic_param_dependence_ratio(trans, 'rel_energy_prev'), + model.generic_param_dependence_ratio(trans, 'rel_energy_next'))) + print('{:10s}: {:.0f} µs'.format(trans, static_model(trans, 'duration'))) static_quality = model.assess(static_model) - print('--- LUT ---') + if len(show_models): + print('--- LUT ---') lut_model = model.get_param_lut() lut_quality = model.assess(lut_model) - print('--- param model ---') + if len(show_models): + print('--- param model ---') param_model, param_info = model.get_fitted(safe_functions_enabled = safe_functions_enabled) - if not tex_output: + if 'param' in show_models or 'all' in show_models: for state in model.states(): for attribute in ['power']: if param_info(state, attribute): @@ -153,10 +184,14 @@ if __name__ == '__main__': print('{:10s}: {:10s}: {}'.format(trans, attribute, param_info(trans, attribute)['function']._model_str)) print('{:10s} {:10s} {}'.format('', '', param_info(trans, attribute)['function']._regression_args)) analytic_quality = model.assess(param_model) - if tex_output: + + if 'tex' in show_models or 'tex' in show_quality: print_text_model_data(model, static_model, static_quality, lut_model, lut_quality, param_model, param_info, analytic_quality) - else: + + if 'table' in show_quality or 'all' in show_quality: model_quality_table([static_quality, analytic_quality, lut_quality], [None, param_info, None]) + if 'summary' in show_quality or 'all' in show_quality: + model_summary_table([static_quality, analytic_quality, lut_quality]) if 'plot-param' in opts: for kv in opts['plot-param'].split(';'): diff --git a/lib/dfatool.py b/lib/dfatool.py index ad37f72..06f81c4 100755 --- a/lib/dfatool.py +++ b/lib/dfatool.py @@ -128,6 +128,7 @@ def regression_measures(predicted, actual): 'rmsd' : np.sqrt(np.mean(deviations**2), dtype=np.float64), 'ssr' : np.sum(deviations**2, dtype=np.float64), 'rsq' : r2_score(actual, predicted), + 'count' : len(actual), } #rsq_quotient = np.sum((actual - mean)**2, dtype=np.float64) * np.sum((predicted - mean)**2, dtype=np.float64) @@ -758,16 +759,35 @@ def _compute_param_statistics_parallel(args): 'result' : _compute_param_statistics(*args['args']) } +def all_params_are_numeric(data, param_idx): + param_values = list(map(lambda x: x[param_idx], data['param'])) + if len(list(filter(is_numeric, param_values))) == len(param_values): + return True + return False + def _compute_param_statistics(by_name, by_param, parameter_names, num_args, state_or_trans, key): ret = { 'std_static' : np.std(by_name[state_or_trans][key]), 'std_param_lut' : np.mean([np.std(by_param[x][key]) for x in by_param.keys() if x[0] == state_or_trans]), 'std_by_param' : {}, 'std_by_arg' : [], + 'corr_by_param' : {}, + 'corr_by_arg' : [], } for param_idx, param in enumerate(parameter_names): ret['std_by_param'][param] = _mean_std_by_param(by_param, state_or_trans, key, param_idx) + if all_params_are_numeric(by_name[state_or_trans], param_idx): + param_values = np.array(list((map(lambda x: x[param_idx], by_name[state_or_trans]['param'])))) + try: + ret['corr_by_param'][param] = np.corrcoef(by_name[state_or_trans][key], param_values)[0, 1] + except FloatingPointError as fpe: + # Typically happens when all parameter values are identical. + # Building a correlation coefficient is pointless in this case + # -> assume no correlation + ret['corr_by_param'][param] = 0. + else: + ret['corr_by_param'][param] = 0. if arg_support_enabled and state_or_trans in num_args: for arg_index in range(num_args[state_or_trans]): ret['std_by_arg'].append(_mean_std_by_param(by_param, state_or_trans, key, len(parameter_names) + arg_index)) @@ -793,6 +813,9 @@ def _mean_std_by_param(by_param, state_or_tran, key, param_index): print('[W] parameter value partition for {} is empty'.format(param_value)) return np.mean([np.std(partition) for partition in partitions]) +#def _corr_by_param(by_name, state_or_tran, key, param_index): +# + class EnergyModel: def __init__(self, preprocessed_data, ignore_trace_indexes = None, discard_outliers = None, function_override = {}, verbose = True): @@ -851,10 +874,6 @@ class EnergyModel: self.stats[state_or_trans] = {} for key in self.by_name[state_or_trans]['attributes']: if key in self.by_name[state_or_trans]: - #try: - # print(state_or_trans, key, np.corrcoef(self.by_name[state_or_trans][key], np.array(self.by_name[state_or_trans]['param']).T)) - #except TypeError as e: - # print(state_or_trans, key, e) self.stats[state_or_trans][key] = _compute_param_statistics(self.by_name, self.by_param, self._parameter_names, self._num_args, state_or_trans, key) #queue.append({ # 'state_or_trans' : state_or_trans, @@ -1109,14 +1128,58 @@ class EnergyModel: return self._parameter_names def assess(self, model_function): - results = {} + detailed_results = {} + model_energy_list = [] + real_energy_list = [] + model_duration_list = [] + real_duration_list = [] + model_timeout_list = [] + real_timeout_list = [] for name, elem in sorted(self.by_name.items()): - results[name] = {} + detailed_results[name] = {} for key in elem['attributes']: predicted_data = np.array(list(map(lambda i: model_function(name, key, param=elem['param'][i]), range(len(elem[key]))))) measures = regression_measures(predicted_data, elem[key]) - results[name][key] = measures - return results + detailed_results[name][key] = measures + + for trace in self.traces: + for rep_id in range(len(trace['trace'][0]['offline'])): + model_energy = 0. + real_energy = 0. + model_duration = 0. + real_duration = 0. + model_timeout = 0. + real_timeout = 0. + for trace_part in trace['trace']: + name = trace_part['name'] + isa = trace_part['isa'] + if name != 'UNINITIALIZED': + param = trace_part['offline_aggregates']['param'][rep_id] + power = trace_part['offline'][rep_id]['uW_mean'] + duration = trace_part['offline'][rep_id]['us'] + real_energy += power * duration + if isa == 'state': + model_energy += model_function(name, 'power', param=param) * duration + else: + model_energy += model_function(name, 'energy', param=param) + real_duration += duration + model_duration += model_function(name, 'duration', param=param) + if 'plan' in trace_part and trace_part['plan']['level'] == 'epilogue': + real_timeout += trace_part['offline'][rep_id]['timeout'] + model_timeout += model_function(name, 'timeout', param=param) + real_energy_list.append(real_energy) + model_energy_list.append(model_energy) + real_duration_list.append(real_duration) + model_duration_list.append(model_duration) + real_timeout_list.append(real_timeout) + model_timeout_list.append(model_timeout) + + return { + 'by_dfa_component' : detailed_results, + 'duration_by_trace' : regression_measures(np.array(model_duration_list), np.array(real_duration_list)), + 'energy_by_trace' : regression_measures(np.array(model_energy_list), np.array(real_energy_list)), + 'timeout_by_trace' : regression_measures(np.array(model_timeout_list), np.array(real_timeout_list)), + } |