diff options
| -rwxr-xr-x | bin/analyze-archive.py | 2 | ||||
| -rw-r--r-- | lib/dfatool.py | 45 |
2 files changed, 26 insertions, 21 deletions
diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index 4551cda..fac8b00 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -404,7 +404,7 @@ if __name__ == '__main__': if 'table' in show_quality or 'all' in show_quality: model_quality_table([static_quality, analytic_quality, lut_quality], [None, param_info, None]) if 'summary' in show_quality or 'all' in show_quality: - model_summary_table([static_quality, analytic_quality, lut_quality]) + model_summary_table([model.assess_on_traces(static_model), model.assess_on_traces(param_model), model.assess_on_traces(lut_model)]) if 'plot-param' in opts: for kv in opts['plot-param'].split(';'): diff --git a/lib/dfatool.py b/lib/dfatool.py index 363c2a2..4ba4911 100644 --- a/lib/dfatool.py +++ b/lib/dfatool.py @@ -1743,15 +1743,32 @@ class PTAModel: The by_name entries of this PTAModel are used as ground truth and compared with the values predicted by model_function. - If 'traces' was set when creating this object, the model quality is - also assessed on a per-trace basis. - For proper model assessments, the data used to generate model_function and the data fed into this AnalyticModel instance must be mutually exclusive (e.g. by performing cross validation). Otherwise, overfitting cannot be detected. """ detailed_results = {} + for name, elem in sorted(self.by_name.items()): + detailed_results[name] = {} + for key in elem['attributes']: + predicted_data = np.array(list(map(lambda i: model_function(name, key, param=elem['param'][i]), range(len(elem[key]))))) + measures = regression_measures(predicted_data, elem[key]) + detailed_results[name][key] = measures + + return { + 'by_name' : detailed_results + } + + + def assess_on_traces(self, model_function): + """ + Calculate MAE, SMAPE, etc. of model_function for each trace known to this PTAModel instance. + + :returns: dict of `duration_by_trace`, `energy_by_trace`, `timeout_by_trace`, `rel_energy_by_trace` and `state_energy_by_trace`. + Each entry holds regression measures for the corresponding measure. Note that the determined model quality heavily depends on the + traces: small-ish absolute errors in states which frequently occur may have more effect than large absolute errors in rarely occuring states + """ model_energy_list = [] real_energy_list = [] model_rel_energy_list = [] @@ -1760,12 +1777,6 @@ class PTAModel: real_duration_list = [] model_timeout_list = [] real_timeout_list = [] - for name, elem in sorted(self.by_name.items()): - detailed_results[name] = {} - for key in elem['attributes']: - predicted_data = np.array(list(map(lambda i: model_function(name, key, param=elem['param'][i]), range(len(elem[key]))))) - measures = regression_measures(predicted_data, elem[key]) - detailed_results[name][key] = measures for trace in self.traces: if trace['id'] not in self.ignore_trace_indexes: @@ -1818,21 +1829,15 @@ class PTAModel: real_timeout_list.append(real_timeout) model_timeout_list.append(model_timeout) - if len(self.traces): - return { - 'by_name' : detailed_results, - 'duration_by_trace' : regression_measures(np.array(model_duration_list), np.array(real_duration_list)), - 'energy_by_trace' : regression_measures(np.array(model_energy_list), np.array(real_energy_list)), - 'timeout_by_trace' : regression_measures(np.array(model_timeout_list), np.array(real_timeout_list)), - 'rel_energy_by_trace' : regression_measures(np.array(model_rel_energy_list), np.array(real_energy_list)), - 'state_energy_by_trace' : regression_measures(np.array(model_state_energy_list), np.array(real_energy_list)), - } return { - 'by_name' : detailed_results + 'duration_by_trace' : regression_measures(np.array(model_duration_list), np.array(real_duration_list)), + 'energy_by_trace' : regression_measures(np.array(model_energy_list), np.array(real_energy_list)), + 'timeout_by_trace' : regression_measures(np.array(model_timeout_list), np.array(real_timeout_list)), + 'rel_energy_by_trace' : regression_measures(np.array(model_rel_energy_list), np.array(real_energy_list)), + 'state_energy_by_trace' : regression_measures(np.array(model_state_energy_list), np.array(real_energy_list)), } - class MIMOSA: """ MIMOSA log loader for DFA traces with auto-calibration. |