diff options
Diffstat (limited to 'bin/analyze-timing.py')
| -rwxr-xr-x | bin/analyze-timing.py | 321 |
1 files changed, 321 insertions, 0 deletions
diff --git a/bin/analyze-timing.py b/bin/analyze-timing.py new file mode 100755 index 0000000..b0aaaca --- /dev/null +++ b/bin/analyze-timing.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +""" +analyze-timing -- generate analytic energy model from annotated OnboardTimerHarness traces. + +Usage: +PYTHONPATH=lib bin/analyze-timing.py [options] <tracefiles ...> + +analyze-timing generates an analytic energy model (``AnalyticModel``)from one or more annotated +traces generated by generate-dfa-benchmark using OnboardTimerHarness. By default, it does nothing else -- +use one of the --plot-* or --show-* options to examine the generated model. + +Options: +--plot-unparam=<name>:<attribute>:<Y axis label>[;<name>:<attribute>:<label>;...] + Plot all mesurements for <name> <attribute> without regard for parameter values. + X axis is measurement number/id. + +--plot-param=<name> <attribute> <parameter> [gplearn function][;<name> <attribute> <parameter> [function];...] + Plot measurements for <name> <attribute> by <parameter>. + X axis is parameter value. + Plots the model function as one solid line for each combination of non-<parameter> + parameters. Also plots the corresponding measurements. + If gplearn function is set, it is plotted using dashed lines. + +--show-models=<static|paramdetection|param|all|tex> + static: show static model values as well as parameter detection heuristic + paramdetection: show stddev of static/lut/fitted model + param: show parameterized model functions and regression variable values + all: all of the above + tex: print tex/pgfplots-compatible model data on stdout + +--show-quality=<table|summary|all|tex> + table: show static/fitted/lut SMAPE and MAE for each name and attribute + summary: show static/fitted/lut SMAPE and MAE for each attribute, averaged over all states/transitions + all: all of the above + tex: print tex/pgfplots-compatible model quality data on stdout + +--ignored-trace-indexes=<i1,i2,...> + Specify traces which should be ignored due to bogus data. 1 is the first + trace, 2 the second, and so on. + +--cross-validate=<method>:<count> + Perform cross validation when computing model quality. + Only works with --show-quality=table at the moment. + If <method> is "montecarlo": Randomly divide data into 2/3 training and 1/3 + validation, <count> times. Reported model quality is the average of all + validation runs. Data is partitioned without regard for parameter values, + so a specific parameter combination may be present in both training and + validation sets or just one of them. + +--function-override=<name attribute function>[;<name> <attribute> <function>;...] + Manually specify the function to fit for <name> <attribute>. A function + specified this way bypasses parameter detection: It is always assigned, + even if the model seems to be independent of the parameters it references. + +--with-safe-functions + If set, include "safe" functions (safe_log, safe_inv, safe_sqrt) which are + also defined for cases such as safe_inv(0) or safe_sqrt(-1). This allows + a greater range of functions to be tried during fitting. + +--hwmodel=<hwmodel.json> + Load DFA hardware model from JSON + +--export-energymodel=<model.json> + Export energy model. Requires --hwmodel. +""" + +import getopt +import json +import plotter +import re +import sys +from dfatool import AnalyticModel, TimingData, pta_trace_to_aggregate +from dfatool import soft_cast_int, is_numeric, gplearn_to_function +from dfatool import CrossValidator + +opts = {} + +def print_model_quality(results): + for state_or_tran in results.keys(): + print() + for key, result in results[state_or_tran].items(): + if 'smape' in result: + print('{:20s} {:15s} {:.2f}% / {:.0f}'.format( + state_or_tran, key, result['smape'], result['mae'])) + else: + print('{:20s} {:15s} {:.0f}'.format( + state_or_tran, key, result['mae'])) + +def format_quality_measures(result): + if 'smape' in result: + return '{:6.2f}% / {:9.0f}'.format(result['smape'], result['mae']) + else: + return '{:6} {:9.0f}'.format('', result['mae']) + +def model_quality_table(result_lists, info_list): + for state_or_tran in result_lists[0]['by_name'].keys(): + for key in result_lists[0]['by_name'][state_or_tran].keys(): + buf = '{:20s} {:15s}'.format(state_or_tran, key) + for i, results in enumerate(result_lists): + info = info_list[i] + buf += ' ||| ' + if info == None or info(state_or_tran, key): + result = results['by_name'][state_or_tran][key] + buf += format_quality_measures(result) + else: + buf += '{:6}----{:9}'.format('', '') + print(buf) + +def model_summary_table(result_list): + buf = 'transition duration' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['duration_by_trace']) + print(buf) + buf = 'total energy ' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['energy_by_trace']) + print(buf) + buf = 'rel total energy ' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['rel_energy_by_trace']) + print(buf) + buf = 'state-only energy ' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['state_energy_by_trace']) + print(buf) + buf = 'transition timeout ' + for results in result_list: + if len(buf): + buf += ' ||| ' + buf += format_quality_measures(results['timeout_by_trace']) + print(buf) + + +def print_text_model_data(model, pm, pq, lm, lq, am, ai, aq): + print('') + print(r'key attribute $1 - \frac{\sigma_X}{...}$') + for state_or_tran in model.by_name.keys(): + for attribute in model.attributes(state_or_tran): + print('{} {} {:.8f}'.format(state_or_tran, attribute, model.stats.generic_param_dependence_ratio(state_or_tran, attribute))) + + print('') + print(r'key attribute parameter $1 - \frac{...}{...}$') + for state_or_tran in model.by_name.keys(): + for attribute in model.attributes(state_or_tran): + for param in model.parameters(): + print('{} {} {} {:.8f}'.format(state_or_tran, attribute, param, model.stats.param_dependence_ratio(state_or_tran, attribute, param))) + if state_or_tran in model._num_args: + for arg_index in range(model._num_args[state_or_tran]): + print('{} {} {:d} {:.8f}'.format(state_or_tran, attribute, arg_index, model.stats.arg_dependence_ratio(state_or_tran, attribute, arg_index))) + +if __name__ == '__main__': + + ignored_trace_indexes = [] + discard_outliers = None + safe_functions_enabled = False + function_override = {} + show_models = [] + show_quality = [] + hwmodel = None + energymodel_export_file = None + xv_method = None + xv_count = 10 + + try: + optspec = ( + 'plot-unparam= plot-param= show-models= show-quality= ' + 'ignored-trace-indexes= discard-outliers= function-override= ' + 'cross-validate= ' + 'with-safe-functions hwmodel= export-energymodel=' + ) + raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' ')) + + for option, parameter in raw_opts: + optname = re.sub(r'^--', '', option) + opts[optname] = parameter + + if 'ignored-trace-indexes' in opts: + ignored_trace_indexes = list(map(int, opts['ignored-trace-indexes'].split(','))) + if 0 in ignored_trace_indexes: + print('[E] arguments to --ignored-trace-indexes start from 1') + + if 'discard-outliers' in opts: + discard_outliers = float(opts['discard-outliers']) + + if 'function-override' in opts: + for function_desc in opts['function-override'].split(';'): + state_or_tran, attribute, *function_str = function_desc.split(' ') + function_override[(state_or_tran, attribute)] = ' '.join(function_str) + + if 'show-models' in opts: + show_models = opts['show-models'].split(',') + + if 'show-quality' in opts: + show_quality = opts['show-quality'].split(',') + + if 'cross-validate' in opts: + xv_method, xv_count = opts['cross-validate'].split(':') + xv_count = int(xv_count) + + if 'with-safe-functions' in opts: + safe_functions_enabled = True + + if 'hwmodel' in opts: + with open(opts['hwmodel'], 'r') as f: + hwmodel = json.load(f) + + except getopt.GetoptError as err: + print(err) + sys.exit(2) + + raw_data = TimingData(args) + + preprocessed_data = raw_data.get_preprocessed_data() + by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data, ignored_trace_indexes) + model = AnalyticModel(by_name, parameters) + + if xv_method: + xv = CrossValidator(AnalyticModel, by_name, parameters, arg_count) + + if 'plot-unparam' in opts: + for kv in opts['plot-unparam'].split(';'): + state_or_trans, attribute, ylabel = kv.split(':') + fname = 'param_y_{}_{}.pdf'.format(state_or_trans,attribute) + plotter.plot_y(model.by_name[state_or_trans][attribute], xlabel = 'measurement #', ylabel = ylabel, output = fname) + + if len(show_models): + print('--- simple static model ---') + static_model = model.get_static() + if 'static' in show_models or 'all' in show_models: + for trans in model.names: + print('{:10s}: {:.0f} µs'.format(trans, static_model(trans, 'duration'))) + + if xv_method == 'montecarlo': + static_quality = xv.montecarlo(lambda m: m.get_static(), xv_count) + else: + static_quality = model.assess(static_model) + + if len(show_models): + print('--- LUT ---') + lut_model = model.get_param_lut() + + if xv_method == 'montecarlo': + lut_quality = xv.montecarlo(lambda m: m.get_param_lut(fallback=True), xv_count) + else: + lut_quality = model.assess(lut_model) + + if len(show_models): + print('--- param model ---') + + param_model, param_info = model.get_fitted(safe_functions_enabled = safe_functions_enabled) + + if 'paramdetection' in show_models or 'all' in show_models: + for transition in model.names: + for attribute in ['duration']: + info = param_info(transition, attribute) + print('{:10s} {:10s} non-param stddev {:f}'.format( + transition, attribute, model.stats.stats[transition][attribute]['std_static'] + )) + print('{:10s} {:10s} param-lut stddev {:f}'.format( + transition, attribute, model.stats.stats[transition][attribute]['std_param_lut'] + )) + for param in sorted(model.stats.stats[transition][attribute]['std_by_param'].keys()): + print('{:10s} {:10s} {:10s} stddev {:f}'.format( + transition, attribute, param, model.stats.stats[transition][attribute]['std_by_param'][param] + )) + if info != None: + for param_name in sorted(info['fit_result'].keys(), key=str): + param_fit = info['fit_result'][param_name]['results'] + for function_type in sorted(param_fit.keys()): + function_rmsd = param_fit[function_type]['rmsd'] + print('{:10s} {:10s} {:10s} mean {:10s} RMSD {:.0f}'.format( + transition, attribute, str(param_name), function_type, function_rmsd + )) + + if 'param' in show_models or 'all' in show_models: + for trans in model.names: + for attribute in ['duration']: + if param_info(trans, attribute): + print('{:10s}: {:10s}: {}'.format(trans, attribute, param_info(trans, attribute)['function']._model_str)) + print('{:10s} {:10s} {}'.format('', '', param_info(trans, attribute)['function']._regression_args)) + + if xv_method == 'montecarlo': + analytic_quality = xv.montecarlo(lambda m: m.get_fitted()[0], xv_count) + else: + analytic_quality = model.assess(param_model) + + if 'tex' in show_models or 'tex' in show_quality: + print_text_model_data(model, static_model, static_quality, lut_model, lut_quality, param_model, param_info, analytic_quality) + + if 'table' in show_quality or 'all' in show_quality: + model_quality_table([static_quality, analytic_quality, lut_quality], [None, param_info, None]) + if 'summary' in show_quality or 'all' in show_quality: + model_summary_table([static_quality, analytic_quality, lut_quality]) + + if 'plot-param' in opts: + for kv in opts['plot-param'].split(';'): + state_or_trans, attribute, param_name, *function = kv.split(' ') + if len(function): + function = gplearn_to_function(' '.join(function)) + else: + function = None + plotter.plot_param(model, state_or_trans, attribute, model.param_index(param_name), extra_function=function) + + if 'export-energymodel' in opts: + if not hwmodel: + print('[E] --export-energymodel requires --hwmodel to be set') + sys.exit(1) + json_model = model.to_json() + with open(opts['export-energymodel'], 'w') as f: + json.dump(json_model, f, indent = 2, sort_keys = True) + + + sys.exit(0) |