summaryrefslogtreecommitdiff
path: root/bin/eval-rel-energy.py
blob: 4803c519361a77023dbc5c539c5c3a6011205f48 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#!/usr/bin/env python3

import getopt
import re
import sys
from dfatool import PTAModel, RawData, pta_trace_to_aggregate

opts = {}


def get_file_groups(args):
    groups = []
    index_low = 0
    while ':' in args[index_low:]:
        index_high = args[index_low:].index(':') + index_low
        groups.append(args[index_low: index_high])
        index_low = index_high + 1
    groups.append(args[index_low:])
    return groups


if __name__ == '__main__':

    ignored_trace_indexes = []
    discard_outliers = None
    safe_functions_enabled = False
    function_override = {}
    show_models = []
    show_quality = []

    try:
        optspec = (
            'plot-unparam= plot-param= show-models= show-quality= '
            'ignored-trace-indexes= discard-outliers= function-override= '
            'with-safe-functions'
        )
        raw_opts, args = getopt.getopt(sys.argv[1:], "", optspec.split(' '))

        for option, parameter in raw_opts:
            optname = re.sub(r'^--', '', option)
            opts[optname] = parameter

            if 'ignored-trace-indexes' in opts:
                ignored_trace_indexes = list(map(int, opts['ignored-trace-indexes'].split(',')))
                if 0 in ignored_trace_indexes:
                    print('[E] arguments to --ignored-trace-indexes start from 1')

            if 'discard-outliers' in opts:
                discard_outliers = float(opts['discard-outliers'])

            if 'function-override' in opts:
                for function_desc in opts['function-override'].split(';'):
                    state_or_tran, attribute, *function_str = function_desc.split(' ')
                    function_override[(state_or_tran, attribute)] = ' '.join(function_str)

            if 'show-models' in opts:
                show_models = opts['show-models'].split(',')

            if 'show-quality' in opts:
                show_quality = opts['show-quality'].split(',')

            if 'with-safe-functions' in opts:
                safe_functions_enabled = True

    except getopt.GetoptError as err:
        print(err)
        sys.exit(2)

    score_absolute = 0
    score_relative = 0

    for file_group in get_file_groups(args):
        print('')
        print('{}:'.format(' '.join(file_group)))
        raw_data = RawData(file_group)

        preprocessed_data = raw_data.get_preprocessed_data(verbose=False)
        by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data, ignored_trace_indexes)
        model = PTAModel(by_name, parameters, arg_count,
                         traces=preprocessed_data,
                         ignore_trace_indexes=ignored_trace_indexes,
                         discard_outliers=discard_outliers,
                         function_override=function_override,
                         verbose=False)

        lut_quality = model.assess(model.get_param_lut())

        for trans in model.transitions():
            absolute_quality = lut_quality['by_name'][trans]['energy']
            relative_quality = lut_quality['by_name'][trans]['rel_energy_prev']
            if absolute_quality['mae'] < relative_quality['mae']:
                best = 'absolute'
                score_absolute += 1
            else:
                best = 'relative'
                score_relative += 1

            print('{:20s}: {:s}   (diff {:.0f} / {:.2f}%, abs {:.0f} / {:.2f}%, rel {:.0f} / {:.2f}%)'.format(
                trans, best,
                abs(absolute_quality['mae'] - relative_quality['mae']),
                abs(absolute_quality['mae'] - relative_quality['mae']) * 100 / max(absolute_quality['mae'], relative_quality['mae']),
                absolute_quality['mae'], absolute_quality['smape'],
                relative_quality['mae'], relative_quality['smape']))

    sys.exit(0)