1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
#!/usr/bin/env python3
"""eval-kconfig - tbd
"""
import argparse
import json
import kconfiglib
import logging
import os
import sys
import numpy as np
from dfatool import kconfig, validation
from dfatool.loader import KConfigAttributes
from dfatool.model import KConfigModel
from versuchung.experiment import Experiment
from versuchung.types import String, Bool, Integer
from versuchung.files import File, Directory
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__
)
parser.add_argument(
"--log-level",
default=logging.INFO,
type=lambda level: getattr(logging, level.upper()),
help="Set log level",
)
parser.add_argument(
"--attribute", choices=["rom", "ram"], default="rom", help="Model attribute"
)
parser.add_argument(
"--with-choice-node", action="store_true", help="Use non-binary Choice Nodes"
)
parser.add_argument(
"--max-loss",
type=float,
help="Maximum acceptable model loss for DecisionTree Leaves",
default=10,
)
# Falls die population exhaustive ist, kann man nun den generalization error berechnen
parser.add_argument(
"--sample-size",
type=int,
help="Perform model generation and validation with N random samples from the population",
metavar="N",
)
parser.add_argument("kconfig_path", type=str, help="Path to Kconfig file")
parser.add_argument(
"experiment_root", type=str, help="Experiment results directory"
)
parser.add_argument("model", type=str, help="JSON model", nargs="?")
args = parser.parse_args()
if isinstance(args.log_level, int):
logging.basicConfig(level=args.log_level)
else:
print(f"Invalid log level. Setting log level to INFO.", file=sys.stderr)
data = KConfigAttributes(args.kconfig_path, args.experiment_root)
k = 10
if args.sample_size:
shuffled_data_indices = np.random.permutation(np.arange(len(data.data)))
sample_indices = shuffled_data_indices[: args.sample_size]
nonsample_indices = shuffled_data_indices[args.sample_size :]
partition_pairs = validation._xv_partitions_kfold(args.sample_size, k)
partition_pairs = list(
map(
lambda tv: (shuffled_data_indices[tv[0]], shuffled_data_indices[tv[1]]),
partition_pairs,
)
)
else:
partition_pairs = validation._xv_partitions_kfold(len(data.data), k)
measures = list()
for training_set, validation_set in partition_pairs:
model = KConfigModel.from_benchmark(data, args.attribute, indices=training_set)
model.with_choice_node = args.with_choice_node
if args.max_loss:
model.max_loss = args.max_loss
model.build_tree()
measures.append(model.assess_benchmark(data, indices=validation_set))
aggregate = dict()
for measure in measures[0].keys():
aggregate[measure] = np.mean(list(map(lambda m: m[measure], measures)))
aggregate["unpredictable_count"] = np.sum(
list(map(lambda m: m["unpredictable_count"], measures))
)
print("10-fold Cross Validation:")
print(f"MAE: {aggregate['mae']:.0f} B")
print(f"SMAPE: {aggregate['smape']:.1f} %")
print(f"Unpredictable Configurations: {aggregate['unpredictable_count']}")
print(aggregate)
if args.sample_size:
print("Estimated Generalization Error")
model = KConfigModel.from_benchmark(
data, args.attribute, indices=sample_indices
)
model.with_choice_node = args.with_choice_node
if args.max_loss:
model.max_loss = args.max_loss
model.build_tree()
generalization_measure = model.assess_benchmark(data, indices=nonsample_indices)
print(f"MAE: {generalization_measure['mae']:.0f} B")
print(f"SMAPE: {generalization_measure['smape']:.1f} %")
print(
f"Unpredictable Configurations: {generalization_measure['unpredictable_count']}"
)
"""
if args.model:
with open(args.model, "r") as f:
model = KConfigModel.from_json(json.load(f))
else:
model = KConfigModel.from_benchmark(data, args.attribute)
model.build_tree()
"""
if __name__ == "__main__":
main()
|
