add kconfig model cross validation

1 files changed, 38 insertions, 3 deletions

diff --git a/lib/model.py b/lib/model.py
index f330327..f422204 100644
--- a/lib/model.py
+++ b/lib/model.py
@@ -1158,7 +1158,11 @@ class PTAModel:
 
 
 class KConfigModel:
-    """Decision-Tree Model for a specific system attribute such as ROM or RAM usage"""
+    """
+    Decision-Tree Model for a specific system attribute such as ROM or RAM usage.
+
+    See Guo et al: "Data-efficient performance learning for configurable systems", 2017
+    """
 
     class Node:
         pass
@@ -1280,6 +1284,8 @@ class KConfigModel:
             kconf_choice = next(
                 filter(lambda choice: choice.name == self.symbol, kconf.choices)
             )
+            if kconf_choice.selection is None:
+                return None
             selection = kconf_choice.selection.name
             if selection in self.choice:
                 return self.choice[selection].model(kconf)
@@ -1299,9 +1305,14 @@ class KConfigModel:
             return ret
 
     @classmethod
-    def from_benchmark(cls, kconfig_benchmark, attribute):
+    def from_benchmark(cls, kconfig_benchmark, attribute, indices=None):
         self = cls()
-        self.data = kconfig_benchmark.data
+        if indices is None:
+            self.data = kconfig_benchmark.data
+        else:
+            self.data = list()
+            for i in indices:
+                self.data.append(kconfig_benchmark.data[i])
         self.symbols = kconfig_benchmark.symbol_names
         self.choices = kconfig_benchmark.choice_names
         self.symbol = kconfig_benchmark.symbol
@@ -1459,3 +1470,27 @@ class KConfigModel:
                     )
 
         return node
+
+    def assess_benchmark(self, kconfig_benchmark, indices=None):
+        if indices is None:
+            indices = range(len(kconfig_benchmark.data))
+
+        kconf = kconfig_benchmark.kconf
+
+        predictions = list()
+        values = list()
+        unpredictable_count = 0
+
+        for i in indices:
+            kconf.load_config(kconfig_benchmark.configs[i])
+            prediction = self.model.model(kconf)
+
+            if prediction is None:
+                unpredictable_count += 1
+            else:
+                predictions.append(self.model.model(kconf))
+                values.append(self.attr_function(kconfig_benchmark.data[i]))
+
+        measures = regression_measures(np.array(predictions), np.array(values))
+        measures["unpredictable_count"] = unpredictable_count
+        return measures