kconfig: model generation and validation with limited sample size

eval-kconfig estimates the generalization error in this case

1 files changed, 23 insertions, 1 deletions

diff --git a/bin/analyze-kconfig.py b/bin/analyze-kconfig.py
index f7ae448..ff220b0 100755
--- a/bin/analyze-kconfig.py
+++ b/bin/analyze-kconfig.py
@@ -13,6 +13,8 @@ import kconfiglib
 import logging
 import os
 
+import numpy as np
+
 from dfatool.loader import KConfigAttributes
 from dfatool.model import KConfigModel
 
@@ -48,6 +50,15 @@ def main():
         type=lambda level: getattr(logging, level.upper()),
         help="Set log level",
     )
+    parser.add_argument(
+        "--info", action="store_true", help="Show Kconfig and benchmark information"
+    )
+    parser.add_argument(
+        "--sample-size",
+        type=int,
+        help="Restrict model generation to N random samples",
+        metavar="N",
+    )
     parser.add_argument("kconfig_path", type=str, help="Path to Kconfig file")
     parser.add_argument(
         "model",
@@ -64,7 +75,15 @@ def main():
 
     if os.path.isdir(args.model):
         data = KConfigAttributes(args.kconfig_path, args.model)
-        model = KConfigModel.from_benchmark(data, args.attribute)
+
+        if args.sample_size:
+            shuffled_data_indices = np.random.permutation(np.arange(len(data.data)))
+            sample_indices = shuffled_data_indices[: args.sample_size]
+            model = KConfigModel.from_benchmark(
+                data, args.attribute, indices=sample_indices
+            )
+        else:
+            model = KConfigModel.from_benchmark(data, args.attribute)
         if args.max_loss:
             model.max_loss = args.max_loss
         model.build_tree()
@@ -73,6 +92,9 @@ def main():
         with open(args.model, "r") as f:
             model = KConfigModel.from_json(json.load(f))
 
+    if args.info:
+        print("TODO")
+
     if args.export_tree:
         with open(args.export_tree, "w") as f:
             json.dump(model.to_json(), f)