From 52a0aaaca1427d5f3b35625f681dc44e01f4d8db Mon Sep 17 00:00:00 2001
From: Daniel Friesel <daniel.friesel@uos.de>
Date: Thu, 24 Feb 2022 15:53:23 +0100
Subject: add --export-xv option for t-tests and the like

---
 bin/analyze-archive.py | 1 +
 bin/analyze-kconfig.py | 1 +
 lib/cli.py             | 6 ++++++
 lib/validation.py      | 7 +++++++
 4 files changed, 15 insertions(+)

diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py
index e9e5171..a5706de 100755
--- a/bin/analyze-archive.py
+++ b/bin/analyze-archive.py
@@ -646,6 +646,7 @@ if __name__ == "__main__":
             PTAModel, by_name, parameters, arg_count, force_tree=args.force_tree
         )
         xv.parameter_aware = args.parameter_aware_cross_validation
+        xv.export_filename = args.export_xv
     else:
         xv_method = None
 
diff --git a/bin/analyze-kconfig.py b/bin/analyze-kconfig.py
index 565203b..0bf6ff8 100755
--- a/bin/analyze-kconfig.py
+++ b/bin/analyze-kconfig.py
@@ -214,6 +214,7 @@ def main():
             max_std=max_std,
         )
         xv.parameter_aware = args.parameter_aware_cross_validation
+        xv.export_filename = args.export_xv
     else:
         xv_method = None
 
diff --git a/lib/cli.py b/lib/cli.py
index 0de5505..3af8cc1 100644
--- a/lib/cli.py
+++ b/lib/cli.py
@@ -177,6 +177,12 @@ def add_standard_arguments(parser):
         type=str,
         help="Export model and model quality to LaTeX dataref file",
     )
+    parser.add_argument(
+        "--export-xv",
+        metavar="FILE",
+        type=str,
+        help="Export raw cross-validation results to FILE for later analysis (e.g. to compare different modeling approaches by means of a t-test)",
+    )
     parser.add_argument(
         "--info",
         action="store_true",
diff --git a/lib/validation.py b/lib/validation.py
index 89bc67c..6e8fa17 100644
--- a/lib/validation.py
+++ b/lib/validation.py
@@ -103,6 +103,7 @@ class CrossValidator:
         self.names = sorted(by_name.keys())
         self.parameters = sorted(parameters)
         self.parameter_aware = False
+        self.export_filename = None
         self.args = args
         self.kwargs = kwargs
 
@@ -233,6 +234,12 @@ class CrossValidator:
                             res[name][attribute][measure]
                         )
 
+        if self.export_filename:
+            import json
+
+            with open(self.export_filename, "w") as f:
+                json.dump(ret, f)
+
         for name in self.names:
             for attribute in self.by_name[name]["attributes"]:
                 for measure in ("mae", "rmsd", "mape", "smape"):
-- 
cgit v1.2.3