diff options
| -rwxr-xr-x | bin/analyze-archive.py | 1 | ||||
| -rwxr-xr-x | bin/analyze-kconfig.py | 1 | ||||
| -rw-r--r-- | lib/cli.py | 6 | ||||
| -rw-r--r-- | lib/validation.py | 7 |
4 files changed, 15 insertions, 0 deletions
diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index e9e5171..a5706de 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -646,6 +646,7 @@ if __name__ == "__main__": PTAModel, by_name, parameters, arg_count, force_tree=args.force_tree ) xv.parameter_aware = args.parameter_aware_cross_validation + xv.export_filename = args.export_xv else: xv_method = None diff --git a/bin/analyze-kconfig.py b/bin/analyze-kconfig.py index 565203b..0bf6ff8 100755 --- a/bin/analyze-kconfig.py +++ b/bin/analyze-kconfig.py @@ -214,6 +214,7 @@ def main(): max_std=max_std, ) xv.parameter_aware = args.parameter_aware_cross_validation + xv.export_filename = args.export_xv else: xv_method = None @@ -178,6 +178,12 @@ def add_standard_arguments(parser): help="Export model and model quality to LaTeX dataref file", ) parser.add_argument( + "--export-xv", + metavar="FILE", + type=str, + help="Export raw cross-validation results to FILE for later analysis (e.g. to compare different modeling approaches by means of a t-test)", + ) + parser.add_argument( "--info", action="store_true", help="Show benchmark information (number of measurements, parameter values, ...)", diff --git a/lib/validation.py b/lib/validation.py index 89bc67c..6e8fa17 100644 --- a/lib/validation.py +++ b/lib/validation.py @@ -103,6 +103,7 @@ class CrossValidator: self.names = sorted(by_name.keys()) self.parameters = sorted(parameters) self.parameter_aware = False + self.export_filename = None self.args = args self.kwargs = kwargs @@ -233,6 +234,12 @@ class CrossValidator: res[name][attribute][measure] ) + if self.export_filename: + import json + + with open(self.export_filename, "w") as f: + json.dump(ret, f) + for name in self.names: for attribute in self.by_name[name]["attributes"]: for measure in ("mae", "rmsd", "mape", "smape"): |