analyze-kconfig: add dataref export

2 files changed, 88 insertions, 19 deletions

diff --git a/bin/analyze-kconfig.py b/bin/analyze-kconfig.py
index 27540cf..56b2925 100755
--- a/bin/analyze-kconfig.py
+++ b/bin/analyze-kconfig.py
@@ -53,12 +53,23 @@ def main():
         help="Export observations (intermediate and generic benchmark data representation) to FILE",
     )
     parser.add_argument(
+        "--export-observations-only",
+        action="store_true",
+        help="Exit after exporting observations",
+    )
+    parser.add_argument(
         "--export-model",
         type=str,
         help="Export kconfig-webconf NFP model to file",
         metavar="FILE",
     )
     parser.add_argument(
+        "--export-dref",
+        type=str,
+        help="Export model and model quality to LaTeX dataref file",
+        metavar="FILE",
+    )
+    parser.add_argument(
         "--config",
         type=str,
         help="Show model results for symbols in .config file",
@@ -129,8 +140,13 @@ def main():
         if args.export_observations:
             import lzma
 
+            print(
+                f"Exporting {len(observations)} observations to {args.export_observations}"
+            )
             with lzma.open(args.export_observations, "wt") as f:
                 json.dump(observations, f)
+            if args.export_observations_only:
+                return
     else:
         # show-failing-symbols, show-nop-symbols, DFATOOL_KCONF_WITH_CHOICE_NODES, DFATOOL_KCONF_IGNORE_NUMERIC, and DFATOOL_KCONF_IGNORE_STRING have no effect
         # in this branch.
@@ -142,7 +158,7 @@ def main():
     by_name, parameter_names = dfatool.utils.observations_to_by_name(observations)
 
     # Release memory
-    observations = None
+    del observations
 
     if args.max_std:
         max_std = dict()
@@ -183,14 +199,41 @@ def main():
     else:
         xv_method = None
 
+    static_model = model.get_static()
+    try:
+        lut_model = model.get_param_lut()
+    except RuntimeError as e:
+        if args.force_tree:
+            # this is to be expected
+            logging.debug(f"Skipping LUT model: {e}")
+        else:
+            logging.warning(f"Skipping LUT model: {e}")
+        lut_model = None
     param_model, param_info = model.get_fitted()
 
     if xv_method == "montecarlo":
+        static_quality = xv.montecarlo(lambda m: m.get_static(), xv_count)
         analytic_quality = xv.montecarlo(lambda m: m.get_fitted()[0], xv_count)
+        if lut_model:
+            lut_quality = xv.montecarlo(
+                lambda m: m.get_param_lut(fallback=True), xv_count
+            )
+        else:
+            lut_quality = None
     elif xv_method == "kfold":
+        static_quality = xv.kfold(lambda m: m.get_static(), xv_count)
         analytic_quality = xv.kfold(lambda m: m.get_fitted()[0], xv_count)
+        if lut_model:
+            lut_quality = xv.kfold(lambda m: m.get_param_lut(fallback=True), xv_count)
+        else:
+            lut_quality = None
     else:
+        static_quality = model.assess(static_model)
         analytic_quality = model.assess(param_model)
+        if lut_model:
+            lut_quality = model.assess(lut_model)
+        else:
+            lut_quality = None
 
     print("Model Error on Training Data:")
     for name in model.names:
@@ -221,6 +264,25 @@ def main():
         with open(args.export_model, "w") as f:
             json.dump(json_model, f, sort_keys=True, cls=dfatool.utils.NpEncoder)
 
+    if xv_method == "montecarlo":
+        static_quality = xv.montecarlo(lambda m: m.get_static(), xv_count)
+    elif xv_method == "kfold":
+        static_quality = xv.kfold(lambda m: m.get_static(), xv_count)
+    else:
+        static_quality = model.assess(static_model)
+
+    if args.export_dref:
+        dref = model.to_dref(static_quality, lut_quality, analytic_quality)
+        with open(args.export_dref, "w") as f:
+            for k, v in dref.items():
+                if type(v) is not tuple:
+                    v = (v, None)
+                if v[1] is None:
+                    prefix = r"\drefset{"
+                else:
+                    prefix = r"\drefset" + f"[unit={v[1]}]" + "{"
+                print(f"{prefix}/{k}" + "}{" + str(v[0]) + "}", file=f)
+
     if args.config:
         kconf = kconfiglib.Kconfig(args.kconfig_path)
         kconf.load_config(args.config)
diff --git a/lib/model.py b/lib/model.py
index b6318d7..ffd29b9 100644
--- a/lib/model.py
+++ b/lib/model.py
@@ -231,6 +231,10 @@ class AnalyticModel:
             for k, v in attr.items():
                 static_model[name][k] = v.get_static(use_mean=use_mean)
                 lut_model[name][k] = dict()
+                if not v.by_param:
+                    raise RuntimeError(
+                        f"ModelAttribute({name}, {k}).by_param is None. Did you run ParallelParamStats.compute?"
+                    )
                 for param, model_value in v.by_param.items():
                     lut_model[name][k][param] = v.get_lut(param, use_mean=use_mean)
 
@@ -408,9 +412,6 @@ class AnalyticModel:
         ret = dict()
         for name in self.names:
             for attr_name, attr in self.attr_by_name[name].items():
-                e_static = static_quality[name][attr_name]
-                e_lut = lut_quality[name][attr_name]
-                e_model = model_quality[name][attr_name]
                 unit = None
                 if "power" in attr.attr:
                     unit = r"\micro\watt"
@@ -420,21 +421,12 @@ class AnalyticModel:
                     unit = r"\micro\second"
                 for k, v in attr.to_dref(unit).items():
                     ret[f"data/{name}/{attr_name}/{k}"] = v
+                e_static = static_quality[name][attr_name]
                 ret[f"error/static/{name}/{attr_name}/mae"] = (e_static["mae"], unit)
-                ret[f"error/lut/{name}/{attr_name}/mae"] = (e_lut["mae"], unit)
-                ret[f"error/model/{name}/{attr_name}/mae"] = (e_model["mae"], unit)
                 ret[f"error/static/{name}/{attr_name}/smape"] = (
                     e_static["smape"],
                     r"\percent",
                 )
-                ret[f"error/lut/{name}/{attr_name}/smape"] = (
-                    e_lut["smape"],
-                    r"\percent",
-                )
-                ret[f"error/model/{name}/{attr_name}/smape"] = (
-                    e_model["smape"],
-                    r"\percent",
-                )
                 try:
                     ret[f"error/static/{name}/{attr_name}/mape"] = (
                         e_static["mape"],
@@ -442,13 +434,28 @@ class AnalyticModel:
                     )
                 except KeyError:
                     logger.warning(f"{name} {attr_name} static model has no MAPE")
-                try:
-                    ret[f"error/lut/{name}/{attr_name}/mape"] = (
-                        e_lut["mape"],
+
+                if lut_quality is not None:
+                    e_lut = lut_quality[name][attr_name]
+                    ret[f"error/lut/{name}/{attr_name}/mae"] = (e_lut["mae"], unit)
+                    ret[f"error/lut/{name}/{attr_name}/smape"] = (
+                        e_lut["smape"],
                         r"\percent",
                     )
-                except KeyError:
-                    logger.warning(f"{name} {attr_name} LUT model has no MAPE")
+                    try:
+                        ret[f"error/lut/{name}/{attr_name}/mape"] = (
+                            e_lut["mape"],
+                            r"\percent",
+                        )
+                    except KeyError:
+                        logger.warning(f"{name} {attr_name} LUT model has no MAPE")
+
+                e_model = model_quality[name][attr_name]
+                ret[f"error/model/{name}/{attr_name}/mae"] = (e_model["mae"], unit)
+                ret[f"error/model/{name}/{attr_name}/smape"] = (
+                    e_model["smape"],
+                    r"\percent",
+                )
                 try:
                     ret[f"error/model/{name}/{attr_name}/mape"] = (
                         e_model["mape"],