Merge branch 'master' into decisiontrees

7 files changed, 687 insertions, 158 deletions

diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py
index 4c442af..5a6b8f0 100755
--- a/bin/analyze-archive.py
+++ b/bin/analyze-archive.py
@@ -101,6 +101,9 @@ Options:
 
 --export-energymodel=<model.json>
     Export energy model. Works out of the box for v1 and v2 logfiles. Requires --hwmodel for v0 logfiles.
+
+--no-cache
+    Do not load cached measurement results
 """
 
 import getopt
@@ -142,6 +145,15 @@ def format_quality_measures(result):
 
 
 def model_quality_table(result_lists, info_list):
+    print(
+        "{:20s} {:15s}       {:19s}       {:19s}       {:19s}".format(
+            "key",
+            "attribute",
+            "static".center(19),
+            "parameterized".center(19),
+            "LUT".center(19),
+        )
+    )
     for state_or_tran in result_lists[0]["by_name"].keys():
         for key in result_lists[0]["by_name"][state_or_tran].keys():
             buf = "{:20s} {:15s}".format(state_or_tran, key)
@@ -152,7 +164,7 @@ def model_quality_table(result_lists, info_list):
                     result = results["by_name"][state_or_tran][key]
                     buf += format_quality_measures(result)
                 else:
-                    buf += "{:6}----{:9}".format("", "")
+                    buf += "{:7}----{:8}".format("", "")
             print(buf)
 
 
@@ -300,7 +312,7 @@ if __name__ == "__main__":
 
     try:
         optspec = (
-            "info "
+            "info no-cache "
             "plot-unparam= plot-param= plot-traces= show-models= show-quality= "
             "ignored-trace-indexes= function-override= "
             "export-traces= "
@@ -362,11 +374,18 @@ if __name__ == "__main__":
         sys.exit(2)
 
     raw_data = RawData(
-        args, with_traces=("export-traces" in opt or "plot-traces" in opt)
+        args,
+        with_traces=("export-traces" in opt or "plot-traces" in opt),
+        skip_cache=("no-cache" in opt),
     )
 
     if "info" in opt:
         print(" ".join(raw_data.filenames) + ":")
+        if raw_data.ptalog:
+            options = " --".join(
+                map(lambda kv: f"{kv[0]}={str(kv[1])}", raw_data.ptalog["opt"].items())
+            )
+            print(f"    Options: --{options}")
         if raw_data.version <= 1:
             data_source = "MIMOSA"
         elif raw_data.version == 2:
@@ -420,7 +439,7 @@ if __name__ == "__main__":
             )
             sys.exit(2)
 
-        if len(traces) > 20:
+        if len(traces) > 40:
             print(f"""Truncating plot to 40 of {len(traces)} traces (random sample)""")
             traces = random.sample(traces, 40)
 
@@ -693,7 +712,7 @@ if __name__ == "__main__":
         )
 
     if "overall" in show_quality or "all" in show_quality:
-        print("overall static/param/lut MAE assuming equal state distribution:")
+        print("overall state static/param/lut MAE assuming equal state distribution:")
         print(
             "    {:6.1f}  /  {:6.1f}  /  {:6.1f}  µW".format(
                 model.assess_states(static_model),
@@ -701,15 +720,30 @@ if __name__ == "__main__":
                 model.assess_states(lut_model),
             )
         )
-        print("overall static/param/lut MAE assuming 95% STANDBY1:")
-        distrib = {"STANDBY1": 0.95, "POWERDOWN": 0.03, "TX": 0.01, "RX": 0.01}
-        print(
-            "    {:6.1f}  /  {:6.1f}  /  {:6.1f}  µW".format(
-                model.assess_states(static_model, distribution=distrib),
-                model.assess_states(param_model, distribution=distrib),
-                model.assess_states(lut_model, distribution=distrib),
+        distrib = dict()
+        num_states = len(model.states())
+        p95_state = None
+        for state in model.states():
+            distrib[state] = 1.0 / num_states
+
+        if "STANDBY1" in model.states():
+            p95_state = "STANDBY1"
+        elif "SLEEP" in model.states():
+            p95_state = "SLEEP"
+
+        if p95_state is not None:
+            for state in distrib.keys():
+                distrib[state] = 0.05 / (num_states - 1)
+            distrib[p95_state] = 0.95
+
+            print(f"overall state static/param/lut MAE assuming 95% {p95_state}:")
+            print(
+                "    {:6.1f}  /  {:6.1f}  /  {:6.1f}  µW".format(
+                    model.assess_states(static_model, distribution=distrib),
+                    model.assess_states(param_model, distribution=distrib),
+                    model.assess_states(lut_model, distribution=distrib),
+                )
             )
-        )
 
     if "summary" in show_quality or "all" in show_quality:
         model_summary_table(
diff --git a/bin/generate-dfa-benchmark.py b/bin/generate-dfa-benchmark.py
index 1410c28..64f8f73 100755
--- a/bin/generate-dfa-benchmark.py
+++ b/bin/generate-dfa-benchmark.py
@@ -223,17 +223,11 @@ def benchmark_from_runs(
                     )
             elif opt["sleep"]:
                 if "energytrace" in opt:
-                    outbuf.write(
-                        "arch.sleep_ms({:d}); // {}\n".format(
-                            opt["sleep"], transition.destination.name
-                        )
-                    )
+                    outbuf.write(f"// -> {transition.destination.name}\n")
+                    outbuf.write(target.sleep_ms(opt["sleep"]))
                 else:
-                    outbuf.write(
-                        "arch.delay_ms({:d}); // {}\n".format(
-                            opt["sleep"], transition.destination.name
-                        )
-                    )
+                    outbuf.write(f"// -> {transition.destination.name}\n")
+                    outbuf.write("arch.delay_ms({:d});\n".format(opt["sleep"]))
 
         outbuf.write(harness.stop_run(num_traces))
         if dummy:
@@ -289,7 +283,7 @@ def run_benchmark(
         needs_split = True
     else:
         try:
-            runner.build(arch, app, run_args)
+            target.build(app, run_args)
         except RuntimeError:
             if len(runs) > 50:
                 # Application is too large -> split up runs
@@ -342,14 +336,14 @@ def run_benchmark(
         i = 0
         while i < opt["repeat"]:
             print(f"""[RUN] flashing benchmark {i+1}/{opt["repeat"]}""")
-            runner.flash(arch, app, run_args)
+            target.flash(app, run_args)
             if "mimosa" in opt:
-                monitor = runner.get_monitor(
-                    arch, callback=harness.parser_cb, mimosa=opt["mimosa"]
+                monitor = target.get_monitor(
+                    callback=harness.parser_cb, mimosa=opt["mimosa"]
                 )
             elif "energytrace" in opt:
-                monitor = runner.get_monitor(
-                    arch, callback=harness.parser_cb, energytrace=opt["energytrace"]
+                monitor = target.get_monitor(
+                    callback=harness.parser_cb, energytrace=opt["energytrace"]
                 )
 
             sync_error = False
@@ -400,8 +394,8 @@ def run_benchmark(
 
         return [(runs, harness, monitor, files)]
     else:
-        runner.flash(arch, app, run_args)
-        monitor = runner.get_monitor(arch, callback=harness.parser_cb)
+        target.flash(app, run_args)
+        monitor = target.get_monitor(callback=harness.parser_cb)
 
         if arch == "posix":
             print("[RUN] Will run benchmark for {:.0f} seconds".format(run_timeout))
@@ -518,6 +512,11 @@ if __name__ == "__main__":
         print(err)
         sys.exit(2)
 
+    if "msp430fr" in opt["arch"]:
+        target = runner.Arch(opt["arch"], ["cpu_freq=8000000"])
+    else:
+        target = runner.Arch(opt["arch"])
+
     modelfile = args[0]
 
     pta = PTA.from_file(modelfile)
@@ -594,8 +593,8 @@ if __name__ == "__main__":
         if "codegen" in driver_definition and "flags" in driver_definition["codegen"]:
             if run_flags is None:
                 run_flags = driver_definition["codegen"]["flags"]
-    if run_flags is None:
-        run_flags = opt["run"].split()
+    if "run" in opt:
+        run_flags.extend(opt["run"].split())
 
     runs = list(
         pta.dfs(
@@ -642,7 +641,7 @@ if __name__ == "__main__":
             gpio_pin=timer_pin,
             gpio_mode=gpio_mode,
             pta=pta,
-            counter_limits=runner.get_counter_limits_us(opt["arch"]),
+            counter_limits=target.get_counter_limits_us(run_flags),
             log_return_values=need_return_values,
             repeat=1,
         )
@@ -650,7 +649,7 @@ if __name__ == "__main__":
         harness = OnboardTimerHarness(
             gpio_pin=timer_pin,
             pta=pta,
-            counter_limits=runner.get_counter_limits_us(opt["arch"]),
+            counter_limits=target.get_counter_limits_us(run_flags),
             log_return_values=need_return_values,
             repeat=opt["repeat"],
         )
diff --git a/lib/loader.py b/lib/loader.py
index c35eb4c..57b3d30 100644
--- a/lib/loader.py
+++ b/lib/loader.py
@@ -242,7 +242,7 @@ class RawData:
     file system, making subsequent loads near-instant.
     """
 
-    def __init__(self, filenames, with_traces=False):
+    def __init__(self, filenames, with_traces=False, skip_cache=False):
         """
         Create a new RawData object.
 
@@ -321,7 +321,7 @@ class RawData:
                 self.pta = self.ptalog["pta"]
 
         self.set_cache_file()
-        if not with_traces:
+        if not with_traces and not skip_cache:
             self.load_cache()
 
     def set_cache_file(self):
@@ -481,7 +481,7 @@ class RawData:
             if sorted(online_trace_part["parameter"].keys()) != self._parameter_names:
                 processed_data[
                     "error"
-                ] = "Offline #{off_idx:d} (online {on_name:s} @ {on_idx:d}/{on_sub:d}) has inconsistent parameter set: should be {param_want:s}, is {param_is:s}".format(
+                ] = "Offline #{off_idx:d} (online {on_name:s} @ {on_idx:d}/{on_sub:d}) has inconsistent parameter set: should be {param_want}, is {param_is}".format(
                     off_idx=offline_idx,
                     on_idx=online_run_idx,
                     on_sub=online_trace_part_idx,
diff --git a/lib/model.py b/lib/model.py
index f53f645..41cf726 100644
--- a/lib/model.py
+++ b/lib/model.py
@@ -5,6 +5,7 @@ import numpy as np
 from scipy import optimize
 from sklearn.metrics import r2_score
 from multiprocessing import Pool
+from .automata import PTA
 from .functions import analytic
 from .functions import AnalyticFunction
 from .parameters import ParamStats
@@ -1111,13 +1112,16 @@ class PTAModel:
         static_quality = self.assess(static_model)
         param_model, param_info = self.get_fitted()
         analytic_quality = self.assess(param_model)
-        self.pta.update(
+        pta = self.pta
+        if pta is None:
+            pta = PTA(self.states(), parameters=self._parameter_names)
+        pta.update(
             static_model,
             param_info,
             static_error=static_quality["by_name"],
             analytic_error=analytic_quality["by_name"],
         )
-        return self.pta.to_json()
+        return pta.to_json()
 
     def states(self):
         """Return sorted list of state names."""
diff --git a/lib/runner.py b/lib/runner.py
index aeb8600..71ca799 100644
--- a/lib/runner.py
+++ b/lib/runner.py
@@ -311,113 +311,157 @@ class ShellMonitor:
         pass
 
 
-def build(arch, app, opts=[]):
-    command = ["make", "arch={}".format(arch), "app={}".format(app), "clean"]
-    command.extend(opts)
-    res = subprocess.run(
-        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True
-    )
-    if res.returncode != 0:
-        raise RuntimeError(
-            "Build failure, executing {}:\n".format(command) + res.stderr
+class Arch:
+    def __init__(self, name, opts=list()):
+        self.name = name
+        self.opts = opts
+        self.info = self.get_info()
+
+    def build(self, app, opts=list()):
+        command = ["make", "arch={}".format(self.name), "app={}".format(app), "clean"]
+        command.extend(self.opts)
+        command.extend(opts)
+        res = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
         )
-    command = ["make", "-B", "arch={}".format(arch), "app={}".format(app)]
-    command.extend(opts)
-    res = subprocess.run(
-        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True
-    )
-    if res.returncode != 0:
-        raise RuntimeError(
-            "Build failure, executing {}:\n ".format(command) + res.stderr
+        if res.returncode != 0:
+            raise RuntimeError(
+                "Build failure, executing {}:\n".format(command) + res.stderr
+            )
+        command = ["make", "-B", "arch={}".format(self.name), "app={}".format(app)]
+        command.extend(self.opts)
+        command.extend(opts)
+        res = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
         )
-    return command
-
-
-def flash(arch, app, opts=[]):
-    command = ["make", "arch={}".format(arch), "app={}".format(app), "program"]
-    command.extend(opts)
-    res = subprocess.run(
-        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True
-    )
-    if res.returncode != 0:
-        raise RuntimeError("Flash failure")
-    return command
+        if res.returncode != 0:
+            raise RuntimeError(
+                "Build failure, executing {}:\n ".format(command) + res.stderr
+            )
+        return command
 
+    def flash(self, app, opts=list()):
+        command = ["make", "arch={}".format(self.name), "app={}".format(app), "program"]
+        command.extend(self.opts)
+        command.extend(opts)
+        res = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        if res.returncode != 0:
+            raise RuntimeError("Flash failure")
+        return command
 
-def get_info(arch, opts: list = []) -> list:
-    """
-    Return multipass "make info" output.
+    def get_info(self, opts=list()) -> list:
+        """
+        Return multipass "make info" output.
 
-    Returns a list.
-    """
-    command = ["make", "arch={}".format(arch), "info"]
-    command.extend(opts)
-    res = subprocess.run(
-        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True
-    )
-    if res.returncode != 0:
-        raise RuntimeError("make info Failure")
-    return res.stdout.split("\n")
+        Returns a list.
+        """
+        command = ["make", "arch={}".format(self.name), "info"]
+        command.extend(self.opts)
+        command.extend(opts)
+        res = subprocess.run(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+        if res.returncode != 0:
+            raise RuntimeError("make info Failure")
+        return res.stdout.split("\n")
 
+    def _cached_info(self, opts=list()) -> list:
+        if len(opts):
+            return self.get_info(opts)
+        return self.info
 
-def get_monitor(arch: str, **kwargs) -> object:
-    """
-    Return an appropriate monitor for arch, depending on "make info" output.
+    def get_monitor(self, **kwargs) -> object:
+        """
+        Return an appropriate monitor for arch, depending on "make info" output.
 
-    Port and Baud rate are taken from "make info".
+        Port and Baud rate are taken from "make info".
 
-    :param arch: architecture name, e.g. 'msp430fr5994lp' or 'posix'
-    :param energytrace: `EnergyTraceMonitor` options. Returns an EnergyTrace monitor if not None.
-    :param mimosa: `MIMOSAMonitor` options. Returns a MIMOSA monitor if not None.
-    """
-    for line in get_info(arch):
-        if "Monitor:" in line:
-            _, port, arg = line.split(" ")
-            if port == "run":
-                return ShellMonitor(arg, **kwargs)
-            elif "mimosa" in kwargs and kwargs["mimosa"] is not None:
-                mimosa_kwargs = kwargs.pop("mimosa")
-                return MIMOSAMonitor(port, arg, **mimosa_kwargs, **kwargs)
-            elif "energytrace" in kwargs and kwargs["energytrace"] is not None:
-                energytrace_kwargs = kwargs.pop("energytrace").copy()
-                sync_mode = energytrace_kwargs.pop("sync")
-                if sync_mode == "la":
-                    return EnergyTraceLogicAnalyzerMonitor(
-                        port, arg, **energytrace_kwargs, **kwargs
-                    )
+        :param energytrace: `EnergyTraceMonitor` options. Returns an EnergyTrace monitor if not None.
+        :param mimosa: `MIMOSAMonitor` options. Returns a MIMOSA monitor if not None.
+        """
+        for line in self.info:
+            if "Monitor:" in line:
+                _, port, arg = line.split(" ")
+                if port == "run":
+                    return ShellMonitor(arg, **kwargs)
+                elif "mimosa" in kwargs and kwargs["mimosa"] is not None:
+                    mimosa_kwargs = kwargs.pop("mimosa")
+                    return MIMOSAMonitor(port, arg, **mimosa_kwargs, **kwargs)
+                elif "energytrace" in kwargs and kwargs["energytrace"] is not None:
+                    energytrace_kwargs = kwargs.pop("energytrace").copy()
+                    sync_mode = energytrace_kwargs.pop("sync")
+                    if sync_mode == "la":
+                        return EnergyTraceLogicAnalyzerMonitor(
+                            port, arg, **energytrace_kwargs, **kwargs
+                        )
+                    else:
+                        return EnergyTraceMonitor(
+                            port, arg, **energytrace_kwargs, **kwargs
+                        )
                 else:
-                    return EnergyTraceMonitor(port, arg, **energytrace_kwargs, **kwargs)
+                    kwargs.pop("energytrace", None)
+                    kwargs.pop("mimosa", None)
+                    return SerialMonitor(port, arg, **kwargs)
+        raise RuntimeError("Monitor failure")
+
+    def get_counter_limits(self, opts=list()) -> tuple:
+        """Return multipass max counter and max overflow value for arch."""
+        for line in self._cached_info(opts):
+            match = re.match("Counter Overflow: ([^/]*)/(.*)", line)
+            if match:
+                overflow_value = int(match.group(1))
+                max_overflow = int(match.group(2))
+                return overflow_value, max_overflow
+        raise RuntimeError("Did not find Counter Overflow limits")
+
+    def sleep_ms(self, duration: int, opts=list()) -> str:
+        max_sleep = None
+        if "msp430fr" in self.name:
+            cpu_freq = None
+            for line in self._cached_info(opts):
+                match = re.match(r"CPU\s+Freq:\s+(.*)\s+Hz", line)
+                if match:
+                    cpu_freq = int(match.group(1))
+            if cpu_freq is not None and cpu_freq > 8000000:
+                max_sleep = 250
             else:
-                kwargs.pop("energytrace", None)
-                kwargs.pop("mimosa", None)
-                return SerialMonitor(port, arg, **kwargs)
-    raise RuntimeError("Monitor failure")
-
-
-def get_counter_limits(arch: str) -> tuple:
-    """Return multipass max counter and max overflow value for arch."""
-    for line in get_info(arch):
-        match = re.match("Counter Overflow: ([^/]*)/(.*)", line)
-        if match:
-            overflow_value = int(match.group(1))
-            max_overflow = int(match.group(2))
-            return overflow_value, max_overflow
-    raise RuntimeError("Did not find Counter Overflow limits")
-
-
-def get_counter_limits_us(arch: str) -> tuple:
-    """Return duration of one counter step and one counter overflow in us."""
-    cpu_freq = 0
-    overflow_value = 0
-    max_overflow = 0
-    for line in get_info(arch):
-        match = re.match(r"CPU\s+Freq:\s+(.*)\s+Hz", line)
-        if match:
-            cpu_freq = int(match.group(1))
-        match = re.match(r"Counter Overflow:\s+([^/]*)/(.*)", line)
-        if match:
-            overflow_value = int(match.group(1))
-            max_overflow = int(match.group(2))
-    if cpu_freq and overflow_value:
-        return 1000000 / cpu_freq, overflow_value * 1000000 / cpu_freq, max_overflow
-    raise RuntimeError("Did not find Counter Overflow limits")
+                max_sleep = 500
+        if max_sleep is not None and duration > max_sleep:
+            sub_sleep_count = duration // max_sleep
+            tail_sleep = duration % max_sleep
+            ret = f"for (unsigned char i = 0; i < {sub_sleep_count}; i++) {{ arch.sleep_ms({max_sleep}); }}\n"
+            if tail_sleep > 0:
+                ret += f"arch.sleep_ms({tail_sleep});\n"
+            return ret
+        return f"arch.sleep_ms({duration});\n"
+
+    def get_counter_limits_us(self, opts=list()) -> tuple:
+        """Return duration of one counter step and one counter overflow in us."""
+        cpu_freq = 0
+        overflow_value = 0
+        max_overflow = 0
+        for line in self._cached_info(opts):
+            match = re.match(r"CPU\s+Freq:\s+(.*)\s+Hz", line)
+            if match:
+                cpu_freq = int(match.group(1))
+            match = re.match(r"Counter Overflow:\s+([^/]*)/(.*)", line)
+            if match:
+                overflow_value = int(match.group(1))
+                max_overflow = int(match.group(2))
+        if cpu_freq and overflow_value:
+            return 1000000 / cpu_freq, overflow_value * 1000000 / cpu_freq, max_overflow
+        raise RuntimeError("Did not find Counter Overflow limits")
diff --git a/lib/validation.py b/lib/validation.py
index 98d49c1..ee147fe 100644
--- a/lib/validation.py
+++ b/lib/validation.py
@@ -179,6 +179,7 @@ class CrossValidator:
             for attribute in self.by_name[name]["attributes"]:
                 ret["by_name"][name][attribute] = {
                     "mae_list": list(),
+                    "rmsd_list": list(),
                     "smape_list": list(),
                 }
 
@@ -186,21 +187,17 @@ class CrossValidator:
             res = self._single_xv(model_getter, training_and_validation_by_name)
             for name in self.names:
                 for attribute in self.by_name[name]["attributes"]:
-                    ret["by_name"][name][attribute]["mae_list"].append(
-                        res["by_name"][name][attribute]["mae"]
-                    )
-                    ret["by_name"][name][attribute]["smape_list"].append(
-                        res["by_name"][name][attribute]["smape"]
-                    )
+                    for measure in ("mae", "rmsd", "smape"):
+                        ret["by_name"][name][attribute][f"{measure}_list"].append(
+                            res["by_name"][name][attribute][measure]
+                        )
 
         for name in self.names:
             for attribute in self.by_name[name]["attributes"]:
-                ret["by_name"][name][attribute]["mae"] = np.mean(
-                    ret["by_name"][name][attribute]["mae_list"]
-                )
-                ret["by_name"][name][attribute]["smape"] = np.mean(
-                    ret["by_name"][name][attribute]["smape_list"]
-                )
+                for measure in ("mae", "rmsd", "smape"):
+                    ret["by_name"][name][attribute][measure] = np.mean(
+                        ret["by_name"][name][attribute][f"{measure}_list"]
+                    )
 
         return ret
 
diff --git a/test/test_ptamodel.py b/test/test_ptamodel.py
index 94ee842..e8905b1 100755
--- a/test/test_ptamodel.py
+++ b/test/test_ptamodel.py
@@ -2,13 +2,464 @@
 
 from dfatool.loader import RawData, pta_trace_to_aggregate
 from dfatool.model import PTAModel
+from dfatool.utils import by_name_to_by_param
+from dfatool.validation import CrossValidator
 import os
 import unittest
 import pytest
 
+import numpy as np
 
-class TestModels(unittest.TestCase):
-    def test_model_singlefile_rf24(self):
+
+class TestSynthetic(unittest.TestCase):
+    def test_model_validation(self):
+        # rng = np.random.default_rng(seed=1312) # requiresy NumPy >= 1.17
+        np.random.seed(1312)
+        X = np.arange(500) % 50
+        parameter_names = ["p_mod5", "p_linear"]
+
+        s1_duration_base = 70
+        s1_duration_scale = 2
+        s1_power_base = 50
+        s1_power_scale = 7
+        s2_duration_base = 700
+        s2_duration_scale = 1
+        s2_power_base = 1500
+        s2_power_scale = 10
+
+        by_name = {
+            "raw_state_1": {
+                "isa": "state",
+                "param": [(x % 5, x) for x in X],
+                "duration": s1_duration_base
+                + np.random.normal(size=X.size, scale=s1_duration_scale),
+                "power": s1_power_base
+                + X
+                + np.random.normal(size=X.size, scale=s1_power_scale),
+                "attributes": ["duration", "power"],
+            },
+            "raw_state_2": {
+                "isa": "state",
+                "param": [(x % 5, x) for x in X],
+                "duration": s2_duration_base
+                - 2 * X
+                + np.random.normal(size=X.size, scale=s2_duration_scale),
+                "power": s2_power_base
+                + X
+                + np.random.normal(size=X.size, scale=s2_power_scale),
+                "attributes": ["duration", "power"],
+            },
+        }
+        by_param = by_name_to_by_param(by_name)
+        model = PTAModel(by_name, parameter_names, dict())
+        static_model = model.get_static()
+
+        # x ∈ [0, 50] -> mean(X) is 25
+        self.assertAlmostEqual(
+            static_model("raw_state_1", "duration"), s1_duration_base, places=0
+        )
+        self.assertAlmostEqual(
+            static_model("raw_state_1", "power"), s1_power_base + 25, delta=7
+        )
+        self.assertAlmostEqual(
+            static_model("raw_state_2", "duration"), s2_duration_base - 2 * 25, delta=2
+        )
+        self.assertAlmostEqual(
+            static_model("raw_state_2", "power"), s2_power_base + 25, delta=7
+        )
+
+        param_model, param_info = model.get_fitted()
+
+        self.assertAlmostEqual(
+            param_model("raw_state_1", "duration", param=[0, 10]),
+            s1_duration_base,
+            places=0,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_1", "duration", param=[0, 50]),
+            s1_duration_base,
+            places=0,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_1", "duration", param=[0, 70]),
+            s1_duration_base,
+            places=0,
+        )
+
+        self.assertAlmostEqual(
+            param_model("raw_state_1", "power", param=[0, 10]),
+            s1_power_base + 10,
+            places=0,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_1", "power", param=[0, 50]),
+            s1_power_base + 50,
+            places=0,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_1", "power", param=[0, 70]),
+            s1_power_base + 70,
+            places=0,
+        )
+
+        self.assertAlmostEqual(
+            param_model("raw_state_2", "duration", param=[0, 10]),
+            s2_duration_base - 2 * 10,
+            places=0,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_2", "duration", param=[0, 50]),
+            s2_duration_base - 2 * 50,
+            places=0,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_2", "duration", param=[0, 70]),
+            s2_duration_base - 2 * 70,
+            places=0,
+        )
+
+        self.assertAlmostEqual(
+            param_model("raw_state_2", "power", param=[0, 10]),
+            s2_power_base + 10,
+            delta=50,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_2", "power", param=[0, 50]),
+            s2_power_base + 50,
+            delta=50,
+        )
+        self.assertAlmostEqual(
+            param_model("raw_state_2", "power", param=[0, 70]),
+            s2_power_base + 70,
+            delta=50,
+        )
+
+        static_quality = model.assess(static_model)
+        param_quality = model.assess(param_model)
+
+        # static quality reflects normal distribution scale for non-parameterized data
+
+        # the Root Mean Square Deviation must not be greater the scale (i.e., standard deviation) of the normal distribution
+        # Low Mean Absolute Error (< 2)
+        self.assertTrue(static_quality["by_name"]["raw_state_1"]["duration"]["mae"] < 2)
+        # Low Root Mean Square Deviation (< scale == 2)
+        self.assertTrue(
+            static_quality["by_name"]["raw_state_1"]["duration"]["rmsd"] < 2
+        )
+        # Relatively low error percentage (~~ MAE * 100% / s1_duration_base)
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["duration"]["mape"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["mae"]
+            * 100
+            / s1_duration_base,
+            places=1,
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["duration"]["smape"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["mae"]
+            * 100
+            / s1_duration_base,
+            places=1,
+        )
+
+        # static error is high for parameterized data
+
+        # MAE == mean(abs(actual value - model value))
+        # parameter range is [0, 50) -> mean 25, deviation range is [0, 25) -> mean deviation is 12.5 ± gauss scale
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["power"]["mae"], 12.5, delta=1
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["power"]["rmsd"], 16, delta=2
+        )
+        # high percentage error due to low s1_power_base
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["power"]["mape"], 19, delta=2
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["power"]["smape"], 19, delta=2
+        )
+
+        # parameter range is [0, 100) -> mean deviation is 25 ± gauss scale
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["duration"]["mae"], 25, delta=2
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["duration"]["rmsd"], 30, delta=2
+        )
+
+        # low percentage error due to high s2_duration_base (~~ 3.5 %)
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["duration"]["mape"],
+            25 * 100 / s2_duration_base,
+            delta=1,
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["duration"]["smape"],
+            25 * 100 / s2_duration_base,
+            delta=1,
+        )
+
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["power"]["mae"], 12.5, delta=2
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["power"]["rmsd"], 17, delta=2
+        )
+
+        # low percentage error due to high s2_power_base (~~ 1.7 %)
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["power"]["mape"],
+            25 * 100 / s2_power_base,
+            delta=1,
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["power"]["smape"],
+            25 * 100 / s2_power_base,
+            delta=1,
+        )
+
+        # raw_state_1/duration does not depend on parameters and delegates to the static model
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["duration"]["mae"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["mae"],
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["duration"]["rmsd"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["rmsd"],
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["duration"]["mape"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["mape"],
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["duration"]["smape"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["smape"],
+        )
+
+        # fitted param-model quality reflects normal distribution scale for all data
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["power"]["mape"], 0.9, places=1
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["power"]["smape"], 0.9, places=1
+        )
+
+        self.assertTrue(
+            param_quality["by_name"]["raw_state_1"]["power"]["mae"] < s1_power_scale
+        )
+        self.assertTrue(
+            param_quality["by_name"]["raw_state_1"]["power"]["rmsd"] < s1_power_scale
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["power"]["mape"], 7.5, delta=1
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["power"]["smape"], 7.5, delta=1
+        )
+
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["duration"]["mae"],
+            s2_duration_scale,
+            delta=0.2,
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["duration"]["rmsd"],
+            s2_duration_scale,
+            delta=0.2,
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["duration"]["mape"],
+            0.12,
+            delta=0.01,
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["duration"]["smape"],
+            0.12,
+            delta=0.01,
+        )
+
+        # ... unless the signal-to-noise ratio (parameter range = [0 .. 50] vs. scale = 10) is bad, leading to
+        # increased regression errors
+        self.assertTrue(param_quality["by_name"]["raw_state_2"]["power"]["mae"] < 15)
+        self.assertTrue(param_quality["by_name"]["raw_state_2"]["power"]["rmsd"] < 18)
+
+        # still: low percentage error due to high s2_power_base
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["power"]["mape"], 0.9, places=1
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["power"]["smape"], 0.9, places=1
+        )
+
+    def test_model_crossvalidation_10fold(self):
+        # rng = np.random.default_rng(seed=1312) # requiresy NumPy >= 1.17
+        np.random.seed(1312)
+        X = np.arange(500) % 50
+        parameter_names = ["p_mod5", "p_linear"]
+
+        s1_duration_base = 70
+        s1_duration_scale = 2
+        s1_power_base = 50
+        s1_power_scale = 7
+        s2_duration_base = 700
+        s2_duration_scale = 1
+        s2_power_base = 1500
+        s2_power_scale = 10
+
+        by_name = {
+            "raw_state_1": {
+                "isa": "state",
+                "param": [(x % 5, x) for x in X],
+                "duration": s1_duration_base
+                + np.random.normal(size=X.size, scale=s1_duration_scale),
+                "power": s1_power_base
+                + X
+                + np.random.normal(size=X.size, scale=s1_power_scale),
+                "attributes": ["duration", "power"],
+            },
+            "raw_state_2": {
+                "isa": "state",
+                "param": [(x % 5, x) for x in X],
+                "duration": s2_duration_base
+                - 2 * X
+                + np.random.normal(size=X.size, scale=s2_duration_scale),
+                "power": s2_power_base
+                + X
+                + np.random.normal(size=X.size, scale=s2_power_scale),
+                "attributes": ["duration", "power"],
+            },
+        }
+        by_param = by_name_to_by_param(by_name)
+        arg_count = dict()
+        model = PTAModel(by_name, parameter_names, arg_count)
+        validator = CrossValidator(PTAModel, by_name, parameter_names, arg_count)
+
+        static_quality = validator.kfold(lambda m: m.get_static(), 10)
+        param_quality = validator.kfold(lambda m: m.get_fitted()[0], 10)
+
+        print(static_quality)
+
+        # static quality reflects normal distribution scale for non-parameterized data
+
+        # the Root Mean Square Deviation must not be greater the scale (i.e., standard deviation) of the normal distribution
+        # Low Mean Absolute Error (< 2)
+        self.assertTrue(static_quality["by_name"]["raw_state_1"]["duration"]["mae"] < 2)
+        # Low Root Mean Square Deviation (< scale == 2)
+        self.assertTrue(
+            static_quality["by_name"]["raw_state_1"]["duration"]["rmsd"] < 2
+        )
+        # Relatively low error percentage (~~ MAE * 100% / s1_duration_base)
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["duration"]["smape"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["mae"]
+            * 100
+            / s1_duration_base,
+            places=1,
+        )
+
+        # static error is high for parameterized data
+
+        # MAE == mean(abs(actual value - model value))
+        # parameter range is [0, 50) -> mean 25, deviation range is [0, 25) -> mean deviation is 12.5 ± gauss scale
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["power"]["mae"], 12.5, delta=1
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["power"]["rmsd"], 16, delta=2
+        )
+        # high percentage error due to low s1_power_base
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_1"]["power"]["smape"], 19, delta=2
+        )
+
+        # parameter range is [0, 100) -> mean deviation is 25 ± gauss scale
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["duration"]["mae"], 25, delta=2
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["duration"]["rmsd"], 30, delta=2
+        )
+
+        # low percentage error due to high s2_duration_base (~~ 3.5 %)
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["duration"]["smape"],
+            25 * 100 / s2_duration_base,
+            delta=1,
+        )
+
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["power"]["mae"], 12.5, delta=2
+        )
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["power"]["rmsd"], 17, delta=2
+        )
+
+        # low percentage error due to high s2_power_base (~~ 1.7 %)
+        self.assertAlmostEqual(
+            static_quality["by_name"]["raw_state_2"]["power"]["smape"],
+            25 * 100 / s2_power_base,
+            delta=1,
+        )
+
+        # raw_state_1/duration does not depend on parameters and delegates to the static model
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["duration"]["mae"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["mae"],
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["duration"]["rmsd"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["rmsd"],
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["duration"]["smape"],
+            static_quality["by_name"]["raw_state_1"]["duration"]["smape"],
+        )
+
+        # fitted param-model quality reflects normal distribution scale for all data
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["power"]["smape"], 0.9, places=1
+        )
+
+        self.assertTrue(
+            param_quality["by_name"]["raw_state_1"]["power"]["mae"] < s1_power_scale
+        )
+        self.assertTrue(
+            param_quality["by_name"]["raw_state_1"]["power"]["rmsd"] < s1_power_scale
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_1"]["power"]["smape"], 7.5, delta=1
+        )
+
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["duration"]["mae"],
+            s2_duration_scale,
+            delta=0.2,
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["duration"]["rmsd"],
+            s2_duration_scale,
+            delta=0.2,
+        )
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["duration"]["smape"],
+            0.12,
+            delta=0.01,
+        )
+
+        # ... unless the signal-to-noise ratio (parameter range = [0 .. 50] vs. scale = 10) is bad, leading to
+        # increased regression errors
+        self.assertTrue(param_quality["by_name"]["raw_state_2"]["power"]["mae"] < 15)
+        self.assertTrue(param_quality["by_name"]["raw_state_2"]["power"]["rmsd"] < 18)
+
+        # still: low percentage error due to high s2_power_base
+        self.assertAlmostEqual(
+            param_quality["by_name"]["raw_state_2"]["power"]["smape"], 0.9, places=1
+        )
+
+
+class TestFromFile(unittest.TestCase):
+    def test_singlefile_rf24(self):
         raw_data = RawData(["test-data/20170220_164723_RF24_int_A.tar"])
         preprocessed_data = raw_data.get_preprocessed_data()
         by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data)
@@ -162,7 +613,7 @@ class TestModels(unittest.TestCase):
             param_model("RX", "power", param=[1, None, None]), 48647, places=-1
         )
 
-    def test_model_singlefile_mmparam(self):
+    def test_singlefile_mmparam(self):
         raw_data = RawData(["test-data/20161221_123347_mmparam.tar"])
         preprocessed_data = raw_data.get_preprocessed_data()
         by_name, parameters, arg_count = pta_trace_to_aggregate(preprocessed_data)
@@ -201,7 +652,7 @@ class TestModels(unittest.TestCase):
             param_lut_model("ON", "power", param=[None, None]), 17866, places=0
         )
 
-    def test_model_multifile_lm75x(self):
+    def test_multifile_lm75x(self):
         testfiles = [
             "test-data/20170116_124500_LM75x.tar",
             "test-data/20170116_131306_LM75x.tar",
@@ -243,7 +694,7 @@ class TestModels(unittest.TestCase):
         self.assertAlmostEqual(static_model("shutdown", "duration"), 6980, places=0)
         self.assertAlmostEqual(static_model("start", "duration"), 6980, places=0)
 
-    def test_model_multifile_sharp(self):
+    def test_multifile_sharp(self):
         testfiles = [
             "test-data/20170116_145420_sharpLS013B4DN.tar",
             "test-data/20170116_151348_sharpLS013B4DN.tar",
@@ -285,7 +736,7 @@ class TestModels(unittest.TestCase):
         self.assertAlmostEqual(static_model("sendLine", "duration"), 180, places=0)
         self.assertAlmostEqual(static_model("toggleVCOM", "duration"), 30, places=0)
 
-    def test_model_multifile_mmstatic(self):
+    def test_multifile_mmstatic(self):
         testfiles = [
             "test-data/20170116_143516_mmstatic.tar",
             "test-data/20170116_142654_mmstatic.tar",
@@ -325,7 +776,7 @@ class TestModels(unittest.TestCase):
     @pytest.mark.skipif(
         "TEST_SLOW" not in os.environ, reason="slow test, set TEST_SLOW=1 to run"
     )
-    def test_model_multifile_cc1200(self):
+    def test_multifile_cc1200(self):
         testfiles = [
             "test-data/20170125_125433_cc1200.tar",
             "test-data/20170125_142420_cc1200.tar",