use common energytrace loader code

this also fixes a nasty off-by-one in the barcode loader
(now _load_energytrace):

wrong:   interval_start_timestamp = data[:-1, 0] * 1e-6
correct: interval_start_timestamp = data[1:, 0] * 1e-6

5 files changed, 86 insertions, 114 deletions

diff --git a/bin/generate-dfa-benchmark.py b/bin/generate-dfa-benchmark.py
index 50f5e58..3e3454e 100755
--- a/bin/generate-dfa-benchmark.py
+++ b/bin/generate-dfa-benchmark.py
@@ -522,6 +522,7 @@ if __name__ == "__main__":
         sys.exit(2)
 
     if "msp430fr" in opt["arch"]:
+        # target = runner.Arch(opt["arch"], ["cpu_freq=8000000", "with_hfxt=1"])
         target = runner.Arch(opt["arch"], ["cpu_freq=8000000"])
     else:
         target = runner.Arch(opt["arch"])
diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py
index 32d4dae..a8b49bf 100644
--- a/lib/lennart/DataProcessor.py
+++ b/lib/lennart/DataProcessor.py
@@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
 
 
 class DataProcessor:
-    def __init__(self, sync_data, energy_data):
+    def __init__(self, sync_data, et_timestamps, et_power):
         """
         Creates DataProcessor object.
 
@@ -18,13 +18,14 @@ class DataProcessor:
         # high-precision LA/Timer timestamps at synchronization events
         self.sync_timestamps = []
         # low-precision energytrace timestamps
-        self.et_timestamps = []
+        self.et_timestamps = et_timestamps
         # energytrace power values
-        self.et_power_values = []
+        self.et_power_values = et_power
         self.sync_data = sync_data
-        self.energy_data = energy_data
         self.start_offset = 0
 
+        # TODO determine automatically based on minimum (or p1) power draw over measurement area + X
+        # use 0.02 for HFXT runs
         self.power_sync_watt = 0.011
         self.power_sync_len = 0.7
         self.power_sync_max_outliers = 2
@@ -53,8 +54,6 @@ class DataProcessor:
                 f"LogicAnalyzer sync data has length {len(time_stamp_data)}, expected >= 12"
             )
 
-        last_data = [0, 0, 0, 0]
-
         self.raw_sync_timestamps = time_stamp_data
 
         # NEW
@@ -63,11 +62,8 @@ class DataProcessor:
         outliers = 0
         pre_outliers_ts = None
         # TODO only consider the first few and the last few seconds for sync points
-        for i, energytrace_dataset in enumerate(self.energy_data):
-            usedtime = energytrace_dataset[0] - last_data[0]  # in microseconds
-            timestamp = energytrace_dataset[0]
-            usedenergy = energytrace_dataset[3] - last_data[3]
-            power = usedenergy / usedtime * 1e-3  # in watts
+        for i, timestamp in enumerate(self.et_timestamps):
+            power = self.et_power_values[i]
             if power > 0:
                 if power > self.power_sync_watt:
                     if sync_start is None:
@@ -80,27 +76,18 @@ class DataProcessor:
                     outliers += 1
                     if outliers > self.power_sync_max_outliers:
                         if sync_start is not None:
-                            if (
-                                pre_outliers_ts - sync_start
-                            ) / 1_000_000 > self.power_sync_len:
+                            if (pre_outliers_ts - sync_start) > self.power_sync_len:
                                 datasync_timestamps.append(
                                     (
-                                        sync_start / 1_000_000,
-                                        pre_outliers_ts / 1_000_000,
+                                        sync_start,
+                                        pre_outliers_ts,
                                     )
                                 )
                             sync_start = None
 
-                last_data = energytrace_dataset
-
-            self.et_timestamps.append(timestamp / 1_000_000)
-            self.et_power_values.append(power)
-
         if power > self.power_sync_watt:
-            if (self.energy_data[-1][0] - sync_start) / 1_000_000 > self.power_sync_len:
-                datasync_timestamps.append(
-                    (sync_start / 1_000_000, pre_outliers_ts / 1_000_000)
-                )
+            if (self.et_timestamps[-1] - sync_start) > self.power_sync_len:
+                datasync_timestamps.append((sync_start, pre_outliers_ts))
 
         # print(datasync_timestamps)
 
diff --git a/lib/lennart/EnergyInterface.py b/lib/lennart/EnergyInterface.py
index 19aae84..55bf7c1 100644
--- a/lib/lennart/EnergyInterface.py
+++ b/lib/lennart/EnergyInterface.py
@@ -70,39 +70,6 @@ class EnergyInterface(DataInterface):
             return
         self.energytrace.wait()
 
-    def getData(self):
-        """
-        cleans the string data and creates int list
-        :return: list of data, in format [[int,int,int,int], [int,int,int,int], ... ]
-        """
-        energytrace_log = open(self.temp_file)
-        lines = energytrace_log.readlines()[21:]
-        data = []
-        for line in lines:
-            if "MSP430_DisableEnergyTrace" in line:
-                break
-            else:
-                data.append([int(i) for i in line.split()])
-        return data
-
-    @classmethod
-    def getDataFromString(cls, string, delimiter="\\n"):
-        """
-        Parsing the data from string
-
-        :param string: input string which will be parsed
-        :param delimiter: for normal file its \n
-        :return: list of data, in format [[int,int,int,int], [int,int,int,int], ... ]
-        """
-        lines = string.split(delimiter)[21:]
-        data = []
-        for line in lines:
-            if "MSP430_DisableEnergyTrace" in line:
-                break
-            else:
-                data.append([int(i) for i in line.split()])
-        return data
-
     def setFile(self, path):
         """
         changeing the temporary file
diff --git a/lib/loader.py b/lib/loader.py
index 60be648..7a0fc4e 100644
--- a/lib/loader.py
+++ b/lib/loader.py
@@ -1232,6 +1232,47 @@ def pta_trace_to_aggregate(traces, ignore_trace_indexes=[]):
     return by_name, parameter_names, arg_count
 
 
+def _load_energytrace(data_string):
+    """
+    Load log data (raw energytrace .txt file, one line per event).
+
+    :param log_data: raw energytrace log file in 4-column .txt format
+    """
+
+    lines = data_string.decode("ascii").split("\n")
+    data_count = sum(map(lambda x: len(x) > 0 and x[0] != "#", lines))
+    data_lines = filter(lambda x: len(x) > 0 and x[0] != "#", lines)
+
+    data = np.empty((data_count, 4))
+
+    for i, line in enumerate(data_lines):
+        fields = line.split(" ")
+        if len(fields) == 4:
+            timestamp, current, voltage, total_energy = map(int, fields)
+        elif len(fields) == 5:
+            # cpustate = fields[0]
+            timestamp, current, voltage, total_energy = map(int, fields[1:])
+        else:
+            raise RuntimeError('cannot parse line "{}"'.format(line))
+        data[i] = [timestamp, current, voltage, total_energy]
+
+    interval_start_timestamp = data[1:, 0] * 1e-6
+    interval_duration = (data[1:, 0] - data[:-1, 0]) * 1e-6
+    interval_power = (data[1:, 3] - data[:-1, 3]) / (data[1:, 0] - data[:-1, 0]) * 1e-3
+
+    m_duration_us = data[-1, 0] - data[0, 0]
+
+    sample_rate = data_count / (m_duration_us * 1e-6)
+
+    logger.debug(
+        "got {} samples with {} seconds of log data ({} Hz)".format(
+            data_count, m_duration_us * 1e-6, sample_rate
+        )
+    )
+
+    return (interval_start_timestamp, interval_duration, interval_power, sample_rate)
+
+
 class EnergyTraceWithBarcode:
     """
     EnergyTrace log loader for DFA traces.
@@ -1297,44 +1338,12 @@ class EnergyTraceWithBarcode:
             )
             return list()
 
-        lines = log_data[0].decode("ascii").split("\n")
-        data_count = sum(map(lambda x: len(x) > 0 and x[0] != "#", lines))
-        data_lines = filter(lambda x: len(x) > 0 and x[0] != "#", lines)
-
-        data = np.empty((data_count, 4))
-
-        for i, line in enumerate(data_lines):
-            fields = line.split(" ")
-            if len(fields) == 4:
-                timestamp, current, voltage, total_energy = map(int, fields)
-            elif len(fields) == 5:
-                # cpustate = fields[0]
-                timestamp, current, voltage, total_energy = map(int, fields[1:])
-            else:
-                raise RuntimeError('cannot parse line "{}"'.format(line))
-            data[i] = [timestamp, current, voltage, total_energy]
-
-        self.interval_start_timestamp = data[:-1, 0] * 1e-6
-        self.interval_duration = (data[1:, 0] - data[:-1, 0]) * 1e-6
-        self.interval_power = ((data[1:, 3] - data[:-1, 3]) * 1e-9) / (
-            (data[1:, 0] - data[:-1, 0]) * 1e-6
-        )
-
-        m_duration_us = data[-1, 0] - data[0, 0]
-
-        self.sample_rate = data_count / (m_duration_us * 1e-6)
-
-        logger.debug(
-            "got {} samples with {} seconds of log data ({} Hz)".format(
-                data_count, m_duration_us * 1e-6, self.sample_rate
-            )
-        )
-
-        return (
+        (
             self.interval_start_timestamp,
             self.interval_duration,
             self.interval_power,
-        )
+            self.sample_rate,
+        ) = _load_energytrace(log_data[0])
 
     def ts_to_index(self, timestamp):
         """
@@ -1709,7 +1718,12 @@ class EnergyTraceWithLogicAnalyzer:
 
         # Daten laden
         self.sync_data = SigrokResult.fromString(log_data[0])
-        self.energy_data = EnergyInterface.getDataFromString(str(log_data[1]))
+        (
+            self.interval_start_timestamp,
+            self.interval_duration,
+            self.interval_power,
+            self.sample_rate,
+        ) = _load_energytrace(log_data[1])
 
     def analyze_states(self, traces, offline_index: int):
         """
@@ -1743,7 +1757,11 @@ class EnergyTraceWithLogicAnalyzer:
         # print(names[:15])
         from dfatool.lennart.DataProcessor import DataProcessor
 
-        dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data)
+        dp = DataProcessor(
+            sync_data=self.sync_data,
+            et_timestamps=self.interval_start_timestamp,
+            et_power=self.interval_power,
+        )
         dp.run()
         energy_trace_new = dp.getStatesdfatool(
             state_sleep=self.state_duration, with_traces=self.with_traces
@@ -1814,14 +1832,13 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer):
         super().__init__(voltage, state_duration, transition_names, with_traces)
 
     def load_data(self, log_data):
-        from dfatool.lennart.SigrokInterface import SigrokResult
-        from dfatool.lennart.EnergyInterface import EnergyInterface
-
-        # Daten laden
         self.sync_data = None
-        self.energy_data = EnergyInterface.getDataFromString(str(log_data[0]))
-
-        pass
+        (
+            self.interval_start_timestamp,
+            self.interval_duration,
+            self.interval_power,
+            self.sample_rate,
+        ) = _load_energytrace(log_data[0])
 
     def analyze_states(self, traces, offline_index: int):
 
diff --git a/test/test_ptamodel.py b/test/test_ptamodel.py
index fd20502..9172232 100755
--- a/test/test_ptamodel.py
+++ b/test/test_ptamodel.py
@@ -665,24 +665,24 @@ class TestFromFile(unittest.TestCase):
             ),
         )
         static_model = model.get_static()
-        self.assertAlmostEqual(static_model("P14MW", "power"), 14542, places=0)
+        self.assertAlmostEqual(static_model("P14MW", "power"), 14540, places=0)
         self.assertAlmostEqual(static_model("P235UW", "power"), 899, places=0)
         self.assertAlmostEqual(static_model("P3_4MW", "power"), 3974, places=0)
         self.assertAlmostEqual(static_model("SLEEP", "power"), 672, places=0)
         self.assertAlmostEqual(static_model("nop10K", "duration"), 514, places=0)
-        self.assertAlmostEqual(static_model("nop10K", "energy"), 1207636, places=0)
+        self.assertAlmostEqual(static_model("nop10K", "energy"), 1219044, places=0)
         self.assertAlmostEqual(static_model("nop1K0", "duration"), 514, places=0)
-        self.assertAlmostEqual(static_model("nop1K0", "energy"), 1234947, places=0)
+        self.assertAlmostEqual(static_model("nop1K0", "energy"), 1199012, places=0)
         self.assertAlmostEqual(static_model("nop3K3", "duration"), 514, places=0)
-        self.assertAlmostEqual(static_model("nop3K3", "energy"), 1514341, places=0)
+        self.assertAlmostEqual(static_model("nop3K3", "energy"), 1501667, places=0)
         self.assertAlmostEqual(static_model("setup", "duration"), 27, places=0)
-        self.assertAlmostEqual(static_model("setup", "energy"), 19535, places=0)
+        self.assertAlmostEqual(static_model("setup", "energy"), 19907, places=0)
         self.assertAlmostEqual(static_model("switchTo3K3", "duration"), 19, places=0)
         self.assertAlmostEqual(static_model("switchTo3K3", "energy"), 14359, places=0)
         self.assertAlmostEqual(static_model("switchTo47K", "duration"), 19, places=0)
         self.assertAlmostEqual(static_model("switchTo47K", "energy"), 14166, places=0)
         self.assertAlmostEqual(static_model("switchTo750", "duration"), 19, places=0)
-        self.assertAlmostEqual(static_model("switchTo750", "energy"), 14166, places=0)
+        self.assertAlmostEqual(static_model("switchTo750", "energy"), 14131, places=0)
         self.assertAlmostEqual(static_model("switchToNone", "duration"), 19, places=0)
         self.assertAlmostEqual(static_model("switchToNone", "energy"), 14306, places=0)
 
@@ -697,9 +697,9 @@ class TestFromFile(unittest.TestCase):
             "setup trans100u trans10m trans1m trans2m trans5m".split(" "),
         )
         static_model = model.get_static()
-        self.assertAlmostEqual(static_model("IDLE", "power"), 760, places=0)
+        self.assertAlmostEqual(static_model("IDLE", "power"), 766, places=0)
         self.assertAlmostEqual(static_model("setup", "duration"), 15, places=0)
-        self.assertAlmostEqual(static_model("setup", "energy"), 13785, places=0)
+        self.assertAlmostEqual(static_model("setup", "energy"), 13818, places=0)
         self.assertAlmostEqual(static_model("trans100u", "duration"), 146, places=0)
         self.assertAlmostEqual(static_model("trans100u", "energy"), 136794, places=0)
         self.assertAlmostEqual(static_model("trans10m", "duration"), 10084, places=0)
@@ -709,7 +709,7 @@ class TestFromFile(unittest.TestCase):
         self.assertAlmostEqual(static_model("trans2m", "duration"), 2031, places=0)
         self.assertAlmostEqual(static_model("trans2m", "energy"), 10500784, places=0)
         self.assertAlmostEqual(static_model("trans5m", "duration"), 5049, places=0)
-        self.assertAlmostEqual(static_model("trans5m", "energy"), 30519236, places=0)
+        self.assertAlmostEqual(static_model("trans5m", "energy"), 30521933, places=0)
 
     def test_et_timer_dco(self):
         raw_data = RawData(["test-data/20201203-110526-et_timer_dco.tar"])
@@ -722,11 +722,11 @@ class TestFromFile(unittest.TestCase):
             "setup trans100u trans10m trans1m trans2m trans5m".split(" "),
         )
         static_model = model.get_static()
-        self.assertAlmostEqual(static_model("IDLE", "power"), 756, places=0)
+        self.assertAlmostEqual(static_model("IDLE", "power"), 764, places=0)
         self.assertAlmostEqual(static_model("setup", "duration"), 28, places=0)
-        self.assertAlmostEqual(static_model("setup", "energy"), 25714, places=0)
+        self.assertAlmostEqual(static_model("setup", "energy"), 25716, places=0)
         self.assertAlmostEqual(static_model("trans100u", "duration"), 158, places=0)
-        self.assertAlmostEqual(static_model("trans100u", "energy"), 148071, places=0)
+        self.assertAlmostEqual(static_model("trans100u", "energy"), 148072, places=0)
         self.assertAlmostEqual(static_model("trans10m", "duration"), 10097, places=0)
         self.assertAlmostEqual(static_model("trans10m", "energy"), 61416161, places=0)
         self.assertAlmostEqual(static_model("trans1m", "duration"), 1038, places=0)