From 906c1e0c71c8ad58934990ebe686adf7496e0dce Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Wed, 8 Jul 2020 13:45:39 +0200 Subject: Add harness.energytrace_sync="led" for LA / Timing based sync --- bin/generate-dfa-benchmark.py | 3 +++ lib/harness.py | 4 ++++ lib/runner.py | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bin/generate-dfa-benchmark.py b/bin/generate-dfa-benchmark.py index 1410c28..fdfac35 100755 --- a/bin/generate-dfa-benchmark.py +++ b/bin/generate-dfa-benchmark.py @@ -636,8 +636,10 @@ if __name__ == "__main__": elif "energytrace" in opt: # Use barcode sync by default gpio_mode = "bar" + energytrace_sync = None if "sync" in opt["energytrace"] and opt["energytrace"]["sync"] != "bar": gpio_mode = "around" + energytrace_sync = "led" harness = OnboardTimerHarness( gpio_pin=timer_pin, gpio_mode=gpio_mode, @@ -645,6 +647,7 @@ if __name__ == "__main__": counter_limits=runner.get_counter_limits_us(opt["arch"]), log_return_values=need_return_values, repeat=1, + energytrace_sync=energytrace_sync, ) elif "timing" in opt: harness = OnboardTimerHarness( diff --git a/lib/harness.py b/lib/harness.py index ae9c28c..fe6cb9a 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -33,6 +33,7 @@ class TransitionHarness: log_return_values=False, repeat=0, post_transition_delay_us=0, + energytrace_sync=None, ): """ Create a new TransitionHarness @@ -53,6 +54,7 @@ class TransitionHarness: self.log_return_values = log_return_values self.repeat = repeat self.post_transition_delay_us = post_transition_delay_us + self.energytrace_sync = energytrace_sync self.reset() def copy(self): @@ -63,6 +65,7 @@ class TransitionHarness: log_return_values=self.log_return_values, repeat=self.repeat, post_transition_delay_us=self.post_transition_delay_us, + energytrace_sync=self.energytrace_sync, ) new_object.traces = self.traces.copy() new_object.trace_id = self.trace_id @@ -373,6 +376,7 @@ class OnboardTimerHarness(TransitionHarness): pta=self.pta, log_return_values=self.log_return_values, repeat=self.repeat, + energytrace_sync=self.energytrace_sync, ) new_harness.traces = self.traces.copy() new_harness.trace_id = self.trace_id diff --git a/lib/runner.py b/lib/runner.py index 0d4d449..def9c8f 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -172,12 +172,15 @@ class EnergyTraceMonitor(SerialMonitor): def get_files(self) -> list: return [self._output] - # + # Benchmark-Konfiguration. Hier: Die (konstante) Spannung. + # MSP430FR5969: 3,6V (wird aktuell nicht unterstützt) + # MSP430FR5994: 3,3V (default) def get_config(self) -> dict: return { "voltage": self._voltage, } + class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): """EnergyTraceLogicAnalyzerMonitor captures EnergyTrace energy data and LogicAnalyzer timing output.""" -- cgit v1.2.3 From 0bda106da0af4b9ba72f31930669fb5da1311d28 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Wed, 8 Jul 2020 16:06:51 +0200 Subject: Add dummy loader for EnergyTraceWithLogicAnalyzer. Data is not analyzed. Instead, everything takes as long as planned and everything costs 23 mW --- lib/loader.py | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 4 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index c35eb4c..65d497b 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -107,7 +107,14 @@ def _preprocess_mimosa(measurement): def _preprocess_etlog(measurement): setup = measurement["setup"] - etlog = EnergyTraceLog( + + energytrace_class = EnergyTraceWithBarcode + if measurement["sync_mode"] == "la": + energytrace_class = EnergyTraceWithLogicAnalyzer + elif measurement["sync_mode"] == "timer": + energytrace_class = EnergyTraceWithTimer + + etlog = energytrace_class( float(setup["voltage"]), int(setup["state_duration"]), measurement["transition_names"], @@ -406,7 +413,7 @@ class RawData: processed_data["error"] = "; ".join(processed_data["datasource_errors"]) return False - # Note that the low-level parser (EnergyTraceLog) already checks + # Note that the low-level parser (EnergyTraceWithBarcode) already checks # whether the transition count is correct return True @@ -909,6 +916,10 @@ class RawData: new_filenames = list() with tarfile.open(filename) as tf: ptalog = self.ptalog + if "sync" in ptalog["opt"]["energytrace"]: + sync_mode = ptalog["opt"]["energytrace"]["sync"] + else: + sync_mode = "bar" # Benchmark code may be too large to be executed in a single # run, so benchmarks (a benchmark is basically a list of DFA runs) @@ -968,6 +979,7 @@ class RawData: offline_data.append( { "content": tf.extractfile(member).read(), + "sync_mode": sync_mode, "fileno": j, "info": member, "setup": self.setup_by_fileno[j], @@ -1161,7 +1173,7 @@ def pta_trace_to_aggregate(traces, ignore_trace_indexes=[]): return by_name, parameter_names, arg_count -class EnergyTraceLog: +class EnergyTraceWithBarcode: """ EnergyTrace log loader for DFA traces. @@ -1184,7 +1196,7 @@ class EnergyTraceLog: with_traces=False, ): """ - Create a new EnergyTraceLog object. + Create a new EnergyTraceWithBarcode object. :param voltage: supply voltage [V], usually 3.3 V :param state_duration: state duration [ms] @@ -1599,6 +1611,123 @@ class EnergyTraceLog: return None, None, None, None +class EnergyTraceWithLogicAnalyzer: + def __init__( + self, + voltage: float, + state_duration: int, + transition_names: list, + with_traces=False, + ): + """ + Create a new EnergyTraceWithLogicAnalyzer object. + + :param voltage: supply voltage [V], usually 3.3 V + :param state_duration: state duration [ms] + :param transition_names: list of transition names in PTA transition order. + Needed to map barcode synchronization numbers to transitions. + """ + self.voltage = voltage + self.state_duration = state_duration * 1e-3 + self.transition_names = transition_names + self.with_traces = with_traces + self.errors = list() + + def load_data(self, log_data): + # TODO Daten laden + pass + + def analyze_states(self, traces, offline_index: int): + u""" + Split log data into states and transitions and return duration, energy, and mean power for each element. + + :param traces: expected traces, needed to synchronize with the measurement. + traces is a list of runs, traces[*]['trace'] is a single run + (i.e. a list of states and transitions, starting with a transition + and ending with a state). + :param offline_index: This function uses traces[*]['trace'][*]['online_aggregates']['duration'][offline_index] to find sync codes + + :param charges: raw charges (each element describes the charge in pJ transferred during 10 µs) + :param trigidx: "charges" indexes corresponding to a trigger edge, see `trigger_edges` + :param ua_func: charge(pJ) -> current(µA) function as returned by `calibration_function` + + :returns: returns list of states and transitions, starting with a transition and ending with astate + Each element is a dict containing: + * `isa`: 'state' or 'transition' + * `W_mean`: Mittelwert der Leistungsaufnahme + * `W_std`: Standardabweichung der Leistungsaufnahme + * `s`: Dauer + if isa == 'transition, it also contains: + * `W_mean_delta_prev`: Differenz zwischen W_mean und W_mean des vorherigen Zustands + * `W_mean_delta_next`: Differenz zwischen W_mean und W_mean des Folgezustands + """ + + # TODO Tatsächlich Daten auswerten + + energy_trace = list() + + expected_transitions = list() + for trace_number, trace in enumerate(traces): + for state_or_transition_number, state_or_transition in enumerate( + trace["trace"] + ): + if state_or_transition["isa"] == "transition": + try: + expected_transitions.append( + ( + state_or_transition["name"], + state_or_transition["online_aggregates"]["duration"][ + offline_index + ] + * 1e-6, + ) + ) + except IndexError: + self.errors.append( + 'Entry #{} ("{}") in trace #{} has no duration entry for offline_index/repeat_id {}'.format( + state_or_transition_number, + state_or_transition["name"], + trace_number, + offline_index, + ) + ) + return energy_trace + + for name, duration in expected_transitions: + transition = { + "isa": "transition", + "W_mean": max(np.random.normal(0.023, 0.002), 0), + "W_std": 0.0001, + "s": duration, + } + + energy_trace.append(transition) + + if len(energy_trace) > 1: + energy_trace[-1]["W_mean_delta_prev"] = ( + energy_trace[-1]["W_mean"] - energy_trace[-2]["W_mean"] + ) + + state = { + "isa": "state", + "W_mean": max(np.random.normal(0.023, 0.002), 0), + "W_std": 0.0001, + "s": self.state_duration, + } + + energy_trace.append(state) + + energy_trace[-2]["W_mean_delta_next"] = ( + energy_trace[-2]["W_mean"] - energy_trace[-1]["W_mean"] + ) + + return energy_trace + + +class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): + pass + + class MIMOSA: """ MIMOSA log loader for DFA traces with auto-calibration. -- cgit v1.2.3 From 54ce41f2c7710b1bfbc884ce141140dc268710c5 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 9 Jul 2020 11:05:57 +0200 Subject: OnboardTimerHarness: Log state duration as well --- lib/harness.py | 123 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 93 insertions(+), 30 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index ae9c28c..c251ec8 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -175,21 +175,16 @@ class TransitionHarness: """Return C++ code used to start a new run/trace.""" return "ptalog.reset();\n" - def _pass_transition_call(self, transition_id): - if self.gpio_mode == "bar": - barcode_bits = Code128("T{}".format(transition_id), charset="B").modules - if len(barcode_bits) % 8 != 0: - barcode_bits.extend([1] * (8 - (len(barcode_bits) % 8))) - barcode_bytes = [ - 255 - int("".join(map(str, reversed(barcode_bits[i : i + 8]))), 2) - for i in range(0, len(barcode_bits), 8) - ] - inline_array = "".join(map(lambda s: "\\x{:02x}".format(s), barcode_bytes)) - return 'ptalog.startTransition("{}", {});\n'.format( - inline_array, len(barcode_bytes) - ) - else: - return "ptalog.startTransition();\n" + def _get_barcode(self, transition_id): + barcode_bits = Code128("T{}".format(transition_id), charset="B").modules + if len(barcode_bits) % 8 != 0: + barcode_bits.extend([1] * (8 - (len(barcode_bits) % 8))) + barcode_bytes = [ + 255 - int("".join(map(str, reversed(barcode_bits[i : i + 8]))), 2) + for i in range(0, len(barcode_bits), 8) + ] + inline_array = "".join(map(lambda s: "\\x{:02x}".format(s), barcode_bytes)) + return inline_array, len(barcode_bytes) def pass_transition( self, transition_id, transition_code, transition: object = None @@ -201,7 +196,12 @@ class TransitionHarness: `post_transition_delay_us` is set. """ ret = "ptalog.passTransition({:d});\n".format(transition_id) - ret += self._pass_transition_call(transition_id) + if self.gpio_mode == "bar": + ret += """ptalog.startTransition("{}", {});\n""".format( + *self._get_barcode(transition_id) + ) + else: + ret += "ptalog.startTransition();\n" if ( self.log_return_values and transition @@ -398,15 +398,12 @@ class OnboardTimerHarness(TransitionHarness): ] def global_code(self): - ret = '#include "driver/counter.h"\n' - ret += "#define PTALOG_TIMING\n" + ret = "#define PTALOG_TIMING\n" ret += super().global_code() return ret def start_benchmark(self, benchmark_id=0): - ret = "counter.start();\n" - ret += "counter.stop();\n" - ret += "ptalog.passNop(counter);\n" + ret = "ptalog.passNop();\n" ret += super().start_benchmark(benchmark_id) return ret @@ -414,8 +411,12 @@ class OnboardTimerHarness(TransitionHarness): self, transition_id, transition_code, transition: object = None ): ret = "ptalog.passTransition({:d});\n".format(transition_id) - ret += self._pass_transition_call(transition_id) - ret += "counter.start();\n" + if self.gpio_mode == "bar": + ret += """ptalog.startTransition("{}", {});\n""".format( + *self._get_barcode(transition_id) + ) + else: + ret += "ptalog.startTransition();\n" if ( self.log_return_values and transition @@ -424,14 +425,13 @@ class OnboardTimerHarness(TransitionHarness): ret += "transition_return_value = {}\n".format(transition_code) else: ret += "{}\n".format(transition_code) - ret += "counter.stop();\n" if ( self.log_return_values and transition and len(transition.return_value_handlers) ): ret += "ptalog.logReturn(transition_return_value);\n" - ret += "ptalog.stopTransition(counter);\n" + ret += "ptalog.stopTransition();\n" return ret def _append_nondeterministic_parameter_value( @@ -453,11 +453,26 @@ class OnboardTimerHarness(TransitionHarness): res.group(1), res.group(2) ) ) - if re.match(r"\[PTA\] benchmark stop", line): + match = re.match(r"\[PTA\] benchmark stop, cycles=(\S+)/(\S+)", line) + if match: self.repetitions += 1 self.synced = False if self.repeat > 0 and self.repetitions == self.repeat: self.done = True + prev_state_cycles = int(match.group(1)) + prev_state_overflow = int(match.group(2)) + prev_state_duration_us = ( + prev_state_cycles * self.one_cycle_in_us + + prev_state_overflow * self.one_overflow_in_us + - self.nop_cycles * self.one_cycle_in_us + ) + final_state = self.traces[self.trace_id]["trace"][-1] + if "offline_aggregates" not in final_state: + final_state["offline_aggregates"] = {"duration": list()} + final_state["offline_aggregates"]["duration"].append( + prev_state_duration_us + ) + print("[HARNESS] done") return # May be repeated, e.g. if the device is reset shortly after start by @@ -473,14 +488,20 @@ class OnboardTimerHarness(TransitionHarness): self.current_transition_in_trace = 0 if self.log_return_values: res = re.match( - r"\[PTA\] transition=(\S+) cycles=(\S+)/(\S+) return=(\S+)", line + r"\[PTA\] transition=(\S+) prevcycles=(\S+)/(\S+) cycles=(\S+)/(\S+) return=(\S+)", + line, ) else: - res = re.match(r"\[PTA\] transition=(\S+) cycles=(\S+)/(\S+)", line) + res = re.match( + r"\[PTA\] transition=(\S+) prevcycles=(\S+)/(\S+) cycles=(\S+)/(\S+)", + line, + ) if res: transition_id = int(res.group(1)) - cycles = int(res.group(2)) - overflow = int(res.group(3)) + prev_state_cycles = int(res.group(2)) + prev_state_overflow = int(res.group(3)) + cycles = int(res.group(4)) + overflow = int(res.group(5)) if overflow >= self.counter_max_overflow: self.abort = True raise RuntimeError( @@ -493,11 +514,28 @@ class OnboardTimerHarness(TransitionHarness): transition_id, ) ) + if prev_state_overflow >= self.counter_max_overflow: + self.abort = True + raise RuntimeError( + "Counter overflow ({:d}/{:d}) in benchmark id={:d} trace={:d}: state before transition #{:d} (ID {:d})".format( + prev_state_cycles, + prev_state_overflow, + 0, + self.trace_id, + self.current_transition_in_trace, + transition_id, + ) + ) duration_us = ( cycles * self.one_cycle_in_us + overflow * self.one_overflow_in_us - self.nop_cycles * self.one_cycle_in_us ) + prev_state_duration_us = ( + prev_state_cycles * self.one_cycle_in_us + + prev_state_overflow * self.one_overflow_in_us + - self.nop_cycles * self.one_cycle_in_us + ) if duration_us < 0: duration_us = 0 # self.traces contains transitions and states, UART output only contains transitions -> use index * 2 @@ -505,6 +543,14 @@ class OnboardTimerHarness(TransitionHarness): log_data_target = self.traces[self.trace_id]["trace"][ self.current_transition_in_trace * 2 ] + if self.current_transition_in_trace > 0: + prev_state_data = self.traces[self.trace_id]["trace"][ + self.current_transition_in_trace * 2 - 1 + ] + elif self.current_transition_in_trace == 0 and self.trace_id > 0: + prev_state_data = self.traces[self.trace_id - 1]["trace"][-1] + else: + prev_state_data = None except IndexError: transition_name = None if self.pta: @@ -531,6 +577,17 @@ class OnboardTimerHarness(TransitionHarness): log_data_target["isa"], ) ) + if prev_state_data and prev_state_data["isa"] != "state": + self.abort = True + raise RuntimeError( + "Log mismatch in benchmark id={:d} trace={:d}: state before transition #{:d} (ID {:d}): Expected state, got {:s}".format( + 0, + self.trace_id, + self.current_transition_in_trace, + transition_id, + prev_state_data["isa"], + ) + ) if self.pta: transition = self.pta.transitions[transition_id] if transition.name != log_data_target["name"]: @@ -601,4 +658,10 @@ class OnboardTimerHarness(TransitionHarness): if "offline_aggregates" not in log_data_target: log_data_target["offline_aggregates"] = {"duration": list()} log_data_target["offline_aggregates"]["duration"].append(duration_us) + if prev_state_data is not None: + if "offline_aggregates" not in prev_state_data: + prev_state_data["offline_aggregates"] = {"duration": list()} + prev_state_data["offline_aggregates"]["duration"].append( + prev_state_duration_us + ) self.current_transition_in_trace += 1 -- cgit v1.2.3 From b5f7fd47d1ee66f20af50fae145990d4665b56c3 Mon Sep 17 00:00:00 2001 From: Lennart Date: Mon, 13 Jul 2020 13:06:11 +0200 Subject: Added first version, kinda working but some tweaks needed. src directory of repository 'ba-lennart-kaiser' needs to in the python execution path --- lib/harness.py | 33 ++++++++++++++++++++++++++++++++- lib/runner.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index fe6cb9a..3b9db89 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -405,15 +405,46 @@ class OnboardTimerHarness(TransitionHarness): ret = '#include "driver/counter.h"\n' ret += "#define PTALOG_TIMING\n" ret += super().global_code() + if self.energytrace_sync == "led": + #TODO Make nicer + ret += """\nvoid runLASync(){ + // ======================= LED SYNC ================================ + ptalog.passTransition(0); + ptalog.startTransition(); + gpio.led_toggle(0); + gpio.led_toggle(1); + ptalog.stopTransition(counter); + + for (unsigned char i = 0; i < 4; i++) { + arch.sleep_ms(250); + } + + ptalog.passTransition(0); + ptalog.startTransition(); + gpio.led_toggle(0); + gpio.led_toggle(1); + ptalog.stopTransition(counter); + // ======================= LED SYNC ================================ + arch.sleep_ms(250); +}\n\n""" return ret def start_benchmark(self, benchmark_id=0): - ret = "counter.start();\n" + ret = "" + if self.energytrace_sync == "led": + ret += "runLASync();\n" + ret += "counter.start();\n" ret += "counter.stop();\n" ret += "ptalog.passNop(counter);\n" ret += super().start_benchmark(benchmark_id) return ret + def stop_benchmark(self): + ret = super().stop_benchmark() + if self.energytrace_sync == "led": + ret += "runLASync();\n" + return ret + def pass_transition( self, transition_id, transition_code, transition: object = None ): diff --git a/lib/runner.py b/lib/runner.py index def9c8f..3a201fc 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -9,7 +9,7 @@ Functions: get_monitor -- return Monitor class suitable for the selected multipass arch get_counter_limits -- return arch-specific multipass counter limits (max value, max overflow) """ - +import json import os import re import serial @@ -17,6 +17,7 @@ import serial.threaded import subprocess import sys import time +from data.timing.SigrokCLIInterface import SigrokCLIInterface class SerialReader(serial.threaded.Protocol): @@ -156,6 +157,7 @@ class EnergyTraceMonitor(SerialMonitor): self._start_energytrace() def _start_energytrace(self): + print("EnergyTrace Start") cmd = ["msp430-etv", "--save", self._output, "0"] self._logger = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True @@ -187,6 +189,33 @@ class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): def __init__(self, port: str, baud: int, callback=None, voltage=3.3): super().__init__(port=port, baud=baud, callback=callback, voltage=voltage) + #TODO get length + options = {'length': 90, 'fake': False, 'sample_rate': 1_000_000} + self.log_file = 'logic_output_log_%s.json' % (time.strftime("%Y%m%d-%H%M%S")) + + # Initialization of Interfaces + self.sig = SigrokCLIInterface(sample_rate=options['sample_rate'], + sample_count=options['length'] * options['sample_rate'], fake=options['fake']) + + # Start Measurements + print("[ET LA] START MEASURE") + self.sig.runMeasureAsynchronous() + + def close(self): + super().close() + # Read measured data + print("[ET LA] Wait MEASURE") + self.sig.waitForAsynchronousMeasure() + print("[ET LA] WRITE MEASURE") + sync_data = self.sig.getData() + print("[ET LA] MEASURE LEN", len(sync_data.timestamps)) + with open(self.log_file, 'w') as fp: + json.dump(sync_data.getDict(), fp) + + def get_files(self) -> list: + print("[ET LA] FILE REQUEST") + return [self.log_file] + class MIMOSAMonitor(SerialMonitor): """MIMOSAMonitor captures serial output and MIMOSA energy data for a specific amount of time.""" -- cgit v1.2.3 From 427f80213e915cb3a7f5fa63eab910ed7fcb0fe0 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Tue, 14 Jul 2020 11:20:33 +0200 Subject: Make black(1) happy --- lib/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/runner.py b/lib/runner.py index 0d4d449..aeb8600 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -178,6 +178,7 @@ class EnergyTraceMonitor(SerialMonitor): "voltage": self._voltage, } + class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): """EnergyTraceLogicAnalyzerMonitor captures EnergyTrace energy data and LogicAnalyzer timing output.""" -- cgit v1.2.3 From 0a7917342cf27f18f553afdf4a65d6377f671aad Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Tue, 14 Jul 2020 16:49:27 +0200 Subject: support more than one logfile per measurement run --- bin/generate-dfa-benchmark.py | 4 ++-- lib/loader.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bin/generate-dfa-benchmark.py b/bin/generate-dfa-benchmark.py index fdfac35..6540702 100755 --- a/bin/generate-dfa-benchmark.py +++ b/bin/generate-dfa-benchmark.py @@ -393,7 +393,7 @@ def run_benchmark( os.remove(filename) harness.undo(i) else: - files.extend(monitor.get_files()) + files.append(monitor.get_files()) i += 1 harness.restart() @@ -680,7 +680,7 @@ if __name__ == "__main__": "files": list(map(lambda x: x[3], results)), "configs": list(map(lambda x: x[2].get_config(), results)), } - extra_files = flatten(json_out["files"]) + extra_files = flatten(map(flatten, json_out["files"])) if "instance" in pta.codegen: output_prefix = ( opt["data"] + time.strftime("/%Y%m%d-%H%M%S-") + pta.codegen["instance"] diff --git a/lib/loader.py b/lib/loader.py index 65d497b..4934316 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -974,14 +974,16 @@ class RawData: "state_duration": ptalog["opt"]["sleep"], } ) - for repeat_id, etlog_file in enumerate(ptalog["files"][j]): - member = tf.getmember(etlog_file) + for repeat_id, etlog_files in enumerate(ptalog["files"][j]): + members = list(map(tf.getmember, etlog_files)) offline_data.append( { - "content": tf.extractfile(member).read(), + "content": list( + map(lambda f: tf.extractfile(f).read(), members) + ), "sync_mode": sync_mode, "fileno": j, - "info": member, + "info": members[0], "setup": self.setup_by_fileno[j], "repeat_id": repeat_id, "expected_trace": ptalog["traces"][j], @@ -1238,7 +1240,7 @@ class EnergyTraceWithBarcode: ) return list() - lines = log_data.decode("ascii").split("\n") + lines = log_data[0].decode("ascii").split("\n") data_count = sum(map(lambda x: len(x) > 0 and x[0] != "#", lines)) data_lines = filter(lambda x: len(x) > 0 and x[0] != "#", lines) -- cgit v1.2.3 From 2389623346d6b81477b8775635a0627baa90e31f Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Tue, 14 Jul 2020 16:59:14 +0200 Subject: Add --no-cache option --- bin/analyze-archive.py | 9 +++++++-- lib/loader.py | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index 5c7c97e..10fe304 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -104,6 +104,9 @@ Options: --export-energymodel= Export energy model. Works out of the box for v1 and v2 logfiles. Requires --hwmodel for v0 logfiles. + +--no-cache + Do not load cached measurement results """ import getopt @@ -304,7 +307,7 @@ if __name__ == "__main__": try: optspec = ( - "info " + "info no-cache " "plot-unparam= plot-param= plot-traces= show-models= show-quality= " "ignored-trace-indexes= discard-outliers= function-override= " "export-traces= " @@ -369,7 +372,9 @@ if __name__ == "__main__": sys.exit(2) raw_data = RawData( - args, with_traces=("export-traces" in opt or "plot-traces" in opt) + args, + with_traces=("export-traces" in opt or "plot-traces" in opt), + skip_cache=("no-cache" in opt), ) if "info" in opt: diff --git a/lib/loader.py b/lib/loader.py index 4934316..438bbdb 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -249,7 +249,7 @@ class RawData: file system, making subsequent loads near-instant. """ - def __init__(self, filenames, with_traces=False): + def __init__(self, filenames, with_traces=False, skip_cache=False): """ Create a new RawData object. @@ -328,7 +328,7 @@ class RawData: self.pta = self.ptalog["pta"] self.set_cache_file() - if not with_traces: + if not with_traces and not skip_cache: self.load_cache() def set_cache_file(self): -- cgit v1.2.3 From 12a46eeee142233a205cac571e276a68af503e77 Mon Sep 17 00:00:00 2001 From: Lennart Date: Tue, 14 Jul 2020 17:17:09 +0200 Subject: Tested more --- .gitignore | 1 + lib/loader.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 25b1be5..91b6250 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc /htmlcov/ /.coverage* +.idea/ diff --git a/lib/loader.py b/lib/loader.py index 438bbdb..c0a1694 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1637,6 +1637,8 @@ class EnergyTraceWithLogicAnalyzer: def load_data(self, log_data): # TODO Daten laden + print("########################") + print(log_data[0].split("\n")[0]) pass def analyze_states(self, traces, offline_index: int): -- cgit v1.2.3 From e250e8fb30901a910cc22023a0bb99879e2c71b9 Mon Sep 17 00:00:00 2001 From: Lennart Date: Tue, 14 Jul 2020 22:52:21 +0200 Subject: Added into runner.py --- lib/harness.py | 3 +- lib/loader.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- lib/runner.py | 17 ++++--- 3 files changed, 163 insertions(+), 15 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index 3b9db89..7c99f5d 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -440,9 +440,10 @@ class OnboardTimerHarness(TransitionHarness): return ret def stop_benchmark(self): - ret = super().stop_benchmark() + ret = "" if self.energytrace_sync == "led": ret += "runLASync();\n" + ret += super().stop_benchmark() return ret def pass_transition( diff --git a/lib/loader.py b/lib/loader.py index c0a1694..38d59fc 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1621,6 +1621,7 @@ class EnergyTraceWithLogicAnalyzer: transition_names: list, with_traces=False, ): + """ Create a new EnergyTraceWithLogicAnalyzer object. @@ -1636,9 +1637,13 @@ class EnergyTraceWithLogicAnalyzer: self.errors = list() def load_data(self, log_data): - # TODO Daten laden - print("########################") - print(log_data[0].split("\n")[0]) + from data.timing.SigrokInterface import SigrokResult + from data.energy.EnergyInterface import EnergyInterface + + # Daten laden + self.sync_data = SigrokResult.fromString(log_data[0]) + self.energy_data = EnergyInterface.getDataFromString(str(log_data[1])) + pass def analyze_states(self, traces, offline_index: int): @@ -1666,7 +1671,89 @@ class EnergyTraceWithLogicAnalyzer: * `W_mean_delta_next`: Differenz zwischen W_mean und W_mean des Folgezustands """ - # TODO Tatsächlich Daten auswerten + # Tatsächlich Daten auswerten + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # remove Dirty Data from previously running program (happens if logic Analyzer Measurement starts earlier than + # the HW Reset from energytrace) + use_data_after_index = 0 + for x in range(1, len(self.sync_data.timestamps)): + if self.sync_data.timestamps[x] - self.sync_data.timestamps[x - 1] > 1.3: + use_data_after_index = x + break + + time_stamp_data = self.sync_data.timestamps[use_data_after_index:] + + start_offset = None + start_timestamp = None + end_offset = None + end_timestamp = None + + last_values = [] # maybe needed to flatten the data + plot_data_x = [] + plot_data_y = [] + last_data = [0, 0, 0, 0] + + power_sync_watt = 0.0162 + + # MAIN ENERGY DATA ITERATION + for energytrace_dataset in self.energy_data: + usedtime = energytrace_dataset[0] - last_data[0] + usedenergy = energytrace_dataset[3] - last_data[3] + power = usedenergy / usedtime * 10 ** -3 # in watts + + if power > 0: + if power > power_sync_watt and start_offset is None: + # automatic START offset calculation + start_offset = energytrace_dataset[0] / 1_000_000 - time_stamp_data[0] + start_timestamp = energytrace_dataset[0] / 1_000_000 + if power > power_sync_watt and start_offset is not None: # TODO BETTER + # automatic END offset calculation (Drift) + end_offset = energytrace_dataset[0] / 1_000_000 - time_stamp_data[-1] - start_offset + end_timestamp = energytrace_dataset[0] / 1_000_000 + if len(last_values) > 10: + last_values.pop(0) + last_values.append(power) + # print("%s \tUsed Energy: %i \tUsed Time: %i \tPower: %f \t l_10: %f" % (now_data[0], usedenergy, usedtime, power, sum(last_values) / len(last_values))) + plot_data_x.append(energytrace_dataset[0] / 1_000_000) + plot_data_y.append(power) + + last_data = energytrace_dataset + + # -------------------- + # add start offset to the data + modified_timestamps_with_offset = [] + for x in time_stamp_data: + if x + start_offset >= 0: + modified_timestamps_with_offset.append(x + start_offset) + + # -------------------- + # Add drift to datapoints + modified_timestamps_with_drift = [] + endFactor = (end_timestamp + end_offset - start_timestamp) / (end_timestamp - start_timestamp) + for x in modified_timestamps_with_offset: + modified_timestamps_with_drift.append(((x - start_timestamp) * endFactor) + start_timestamp) + + self.start_offset = 0 + def getPowerBetween(start, end): + first_index = 0 + all_power = 0 + all_power_count = 0 + for ind in range(self.start_offset, len(plot_data_x)): + first_index = ind + if plot_data_x[ind] > start: + break + + for ind in range(first_index, len(plot_data_x)): + all_power += plot_data_y[ind] + all_power_count += 1 + if plot_data_x[ind] > end: + self.start_offset = ind - 1 + break + return all_power / all_power_count + + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ energy_trace = list() @@ -1697,7 +1784,65 @@ class EnergyTraceWithLogicAnalyzer: ) return energy_trace - for name, duration in expected_transitions: + + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + end_transition_ts = None + timestamps_sync_start = 0 + energy_trace_new = list() + + for ts_index in range(0 + timestamps_sync_start, int(len(modified_timestamps_with_drift) / 2)): + start_transition_ts = modified_timestamps_with_drift[ts_index * 2] + if end_transition_ts is not None: + power = getPowerBetween(end_transition_ts, start_transition_ts) + #print("STATE", end_transition_ts * 10 ** 6, start_transition_ts * 10 ** 6, (start_transition_ts - end_transition_ts) * 10 ** 6, power) + if (start_transition_ts - end_transition_ts) * 10 ** 6 > 900_000 and power > power_sync_watt * 0.9 and ts_index > 10: + #remove last transition and stop (upcoming data only sync) + del energy_trace_new[-1] + break + pass + + state = { + "isa": "state", + "W_mean": power, + "W_std": 0.0001, + "s": (start_transition_ts - end_transition_ts), #* 10 ** 6, + } + energy_trace_new.append(state) + + energy_trace_new[-2]["W_mean_delta_next"] = ( + energy_trace_new[-2]["W_mean"] - energy_trace_new[-1]["W_mean"] + ) + + # get energy end_transition_ts + end_transition_ts = modified_timestamps_with_drift[ts_index * 2 + 1] + power = getPowerBetween(start_transition_ts, end_transition_ts) + #print("TRANS", start_transition_ts * 10 ** 6, end_transition_ts * 10 ** 6, (end_transition_ts - start_transition_ts) * 10 ** 6, power) + transition = { + "isa": "transition", + "W_mean": power, + "W_std": 0.0001, + "s": (end_transition_ts - start_transition_ts), #* 10 ** 6, + } + + if (end_transition_ts - start_transition_ts) * 10 ** 6 > 2_000_000: + # TODO Last data set corrupted? HOT FIX!!!!!!!!!!!! REMOVE LATER + for x in range(4): + del energy_trace_new[-1] + break + pass + + energy_trace_new.append(transition) + # print(start_transition_ts, "-", end_transition_ts, "-", end_transition_ts - start_transition_ts) + + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + energy_trace_new = energy_trace_new[4:] + + for number, item in enumerate(expected_transitions): + name, duration = item transition = { "isa": "transition", "W_mean": max(np.random.normal(0.023, 0.002), 0), @@ -1725,7 +1870,8 @@ class EnergyTraceWithLogicAnalyzer: energy_trace[-2]["W_mean"] - energy_trace[-1]["W_mean"] ) - return energy_trace + #print(len(energy_trace_new), " - ", len(energy_trace), " - ", ",".join([str(x["s"]) for x in energy_trace_new[-6:]]), " - ", ",".join([str(x["s"]) for x in energy_trace[-6:]])) + return energy_trace_new class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): diff --git a/lib/runner.py b/lib/runner.py index 3a201fc..3092d5a 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -165,6 +165,7 @@ class EnergyTraceMonitor(SerialMonitor): # Benchmark fertig -> externe Hilfsprogramme beenden def close(self): + print("EnergyTrace Close") super().close() self._logger.send_signal(subprocess.signal.SIGINT) stdout, stderr = self._logger.communicate(timeout=15) @@ -172,6 +173,7 @@ class EnergyTraceMonitor(SerialMonitor): # Zusätzliche Dateien, die mit dem Benchmark-Log und -Plan abgespeichert werden sollen # (hier: Die von msp430-etv generierten Logfiles) def get_files(self) -> list: + print("EnergyTrace Get Files") return [self._output] # Benchmark-Konfiguration. Hier: Die (konstante) Spannung. @@ -189,7 +191,7 @@ class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): def __init__(self, port: str, baud: int, callback=None, voltage=3.3): super().__init__(port=port, baud=baud, callback=callback, voltage=voltage) - #TODO get length + #TODO Max length options = {'length': 90, 'fake': False, 'sample_rate': 1_000_000} self.log_file = 'logic_output_log_%s.json' % (time.strftime("%Y%m%d-%H%M%S")) @@ -198,23 +200,22 @@ class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): sample_count=options['length'] * options['sample_rate'], fake=options['fake']) # Start Measurements - print("[ET LA] START MEASURE") self.sig.runMeasureAsynchronous() def close(self): super().close() # Read measured data - print("[ET LA] Wait MEASURE") - self.sig.waitForAsynchronousMeasure() - print("[ET LA] WRITE MEASURE") + #self.sig.waitForAsynchronousMeasure() + self.sig.forceStopMeasure() + time.sleep(0.2) sync_data = self.sig.getData() - print("[ET LA] MEASURE LEN", len(sync_data.timestamps)) with open(self.log_file, 'w') as fp: json.dump(sync_data.getDict(), fp) def get_files(self) -> list: - print("[ET LA] FILE REQUEST") - return [self.log_file] + files = [self.log_file] + files.extend(super().get_files()) + return files class MIMOSAMonitor(SerialMonitor): -- cgit v1.2.3 From 15be4345defa588e512626800ad9e1764160dfab Mon Sep 17 00:00:00 2001 From: Lennart Date: Mon, 20 Jul 2020 23:11:50 +0200 Subject: Added UART version and changed max power. --- lib/loader.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index 531559f..e9bd7d1 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1694,7 +1694,7 @@ class EnergyTraceWithLogicAnalyzer: plot_data_y = [] last_data = [0, 0, 0, 0] - power_sync_watt = 0.0162 + power_sync_watt = 0.015 # MAIN ENERGY DATA ITERATION for energytrace_dataset in self.energy_data: @@ -1735,7 +1735,7 @@ class EnergyTraceWithLogicAnalyzer: modified_timestamps_with_drift.append(((x - start_timestamp) * endFactor) + start_timestamp) self.start_offset = 0 - def getPowerBetween(start, end, base_power=0.001469): + def getPowerBetween(start, end, base_power=0): #0.001469): first_index = 0 all_power = 0 all_power_count = 0 @@ -1841,7 +1841,44 @@ class EnergyTraceWithLogicAnalyzer: energy_trace_new = energy_trace_new[4:] - + # ******************************************************************** + # ******************************************************************** + # ******************************************************************** + # _____ _ ____ _______ _______ _____ _ _ _____ + # | __ \| | / __ \__ __|__ __|_ _| \ | |/ ____| + # | |__) | | | | | | | | | | | | | \| | | __ + # | ___/| | | | | | | | | | | | | . ` | | |_ | + # | | | |___| |__| | | | | | _| |_| |\ | |__| | + # |_| |______\____/ |_| |_| |_____|_| \_|\_____| + # + # ******************************************************************** + # ******************************************************************** + # ******************************************************************** + def calculateRectangleCurve(timestamps, min_value=0, max_value=0.160): + import numpy as np + data = [] + for ts in timestamps: + data.append(ts) + data.append(ts) + + a = np.empty((len(data),)) + a[1::4] = max_value + a[2::4] = max_value + a[3::4] = min_value + a[4::4] = min_value + return data, a # plotting by columns + + import matplotlib.pyplot as plt + rectCurve_with_drift = calculateRectangleCurve(modified_timestamps_with_drift, max_value=max(plot_data_y)) + + plt.plot(plot_data_x, plot_data_y, label='Energy') # plotting by columns + plt.plot(rectCurve_with_drift[0], rectCurve_with_drift[1], '-g', label='With calculated Driftfactor') + leg = plt.legend() + plt.show() + + # ******************************************************************** + # ******************************************************************** + # ******************************************************************** for number, item in enumerate(expected_transitions): name, duration = item @@ -1876,9 +1913,10 @@ class EnergyTraceWithLogicAnalyzer: st = "" for i, x in enumerate(energy_trace_new[-10:]): - st += "(%s|%s|%s)" % (energy_trace[i-10]["name"],x['W_mean'],x['s']) + #st += "(%s|%s|%s)" % (energy_trace[i-10]["name"],x['W_mean'],x['s']) + st += "(%s|%s|%s)\n" % (energy_trace[i-10]["s"], x['s'], x['W_mean']) - #print(st) + #print(st, "\n_______________________") print(len(self.sync_data.timestamps), " - ", len(energy_trace_new), " - ", len(energy_trace), " - ", ",".join([str(x["s"]) for x in energy_trace_new[-6:]]), " - ", ",".join([str(x["s"]) for x in energy_trace[-6:]])) if len(energy_trace_new) < len(energy_trace): return None @@ -1886,8 +1924,45 @@ class EnergyTraceWithLogicAnalyzer: class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): - pass + def __init__( + self, + voltage: float, + state_duration: int, + transition_names: list, + with_traces=False, + ): + + """ + Create a new EnergyTraceWithLogicAnalyzer object. + :param voltage: supply voltage [V], usually 3.3 V + :param state_duration: state duration [ms] + :param transition_names: list of transition names in PTA transition order. + Needed to map barcode synchronization numbers to transitions. + """ + + self.voltage = voltage + self.state_duration = state_duration * 1e-3 + self.transition_names = transition_names + self.with_traces = with_traces + self.errors = list() + + super().__init__(voltage, state_duration, transition_names, with_traces) + + def load_data(self, log_data): + from data.timing.SigrokInterface import SigrokResult + from data.energy.EnergyInterface import EnergyInterface + + # Daten laden + self.sync_data = None + self.energy_data = EnergyInterface.getDataFromString(str(log_data[1])) + + pass + + def analyze_states(self, traces, offline_index: int): + from data.timing.SigrokInterface import SigrokResult + self.sync_data = SigrokResult.fromTraces(traces) + return super().analyze_states(traces, offline_index) class MIMOSA: """ -- cgit v1.2.3 From 1f2f18c5b3ecb37ec8b2e4e9e57749d11ad0bba1 Mon Sep 17 00:00:00 2001 From: Lennart Date: Tue, 18 Aug 2020 17:43:35 +0200 Subject: Updated to Dataprocessor --- lib/loader.py | 234 +++++++++++----------------------------------------------- 1 file changed, 44 insertions(+), 190 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index e9bd7d1..bd6a49d 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -11,6 +11,8 @@ import struct import tarfile import hashlib from multiprocessing import Pool + +from data.processing.DataProcessor import DataProcessor from .utils import running_mean, soft_cast_int logger = logging.getLogger(__name__) @@ -1671,97 +1673,24 @@ class EnergyTraceWithLogicAnalyzer: * `W_mean_delta_next`: Differenz zwischen W_mean und W_mean des Folgezustands """ - # Tatsächlich Daten auswerten - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # remove Dirty Data from previously running program (happens if logic Analyzer Measurement starts earlier than - # the HW Reset from energytrace) - use_data_after_index = 0 - for x in range(1, len(self.sync_data.timestamps)): - if self.sync_data.timestamps[x] - self.sync_data.timestamps[x - 1] > 1.3: - use_data_after_index = x - break - - time_stamp_data = self.sync_data.timestamps[use_data_after_index:] - - start_offset = None - start_timestamp = None - end_offset = None - end_timestamp = None - - last_values = [] # maybe needed to flatten the data - plot_data_x = [] - plot_data_y = [] - last_data = [0, 0, 0, 0] - - power_sync_watt = 0.015 - - # MAIN ENERGY DATA ITERATION - for energytrace_dataset in self.energy_data: - usedtime = energytrace_dataset[0] - last_data[0] - usedenergy = energytrace_dataset[3] - last_data[3] - power = usedenergy / usedtime * 10 ** -3 # in watts - - if power > 0: - if power > power_sync_watt and start_offset is None: - # automatic START offset calculation - start_offset = energytrace_dataset[0] / 1_000_000 - time_stamp_data[0] - start_timestamp = energytrace_dataset[0] / 1_000_000 - if power > power_sync_watt and start_offset is not None: # TODO BETTER - # automatic END offset calculation (Drift) - end_offset = energytrace_dataset[0] / 1_000_000 - time_stamp_data[-1] - start_offset - end_timestamp = energytrace_dataset[0] / 1_000_000 - if len(last_values) > 10: - last_values.pop(0) - last_values.append(power) - # print("%s \tUsed Energy: %i \tUsed Time: %i \tPower: %f \t l_10: %f" % (now_data[0], usedenergy, usedtime, power, sum(last_values) / len(last_values))) - plot_data_x.append(energytrace_dataset[0] / 1_000_000) - plot_data_y.append(power) - - last_data = energytrace_dataset - - # -------------------- - # add start offset to the data - modified_timestamps_with_offset = [] - for x in time_stamp_data: - if x + start_offset >= 0: - modified_timestamps_with_offset.append(x + start_offset) - - # -------------------- - # Add drift to datapoints - modified_timestamps_with_drift = [] - endFactor = (end_timestamp + end_offset - start_timestamp) / (end_timestamp - start_timestamp) - for x in modified_timestamps_with_offset: - modified_timestamps_with_drift.append(((x - start_timestamp) * endFactor) + start_timestamp) - - self.start_offset = 0 - def getPowerBetween(start, end, base_power=0): #0.001469): - first_index = 0 - all_power = 0 - all_power_count = 0 - for ind in range(self.start_offset, len(plot_data_x)): - first_index = ind - if plot_data_x[ind] > start: - break - - for ind in range(first_index, len(plot_data_x)): - all_power += plot_data_y[ind] - all_power_count += 1 - if plot_data_x[ind] > end: - self.start_offset = ind - 1 - break - return (all_power / all_power_count) - base_power - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + names = [] + for trace_number, trace in enumerate(traces): + for state_or_transition in trace["trace"]: + names.append(state_or_transition["name"]) + print(names[:15]) + dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) + dp.run() + energy_trace_new = list() + energy_trace_new.extend(dp.getStatesdfatool()) + #dp.plot() + #dp.plot(names) + energy_trace_new = energy_trace_new[4:] energy_trace = list() - expected_transitions = list() for trace_number, trace in enumerate(traces): - for state_or_transition_number, state_or_transition in enumerate( - trace["trace"] - ): + for state_or_transition_number, state_or_transition in enumerate(trace["trace"]): if state_or_transition["isa"] == "transition": try: expected_transitions.append( @@ -1785,100 +1714,6 @@ class EnergyTraceWithLogicAnalyzer: return energy_trace - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - end_transition_ts = None - timestamps_sync_start = 0 - energy_trace_new = list() - - for ts_index in range(0 + timestamps_sync_start, int(len(modified_timestamps_with_drift) / 2)): - start_transition_ts = modified_timestamps_with_drift[ts_index * 2] - if end_transition_ts is not None: - power = getPowerBetween(end_transition_ts, start_transition_ts) - #print("STATE", end_transition_ts * 10 ** 6, start_transition_ts * 10 ** 6, (start_transition_ts - end_transition_ts) * 10 ** 6, power) - if (start_transition_ts - end_transition_ts) * 10 ** 6 > 900_000 and power > power_sync_watt * 0.9 and ts_index > 10: - #remove last transition and stop (upcoming data only sync) - del energy_trace_new[-1] - break - pass - - state = { - "isa": "state", - "W_mean": power, - "W_std": 0.0001, - "s": (start_transition_ts - end_transition_ts), #* 10 ** 6, - } - energy_trace_new.append(state) - - energy_trace_new[-2]["W_mean_delta_next"] = ( - energy_trace_new[-2]["W_mean"] - energy_trace_new[-1]["W_mean"] - ) - - # get energy end_transition_ts - end_transition_ts = modified_timestamps_with_drift[ts_index * 2 + 1] - power = getPowerBetween(start_transition_ts, end_transition_ts) - #print("TRANS", start_transition_ts * 10 ** 6, end_transition_ts * 10 ** 6, (end_transition_ts - start_transition_ts) * 10 ** 6, power) - transition = { - "isa": "transition", - "W_mean": power, - "W_std": 0.0001, - "s": (end_transition_ts - start_transition_ts), #* 10 ** 6, - } - - if (end_transition_ts - start_transition_ts) * 10 ** 6 > 2_000_000: - # TODO Last data set corrupted? HOT FIX!!!!!!!!!!!! REMOVE LATER - #for x in range(4): - # del energy_trace_new[-1] - #break - pass - - energy_trace_new.append(transition) - # print(start_transition_ts, "-", end_transition_ts, "-", end_transition_ts - start_transition_ts) - - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - energy_trace_new = energy_trace_new[4:] - - # ******************************************************************** - # ******************************************************************** - # ******************************************************************** - # _____ _ ____ _______ _______ _____ _ _ _____ - # | __ \| | / __ \__ __|__ __|_ _| \ | |/ ____| - # | |__) | | | | | | | | | | | | | \| | | __ - # | ___/| | | | | | | | | | | | | . ` | | |_ | - # | | | |___| |__| | | | | | _| |_| |\ | |__| | - # |_| |______\____/ |_| |_| |_____|_| \_|\_____| - # - # ******************************************************************** - # ******************************************************************** - # ******************************************************************** - def calculateRectangleCurve(timestamps, min_value=0, max_value=0.160): - import numpy as np - data = [] - for ts in timestamps: - data.append(ts) - data.append(ts) - - a = np.empty((len(data),)) - a[1::4] = max_value - a[2::4] = max_value - a[3::4] = min_value - a[4::4] = min_value - return data, a # plotting by columns - - import matplotlib.pyplot as plt - rectCurve_with_drift = calculateRectangleCurve(modified_timestamps_with_drift, max_value=max(plot_data_y)) - - plt.plot(plot_data_x, plot_data_y, label='Energy') # plotting by columns - plt.plot(rectCurve_with_drift[0], rectCurve_with_drift[1], '-g', label='With calculated Driftfactor') - leg = plt.legend() - plt.show() - - # ******************************************************************** - # ******************************************************************** - # ******************************************************************** for number, item in enumerate(expected_transitions): name, duration = item @@ -1894,7 +1729,7 @@ class EnergyTraceWithLogicAnalyzer: if len(energy_trace) > 1: energy_trace[-1]["W_mean_delta_prev"] = ( - energy_trace[-1]["W_mean"] - energy_trace[-2]["W_mean"] + energy_trace[-1]["W_mean"] - energy_trace[-2]["W_mean"] ) state = { @@ -1908,18 +1743,37 @@ class EnergyTraceWithLogicAnalyzer: energy_trace.append(state) energy_trace[-2]["W_mean_delta_next"] = ( - energy_trace[-2]["W_mean"] - energy_trace[-1]["W_mean"] + energy_trace[-2]["W_mean"] - energy_trace[-1]["W_mean"] ) - st = "" - for i, x in enumerate(energy_trace_new[-10:]): - #st += "(%s|%s|%s)" % (energy_trace[i-10]["name"],x['W_mean'],x['s']) - st += "(%s|%s|%s)\n" % (energy_trace[i-10]["s"], x['s'], x['W_mean']) + for number, item in enumerate(energy_trace): + name = item["name"] + #print(energy_trace[number - 1]["name"]) + #if name == "state" and "switchTo3K3" in energy_trace[number - 1]["name"]: + #print(name, energy_trace_new[number]["W_mean"]) + + # add next/prev state W_mean_delta + for number, item in enumerate(energy_trace_new): + if item['isa'] == 'transition' and number > 0 and number < len(energy_trace_new) - 1: + item['W_mean_delta_prev'] = energy_trace_new[number - 1] + item['W_mean_delta_next'] = energy_trace_new[number + 1] + + for number, item in enumerate(energy_trace): + name = energy_trace[number]['name'] + + if energy_trace_new[number]['isa'] == 'transition': + print(name, energy_trace_new[number]['count_dp'], energy_trace_new[number]["W_mean"]) + + #st = "" + #for i, x in enumerate(energy_trace_new[-10:]): + # #st += "(%s|%s|%s)" % (energy_trace[i-10]["name"],x['W_mean'],x['s']) + # st += "(%s|%s|%s)\n" % (energy_trace[i-10]["s"], x['s'], x['W_mean']) #print(st, "\n_______________________") - print(len(self.sync_data.timestamps), " - ", len(energy_trace_new), " - ", len(energy_trace), " - ", ",".join([str(x["s"]) for x in energy_trace_new[-6:]]), " - ", ",".join([str(x["s"]) for x in energy_trace[-6:]])) - if len(energy_trace_new) < len(energy_trace): - return None + #print(len(self.sync_data.timestamps), " - ", len(energy_trace_new), " - ", len(energy_trace), " - ", ",".join([str(x["s"]) for x in energy_trace_new[-6:]]), " - ", ",".join([str(x["s"]) for x in energy_trace[-6:]])) + #if len(energy_trace_new) < len(energy_trace): + # return None + return energy_trace_new @@ -1955,7 +1809,7 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): # Daten laden self.sync_data = None - self.energy_data = EnergyInterface.getDataFromString(str(log_data[1])) + self.energy_data = EnergyInterface.getDataFromString(str(log_data[0])) pass -- cgit v1.2.3 From ae154071f9da8105bfa0ca9c20948d471d38e8cd Mon Sep 17 00:00:00 2001 From: Lennart Date: Thu, 20 Aug 2020 13:39:15 +0200 Subject: Added state duration --- lib/loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index bd6a49d..d14680e 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1678,11 +1678,11 @@ class EnergyTraceWithLogicAnalyzer: for trace_number, trace in enumerate(traces): for state_or_transition in trace["trace"]: names.append(state_or_transition["name"]) - print(names[:15]) + #print(names[:15]) dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) dp.run() energy_trace_new = list() - energy_trace_new.extend(dp.getStatesdfatool()) + energy_trace_new.extend(dp.getStatesdfatool(state_sleep=self.state_duration)) #dp.plot() #dp.plot(names) energy_trace_new = energy_trace_new[4:] -- cgit v1.2.3 From b55d5aabeeca7020bbcad34925592ed0eee88d3c Mon Sep 17 00:00:00 2001 From: Lennart Date: Mon, 7 Sep 2020 18:30:53 +0200 Subject: Changed sync Slightly & Black --- lib/harness.py | 10 ++++------ lib/loader.py | 61 +++++++++++++++++++++++++++++++--------------------------- lib/runner.py | 6 +++--- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index 833807e..f0f2837 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -141,9 +141,7 @@ class TransitionHarness: def start_trace(self): """Prepare a new trace/run in the internal `.traces` structure.""" - self.traces.append( - {"id": self.trace_id, "trace": list(),} - ) + self.traces.append({"id": self.trace_id, "trace": list()}) self.trace_id += 1 def append_state(self, state_name, param): @@ -154,7 +152,7 @@ class TransitionHarness: :param param: parameter dict """ self.traces[-1]["trace"].append( - {"name": state_name, "isa": "state", "parameter": param,} + {"name": state_name, "isa": "state", "parameter": param} ) def append_transition(self, transition_name, param, args=[]): @@ -405,7 +403,7 @@ class OnboardTimerHarness(TransitionHarness): ret = "#define PTALOG_TIMING\n" ret += super().global_code() if self.energytrace_sync == "led": - #TODO Make nicer + # TODO Make nicer ret += """\nvoid runLASync(){ // ======================= LED SYNC ================================ ptalog.passTransition(0); @@ -424,7 +422,7 @@ class OnboardTimerHarness(TransitionHarness): gpio.led_toggle(1); ptalog.stopTransition(); // ======================= LED SYNC ================================ - arch.sleep_ms(250); + //arch.sleep_ms(250); }\n\n""" return ret diff --git a/lib/loader.py b/lib/loader.py index d14680e..4ae883e 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1648,7 +1648,7 @@ class EnergyTraceWithLogicAnalyzer: pass - def analyze_states(self, traces, offline_index: int): + def analyze_states(self, traces, offline_index: int, timer_measurement=False): u""" Split log data into states and transitions and return duration, energy, and mean power for each element. @@ -1673,24 +1673,25 @@ class EnergyTraceWithLogicAnalyzer: * `W_mean_delta_next`: Differenz zwischen W_mean und W_mean des Folgezustands """ - names = [] for trace_number, trace in enumerate(traces): for state_or_transition in trace["trace"]: names.append(state_or_transition["name"]) - #print(names[:15]) + # print(names[:15]) dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) - dp.run() + dp.run(timer_measurement) energy_trace_new = list() energy_trace_new.extend(dp.getStatesdfatool(state_sleep=self.state_duration)) - #dp.plot() - #dp.plot(names) + dp.plot() + # dp.plot(names) energy_trace_new = energy_trace_new[4:] energy_trace = list() expected_transitions = list() for trace_number, trace in enumerate(traces): - for state_or_transition_number, state_or_transition in enumerate(trace["trace"]): + for state_or_transition_number, state_or_transitistddevon in enumerate( + trace["trace"] + ): if state_or_transition["isa"] == "transition": try: expected_transitions.append( @@ -1713,8 +1714,6 @@ class EnergyTraceWithLogicAnalyzer: ) return energy_trace - - for number, item in enumerate(expected_transitions): name, duration = item transition = { @@ -1729,7 +1728,7 @@ class EnergyTraceWithLogicAnalyzer: if len(energy_trace) > 1: energy_trace[-1]["W_mean_delta_prev"] = ( - energy_trace[-1]["W_mean"] - energy_trace[-2]["W_mean"] + energy_trace[-1]["W_mean"] - energy_trace[-2]["W_mean"] ) state = { @@ -1743,35 +1742,39 @@ class EnergyTraceWithLogicAnalyzer: energy_trace.append(state) energy_trace[-2]["W_mean_delta_next"] = ( - energy_trace[-2]["W_mean"] - energy_trace[-1]["W_mean"] + energy_trace[-2]["W_mean"] - energy_trace[-1]["W_mean"] ) for number, item in enumerate(energy_trace): name = item["name"] - #print(energy_trace[number - 1]["name"]) - #if name == "state" and "switchTo3K3" in energy_trace[number - 1]["name"]: - #print(name, energy_trace_new[number]["W_mean"]) + # print(energy_trace[number - 1]["name"]) + # if name == "state" and "switchTo3K3" in energy_trace[number - 1]["name"]: + # print(name, energy_trace_new[number]["W_mean"]) # add next/prev state W_mean_delta for number, item in enumerate(energy_trace_new): - if item['isa'] == 'transition' and number > 0 and number < len(energy_trace_new) - 1: - item['W_mean_delta_prev'] = energy_trace_new[number - 1] - item['W_mean_delta_next'] = energy_trace_new[number + 1] + if item["isa"] == "transition" and 0 < number < len(energy_trace_new) - 1: + item["W_mean_delta_prev"] = energy_trace_new[number - 1] + item["W_mean_delta_next"] = energy_trace_new[number + 1] + """ for number, item in enumerate(energy_trace): - name = energy_trace[number]['name'] - - if energy_trace_new[number]['isa'] == 'transition': - print(name, energy_trace_new[number]['count_dp'], energy_trace_new[number]["W_mean"]) - - #st = "" - #for i, x in enumerate(energy_trace_new[-10:]): + name = energy_trace[number]["name"] + if energy_trace_new[number]["isa"] == "transition": + print( + name, + energy_trace_new[number]["count_dp"], + energy_trace_new[number]["W_mean"], + ) + """ + # st = "" + # for i, x in enumerate(energy_trace_new[-10:]): # #st += "(%s|%s|%s)" % (energy_trace[i-10]["name"],x['W_mean'],x['s']) # st += "(%s|%s|%s)\n" % (energy_trace[i-10]["s"], x['s'], x['W_mean']) - #print(st, "\n_______________________") - #print(len(self.sync_data.timestamps), " - ", len(energy_trace_new), " - ", len(energy_trace), " - ", ",".join([str(x["s"]) for x in energy_trace_new[-6:]]), " - ", ",".join([str(x["s"]) for x in energy_trace[-6:]])) - #if len(energy_trace_new) < len(energy_trace): + # print(st, "\n_______________________") + # print(len(self.sync_data.timestamps), " - ", len(energy_trace_new), " - ", len(energy_trace), " - ", ",".join([str(x["s"]) for x in energy_trace_new[-6:]]), " - ", ",".join([str(x["s"]) for x in energy_trace[-6:]])) + # if len(energy_trace_new) < len(energy_trace): # return None return energy_trace_new @@ -1815,8 +1818,10 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): def analyze_states(self, traces, offline_index: int): from data.timing.SigrokInterface import SigrokResult + self.sync_data = SigrokResult.fromTraces(traces) - return super().analyze_states(traces, offline_index) + return super().analyze_states(traces, offline_index, timer_measurement=True) + class MIMOSA: """ diff --git a/lib/runner.py b/lib/runner.py index 3092d5a..8093a55 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -157,7 +157,7 @@ class EnergyTraceMonitor(SerialMonitor): self._start_energytrace() def _start_energytrace(self): - print("EnergyTrace Start") + print("[%s] Starting Measurement" % type(self).__name__) cmd = ["msp430-etv", "--save", self._output, "0"] self._logger = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True @@ -165,15 +165,15 @@ class EnergyTraceMonitor(SerialMonitor): # Benchmark fertig -> externe Hilfsprogramme beenden def close(self): - print("EnergyTrace Close") super().close() self._logger.send_signal(subprocess.signal.SIGINT) stdout, stderr = self._logger.communicate(timeout=15) + print("[%s] Stopped Measurement" % type(self).__name__) # Zusätzliche Dateien, die mit dem Benchmark-Log und -Plan abgespeichert werden sollen # (hier: Die von msp430-etv generierten Logfiles) def get_files(self) -> list: - print("EnergyTrace Get Files") + print("[%s] Getting files" % type(self).__name__) return [self._output] # Benchmark-Konfiguration. Hier: Die (konstante) Spannung. -- cgit v1.2.3 From b5cd0911fd986666b229aadce48f3d372f4b7e13 Mon Sep 17 00:00:00 2001 From: Lennart Date: Mon, 7 Sep 2020 18:44:04 +0200 Subject: test --- lib/harness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/harness.py b/lib/harness.py index f0f2837..ed6cdac 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -422,7 +422,7 @@ class OnboardTimerHarness(TransitionHarness): gpio.led_toggle(1); ptalog.stopTransition(); // ======================= LED SYNC ================================ - //arch.sleep_ms(250); + arch.sleep_ms(200); }\n\n""" return ret -- cgit v1.2.3 From aa0a56a8f1ba32235e1b2664835e35b6641a5715 Mon Sep 17 00:00:00 2001 From: Lennart Date: Wed, 9 Sep 2020 13:20:43 +0200 Subject: changed stuff --- lib/harness.py | 2 +- lib/loader.py | 81 +++++----------------------------------------------------- 2 files changed, 8 insertions(+), 75 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index ed6cdac..51013e1 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -422,7 +422,7 @@ class OnboardTimerHarness(TransitionHarness): gpio.led_toggle(1); ptalog.stopTransition(); // ======================= LED SYNC ================================ - arch.sleep_ms(200); + arch.sleep_ms(250); }\n\n""" return ret diff --git a/lib/loader.py b/lib/loader.py index 4ae883e..b4b2127 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1648,7 +1648,7 @@ class EnergyTraceWithLogicAnalyzer: pass - def analyze_states(self, traces, offline_index: int, timer_measurement=False): + def analyze_states(self, traces, offline_index: int): u""" Split log data into states and transitions and return duration, energy, and mean power for each element. @@ -1679,7 +1679,7 @@ class EnergyTraceWithLogicAnalyzer: names.append(state_or_transition["name"]) # print(names[:15]) dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) - dp.run(timer_measurement) + dp.run() energy_trace_new = list() energy_trace_new.extend(dp.getStatesdfatool(state_sleep=self.state_duration)) dp.plot() @@ -1688,68 +1688,11 @@ class EnergyTraceWithLogicAnalyzer: energy_trace = list() expected_transitions = list() - for trace_number, trace in enumerate(traces): - for state_or_transition_number, state_or_transitistddevon in enumerate( - trace["trace"] - ): - if state_or_transition["isa"] == "transition": - try: - expected_transitions.append( - ( - state_or_transition["name"], - state_or_transition["online_aggregates"]["duration"][ - offline_index - ] - * 1e-6, - ) - ) - except IndexError: - self.errors.append( - 'Entry #{} ("{}") in trace #{} has no duration entry for offline_index/repeat_id {}'.format( - state_or_transition_number, - state_or_transition["name"], - trace_number, - offline_index, - ) - ) - return energy_trace - for number, item in enumerate(expected_transitions): - name, duration = item - transition = { - "isa": "transition", - "W_mean": max(np.random.normal(0.023, 0.002), 0), - "W_std": 0.0001, - "s": duration, - "name": name, - } - - energy_trace.append(transition) - - if len(energy_trace) > 1: - energy_trace[-1]["W_mean_delta_prev"] = ( - energy_trace[-1]["W_mean"] - energy_trace[-2]["W_mean"] - ) - - state = { - "isa": "state", - "W_mean": max(np.random.normal(0.023, 0.002), 0), - "W_std": 0.0001, - "s": self.state_duration, - "name": "state", - } - - energy_trace.append(state) - - energy_trace[-2]["W_mean_delta_next"] = ( - energy_trace[-2]["W_mean"] - energy_trace[-1]["W_mean"] - ) - - for number, item in enumerate(energy_trace): - name = item["name"] - # print(energy_trace[number - 1]["name"]) - # if name == "state" and "switchTo3K3" in energy_trace[number - 1]["name"]: - # print(name, energy_trace_new[number]["W_mean"]) + # Print for debug purposes + # for number, name in enumerate(names): + # if "P15_8MW" in name: + # print(name, energy_trace_new[number]["W_mean"]) # add next/prev state W_mean_delta for number, item in enumerate(energy_trace_new): @@ -1757,16 +1700,6 @@ class EnergyTraceWithLogicAnalyzer: item["W_mean_delta_prev"] = energy_trace_new[number - 1] item["W_mean_delta_next"] = energy_trace_new[number + 1] - """ - for number, item in enumerate(energy_trace): - name = energy_trace[number]["name"] - if energy_trace_new[number]["isa"] == "transition": - print( - name, - energy_trace_new[number]["count_dp"], - energy_trace_new[number]["W_mean"], - ) - """ # st = "" # for i, x in enumerate(energy_trace_new[-10:]): # #st += "(%s|%s|%s)" % (energy_trace[i-10]["name"],x['W_mean'],x['s']) @@ -1820,7 +1753,7 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): from data.timing.SigrokInterface import SigrokResult self.sync_data = SigrokResult.fromTraces(traces) - return super().analyze_states(traces, offline_index, timer_measurement=True) + return super().analyze_states(traces, offline_index) class MIMOSA: -- cgit v1.2.3 From f8cf50af8564094e145124af08e5e12e13ed352f Mon Sep 17 00:00:00 2001 From: Lennart Date: Tue, 22 Sep 2020 00:03:23 +0200 Subject: Changed code import and black formatted --- lib/loader.py | 3 ++- lib/runner.py | 27 ++++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index b4b2127..ea2b183 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -12,7 +12,6 @@ import tarfile import hashlib from multiprocessing import Pool -from data.processing.DataProcessor import DataProcessor from .utils import running_mean, soft_cast_int logger = logging.getLogger(__name__) @@ -1678,6 +1677,8 @@ class EnergyTraceWithLogicAnalyzer: for state_or_transition in trace["trace"]: names.append(state_or_transition["name"]) # print(names[:15]) + from data.processing.DataProcessor import DataProcessor + dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) dp.run() energy_trace_new = list() diff --git a/lib/runner.py b/lib/runner.py index 8093a55..4cab9ed 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -180,9 +180,7 @@ class EnergyTraceMonitor(SerialMonitor): # MSP430FR5969: 3,6V (wird aktuell nicht unterstützt) # MSP430FR5994: 3,3V (default) def get_config(self) -> dict: - return { - "voltage": self._voltage, - } + return {"voltage": self._voltage} class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): @@ -191,13 +189,16 @@ class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): def __init__(self, port: str, baud: int, callback=None, voltage=3.3): super().__init__(port=port, baud=baud, callback=callback, voltage=voltage) - #TODO Max length - options = {'length': 90, 'fake': False, 'sample_rate': 1_000_000} - self.log_file = 'logic_output_log_%s.json' % (time.strftime("%Y%m%d-%H%M%S")) + # TODO Max length + options = {"length": 90, "fake": False, "sample_rate": 1_000_000} + self.log_file = "logic_output_log_%s.json" % (time.strftime("%Y%m%d-%H%M%S")) # Initialization of Interfaces - self.sig = SigrokCLIInterface(sample_rate=options['sample_rate'], - sample_count=options['length'] * options['sample_rate'], fake=options['fake']) + self.sig = SigrokCLIInterface( + sample_rate=options["sample_rate"], + sample_count=options["length"] * options["sample_rate"], + fake=options["fake"], + ) # Start Measurements self.sig.runMeasureAsynchronous() @@ -205,11 +206,11 @@ class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): def close(self): super().close() # Read measured data - #self.sig.waitForAsynchronousMeasure() + # self.sig.waitForAsynchronousMeasure() self.sig.forceStopMeasure() time.sleep(0.2) sync_data = self.sig.getData() - with open(self.log_file, 'w') as fp: + with open(self.log_file, "w") as fp: json.dump(sync_data.getDict(), fp) def get_files(self) -> list: @@ -293,11 +294,7 @@ class MIMOSAMonitor(SerialMonitor): return [self.mim_file] def get_config(self) -> dict: - return { - "offset": self._offset, - "shunt": self._shunt, - "voltage": self._voltage, - } + return {"offset": self._offset, "shunt": self._shunt, "voltage": self._voltage} class ShellMonitor: -- cgit v1.2.3 From c3958b67834268792235faef7cfca86f1d8e8195 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Wed, 7 Oct 2020 15:26:10 +0200 Subject: analyze-archive: move trace plotting to a helper function --- bin/analyze-archive.py | 52 ++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index 66772e6..ca36745 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -223,6 +223,32 @@ def print_html_model_data(model, pm, pq, lm, lq, am, ai, aq): print("") print("") +def plot_traces(preprocessed_data, sot_name): + traces = list() + for trace in preprocessed_data: + for state_or_transition in trace["trace"]: + if state_or_transition["name"] == sot_name: + traces.extend( + map(lambda x: x["uW"], state_or_transition["offline"]) + ) + if len(traces) == 0: + print( + f"""Did not find traces for state or transition {sot_name}. Abort.""", + file=sys.stderr, + ) + sys.exit(2) + + if len(traces) > 40: + print(f"""Truncating plot to 40 of {len(traces)} traces (random sample)""") + traces = random.sample(traces, 40) + + plotter.plot_y( + traces, + xlabel="t [1e-5 s]", + ylabel="P [uW]", + title=sot_name, + family=True, + ) if __name__ == "__main__": @@ -446,31 +472,7 @@ if __name__ == "__main__": json.dump(data, f) if args.plot_traces: - traces = list() - for trace in preprocessed_data: - for state_or_transition in trace["trace"]: - if state_or_transition["name"] == args.plot_traces: - traces.extend( - map(lambda x: x["uW"], state_or_transition["offline"]) - ) - if len(traces) == 0: - print( - f"""Did not find traces for state or transition {args.plot_traces}. Abort.""", - file=sys.stderr, - ) - sys.exit(2) - - if len(traces) > 40: - print(f"""Truncating plot to 40 of {len(traces)} traces (random sample)""") - traces = random.sample(traces, 40) - - plotter.plot_y( - traces, - xlabel="t [1e-5 s]", - ylabel="P [uW]", - title=args.plot_traces, - family=True, - ) + plot_traces(preprocessed_data, args.plot_traces) if raw_data.preprocessing_stats["num_valid"] == 0: print("No valid data available. Abort.", file=sys.stderr) -- cgit v1.2.3 From 81082d00444d3bfd644f2417ef5c1a34d9569a28 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 16 Oct 2020 13:58:20 +0200 Subject: Code aus Lennarts BA-repo --- lib/lennart/DataInterface.py | 24 +++ lib/lennart/DataProcessor.py | 374 ++++++++++++++++++++++++++++++++++++++ lib/lennart/EnergyInterface.py | 119 ++++++++++++ lib/lennart/SigrokAPIInterface.py | 152 ++++++++++++++++ lib/lennart/SigrokCLIInterface.py | 78 ++++++++ lib/lennart/SigrokInterface.py | 165 +++++++++++++++++ lib/lennart/__init__.py | 0 lib/loader.py | 12 +- lib/runner.py | 2 +- 9 files changed, 919 insertions(+), 7 deletions(-) create mode 100644 lib/lennart/DataInterface.py create mode 100644 lib/lennart/DataProcessor.py create mode 100644 lib/lennart/EnergyInterface.py create mode 100644 lib/lennart/SigrokAPIInterface.py create mode 100644 lib/lennart/SigrokCLIInterface.py create mode 100644 lib/lennart/SigrokInterface.py create mode 100644 lib/lennart/__init__.py diff --git a/lib/lennart/DataInterface.py b/lib/lennart/DataInterface.py new file mode 100644 index 0000000..4495db2 --- /dev/null +++ b/lib/lennart/DataInterface.py @@ -0,0 +1,24 @@ +class DataInterface: + def runMeasure(self): + """ + Implemented in subclasses. + + Starts the measurement + """ + raise NotImplementedError("The method not implemented") + + def getData(self): + """ + Implemented in subclasses + + :returns: gathered data + """ + raise NotImplementedError("The method not implemented") + + def forceStopMeasure(self): + """ + Implemented in subclasses + + Force stops the measurement + """ + raise NotImplementedError("The method not implemented") diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py new file mode 100644 index 0000000..df3f41c --- /dev/null +++ b/lib/lennart/DataProcessor.py @@ -0,0 +1,374 @@ +class DataProcessor: + def __init__(self, sync_data, energy_data): + """ + Creates DataProcessor object. + + :param sync_data: input timestamps (SigrokResult) + :param energy_data: List of EnergyTrace datapoints + """ + self.reduced_timestamps = [] + self.modified_timestamps = [] + self.plot_data_x = [] + self.plot_data_y = [] + self.sync_data = sync_data + self.energy_data = energy_data + self.start_offset = 0 + + self.power_sync_watt = 0.011 + self.power_sync_len = 0.7 + self.power_sync_max_outliers = 2 + + def run(self): + """ + Main Function to remove unwanted data, get synchronization points, add the offset and add drift. + :return: None + """ + # remove Dirty Data from previously running program (happens if logic Analyzer Measurement starts earlier than + # the HW Reset from energytrace) + use_data_after_index = 0 + for x in range(1, len(self.sync_data.timestamps)): + if self.sync_data.timestamps[x] - self.sync_data.timestamps[x - 1] > 1.3: + use_data_after_index = x + break + + time_stamp_data = self.sync_data.timestamps[use_data_after_index:] + + last_data = [0, 0, 0, 0] + + # clean timestamp data, if at the end strange ts got added somehow + time_stamp_data = self.removeTooFarDatasets(time_stamp_data) + + self.reduced_timestamps = time_stamp_data + + # NEW + datasync_timestamps = [] + sync_start = 0 + outliers = 0 + pre_outliers_ts = None + for i, energytrace_dataset in enumerate(self.energy_data): + usedtime = energytrace_dataset[0] - last_data[0] # in microseconds + timestamp = energytrace_dataset[0] + usedenergy = energytrace_dataset[3] - last_data[3] + power = usedenergy / usedtime * 10 ** -3 # in watts + if power > 0: + if power > self.power_sync_watt: + if sync_start is None: + sync_start = timestamp + outliers = 0 + else: + # Sync point over or outliers + if outliers == 0: + pre_outliers_ts = timestamp + outliers += 1 + if outliers > self.power_sync_max_outliers: + if sync_start is not None: + if ( + pre_outliers_ts - sync_start + ) / 1_000_000 > self.power_sync_len: + datasync_timestamps.append( + ( + sync_start / 1_000_000, + pre_outliers_ts / 1_000_000, + ) + ) + sync_start = None + + last_data = energytrace_dataset + + self.plot_data_x.append(energytrace_dataset[0] / 1_000_000) + self.plot_data_y.append(power) + + if power > self.power_sync_watt: + if (self.energy_data[-1][0] - sync_start) / 1_000_000 > self.power_sync_len: + datasync_timestamps.append( + (sync_start / 1_000_000, pre_outliers_ts / 1_000_000) + ) + + # print("SYNC SPOTS: ", datasync_timestamps) + # print(time_stamp_data[2]) + + start_offset = datasync_timestamps[0][1] - time_stamp_data[2] + start_timestamp = datasync_timestamps[0][1] + + end_offset = datasync_timestamps[-2][0] - (time_stamp_data[-8] + start_offset) + end_timestamp = datasync_timestamps[-2][0] + print(start_timestamp, end_timestamp) + + # print(start_offset, start_timestamp, end_offset, end_timestamp) + + with_offset = self.addOffset(time_stamp_data, start_offset) + + with_drift = self.addDrift( + with_offset, end_timestamp, end_offset, start_timestamp + ) + + self.modified_timestamps = with_drift + + def addOffset(self, input_timestamps, start_offset): + """ + Add begin offset at start + + :param input_timestamps: List of timestamps (float list) + :param start_offset: Timestamp of last EnergyTrace datapoint at the first sync point + :return: List of modified timestamps (float list) + """ + modified_timestamps_with_offset = [] + for x in input_timestamps: + if x + start_offset >= 0: + modified_timestamps_with_offset.append(x + start_offset) + return modified_timestamps_with_offset + + def removeTooFarDatasets(self, input_timestamps): + """ + Removing datasets, that are to far away at ethe end + + :param input_timestamps: List of timestamps (float list) + :return: List of modified timestamps (float list) + """ + modified_timestamps = [] + for i, x in enumerate(input_timestamps): + # print(x - input_timestamps[i - 1], x - input_timestamps[i - 1] < 2.5) + if x - input_timestamps[i - 1] < 1.6: + modified_timestamps.append(x) + else: + break + return modified_timestamps + + def addDrift(self, input_timestamps, end_timestamp, end_offset, start_timestamp): + """ + Add drift to datapoints + + :param input_timestamps: List of timestamps (float list) + :param end_timestamp: Timestamp of first EnergyTrace datapoint at the second last sync point + :param end_offset: the time between end_timestamp and the timestamp of synchronisation signal + :param start_timestamp: Timestamp of last EnergyTrace datapoint at the first sync point + :return: List of modified timestamps (float list) + """ + endFactor = (end_timestamp + end_offset - start_timestamp) / ( + end_timestamp - start_timestamp + ) + modified_timestamps_with_drift = [] + for x in input_timestamps: + modified_timestamps_with_drift.append( + ((x - start_timestamp) * endFactor) + start_timestamp + ) + + return modified_timestamps_with_drift + + def plot(self, annotateData=None): + """ + Plots the power usage and the timestamps by logic analyzer + + :param annotateData: List of Strings with labels, only needed if annotated plots are wished + :return: None + """ + + def calculateRectangleCurve(timestamps, min_value=0, max_value=0.160): + import numpy as np + + data = [] + for ts in timestamps: + data.append(ts) + data.append(ts) + + a = np.empty((len(data),)) + a[1::4] = max_value + a[2::4] = max_value + a[3::4] = min_value + a[4::4] = min_value + return data, a # plotting by columns + + import matplotlib.pyplot as plt + + fig, ax = plt.subplots() + + if annotateData: + annot = ax.annotate( + "", + xy=(0, 0), + xytext=(20, 20), + textcoords="offset points", + bbox=dict(boxstyle="round", fc="w"), + arrowprops=dict(arrowstyle="->"), + ) + annot.set_visible(True) + + rectCurve_with_drift = calculateRectangleCurve( + self.modified_timestamps, max_value=max(self.plot_data_y) + ) + + plt.plot(self.plot_data_x, self.plot_data_y, label="Leistung") + + plt.plot( + rectCurve_with_drift[0], + rectCurve_with_drift[1], + "-g", + label="Synchronisationsignale mit Driftfaktor", + ) + + plt.xlabel("Zeit [s]") + plt.ylabel("Leistung [W]") + leg = plt.legend() + + def getDataText(x): + # print(x) + for i, xt in enumerate(self.modified_timestamps): + if xt > x: + return "Value: %s" % annotateData[i - 5] + + def update_annot(x, y, name): + annot.xy = (x, y) + text = name + + annot.set_text(text) + annot.get_bbox_patch().set_alpha(0.4) + + def hover(event): + if event.xdata and event.ydata: + annot.set_visible(False) + update_annot(event.xdata, event.ydata, getDataText(event.xdata)) + annot.set_visible(True) + fig.canvas.draw_idle() + + if annotateData: + fig.canvas.mpl_connect("motion_notify_event", hover) + + plt.show() + + def getPowerBetween(self, start, end, state_sleep): # 0.001469 + """ + calculates the average powerusage in interval + NOT SIDE EFFECT FREE, DON'T USE IT EVERYWHERE + + :param start: Start timestamp of interval + :param end: End timestamp of interval + :param state_sleep: Length in seconds of one state, needed for cutting out the UART Sending cycle + :return: float with average power usage + """ + first_index = 0 + all_power = [] + for ind in range(self.start_offset, len(self.plot_data_x)): + first_index = ind + if self.plot_data_x[ind] > start: + break + + nextIndAfterIndex = None + for ind in range(first_index, len(self.plot_data_x)): + nextIndAfterIndex = ind + if ( + self.plot_data_x[ind] > end + or self.plot_data_x[ind] > start + state_sleep + ): + self.start_offset = ind - 1 + break + all_power.append(self.plot_data_y[ind]) + + # TODO Idea remove datapoints that are too far away + def removeSD_Mean_Values(arr): + import numpy + + elements = numpy.array(arr) + + mean = numpy.mean(elements, axis=0) + sd = numpy.std(elements, axis=0) + + return [x for x in arr if (mean - 1 * sd < x < mean + 1.5 * sd)] + + if len(all_power) > 10: + # all_power = removeSD_Mean_Values(all_power) + pass + # TODO algorithm relocate datapoint + + pre_fix_len = len(all_power) + if len(all_power) == 0: + # print("PROBLEM") + all_power.append(self.plot_data_y[nextIndAfterIndex]) + elif len(all_power) == 1: + # print("OKAY") + pass + return pre_fix_len, sum(all_power) / len(all_power) + + def getStatesdfatool(self, state_sleep, algorithm=False): + """ + Calculates the length and energy usage of the states + + :param state_sleep: Length in seconds of one state, needed for cutting out the UART Sending cycle + :param algorithm: possible usage of accuracy algorithm / not implemented yet + :returns: returns list of states and transitions, starting with a transition and ending with astate + Each element is a dict containing: + * `isa`: 'state' or 'transition' + * `W_mean`: Mittelwert der Leistungsaufnahme + * `W_std`: Standardabweichung der Leistungsaufnahme + * `s`: Dauer + """ + if algorithm: + raise NotImplementedError + end_transition_ts = None + timestamps_sync_start = 0 + energy_trace_new = list() + + for ts_index in range( + 0 + timestamps_sync_start, int(len(self.modified_timestamps) / 2) + ): + start_transition_ts = self.modified_timestamps[ts_index * 2] + start_transition_ts_timing = self.reduced_timestamps[ts_index * 2] + + if end_transition_ts is not None: + count_dp, power = self.getPowerBetween( + end_transition_ts, start_transition_ts, state_sleep + ) + + # print("STATE", end_transition_ts * 10 ** 6, start_transition_ts * 10 ** 6, (start_transition_ts - end_transition_ts) * 10 ** 6, power) + if ( + (start_transition_ts - end_transition_ts) * 10 ** 6 > 900_000 + and power > self.power_sync_watt * 0.9 + and ts_index > 10 + ): + # remove last transition and stop (upcoming data only sync) + del energy_trace_new[-1] + break + pass + + state = { + "isa": "state", + "W_mean": power, + "W_std": 0.0001, + "s": ( + start_transition_ts_timing - end_transition_ts_timing + ), # * 10 ** 6, + } + energy_trace_new.append(state) + + energy_trace_new[-2]["W_mean_delta_next"] = ( + energy_trace_new[-2]["W_mean"] - energy_trace_new[-1]["W_mean"] + ) + + # get energy end_transition_ts + end_transition_ts = self.modified_timestamps[ts_index * 2 + 1] + count_dp, power = self.getPowerBetween( + start_transition_ts, end_transition_ts, state_sleep + ) + + # print("TRANS", start_transition_ts * 10 ** 6, end_transition_ts * 10 ** 6, (end_transition_ts - start_transition_ts) * 10 ** 6, power) + end_transition_ts_timing = self.reduced_timestamps[ts_index * 2 + 1] + + transition = { + "isa": "transition", + "W_mean": power, + "W_std": 0.0001, + "s": ( + end_transition_ts_timing - start_transition_ts_timing + ), # * 10 ** 6, + "count_dp": count_dp, + } + + if (end_transition_ts - start_transition_ts) * 10 ** 6 > 2_000_000: + # TODO Last data set corrupted? HOT FIX!!!!!!!!!!!! REMOVE LATER + # for x in range(4): + # del energy_trace_new[-1] + # break + pass + + energy_trace_new.append(transition) + # print(start_transition_ts, "-", end_transition_ts, "-", end_transition_ts - start_transition_ts) + return energy_trace_new diff --git a/lib/lennart/EnergyInterface.py b/lib/lennart/EnergyInterface.py new file mode 100644 index 0000000..2b23667 --- /dev/null +++ b/lib/lennart/EnergyInterface.py @@ -0,0 +1,119 @@ +import re +import subprocess + +from dfatool.lennart.DataInterface import DataInterface + + +class EnergyInterface(DataInterface): + def __init__( + self, + duration_seconds=10, + console_output=False, + temp_file="temp/energytrace.log", + fake=False, + ): + """ + class is not used in embedded into dfatool. + + :param duration_seconds: seconds the EnergyTrace should be running + :param console_output: if EnergyTrace output should be printed to the user + :param temp_file: file path for temporary file + :param fake: if already existing file should be used + """ + self.energytrace = None + self.duration_seconds = duration_seconds + self.console_output = console_output + self.temp_file = temp_file + self.fake = fake + + def runMeasure(self): + """ + starts the measurement, with waiting for done + """ + if self.fake: + return + self.runMeasureAsynchronously() + self.waitForAsynchronousMeasure() + + def runMeasureAsynchronously(self): + """ + starts the measurement, not waiting for done + """ + if self.fake: + return + self.energytrace = subprocess.Popen( + "msp430-etv --save %s %s %s" + % ( + self.temp_file, + self.duration_seconds, + "" if self.console_output else "> /dev/null", + ), + shell=True, + ) + print( + "msp430-etv --save %s %s %s" + % ( + self.temp_file, + self.duration_seconds, + "" if self.console_output else "> /dev/null", + ) + ) + + def waitForAsynchronousMeasure(self): + """ + Wait until is command call is done + """ + if self.fake: + return + self.energytrace.wait() + + def getData(self): + """ + cleans the string data and creates int list + :return: list of data, in format [[int,int,int,int], [int,int,int,int], ... ] + """ + energytrace_log = open(self.temp_file) + lines = energytrace_log.readlines()[21:] + data = [] + for line in lines: + if "MSP430_DisableEnergyTrace" in line: + break + else: + data.append([int(i) for i in line.split()]) + return data + + @classmethod + def getDataFromString(cls, string, delimiter="\\n"): + """ + Parsing the data from string + + :param string: input string which will be parsed + :param delimiter: for normal file its \n + :return: list of data, in format [[int,int,int,int], [int,int,int,int], ... ] + """ + lines = string.split(delimiter)[21:] + data = [] + for line in lines: + if "MSP430_DisableEnergyTrace" in line: + break + else: + data.append([int(i) for i in line.split()]) + return data + + def setFile(self, path): + """ + changeing the temporary file + + :param path: file path of new temp file + :return: None + """ + self.temp_file = path + pass + + def forceStopMeasure(self): + """ + force stops the Measurement, with signals + :return: None + """ + self.energytrace.send_signal(subprocess.signal.SIGINT) + stdout, stderr = self.energytrace.communicate(timeout=15) diff --git a/lib/lennart/SigrokAPIInterface.py b/lib/lennart/SigrokAPIInterface.py new file mode 100644 index 0000000..a8765ba --- /dev/null +++ b/lib/lennart/SigrokAPIInterface.py @@ -0,0 +1,152 @@ +import time + +from dfatool.lennart.SigrokInterface import SigrokInterface + +import sigrok.core as sr +from sigrok.core.classes import * + +from util.ByteHelper import ByteHelper + + +class SigrokAPIInterface(SigrokInterface): + def datafeed_changes(self, device, packet): + """ + Callback type with changes analysis + :param device: device object + :param packet: data (String with binary data) + """ + data = ByteHelper.rawbytes(self.output.receive(packet)) + if data: + # only using every second byte, + # because only every second contains the useful information. + for x in data[1::2]: + self.analyzeData(x) + + def datafeed_in_all(self, device, packet): + """ + Callback type which writes all data into the array + :param device: device object + :param packet: data (String with binary data) + """ + data = ByteHelper.rawbytes(self.output.receive(packet)) + if data: + # only using every second byte, + # because only every second contains the useful information. + self.all_data += data[1::2] + + def datafeed_file(self, device, packet): + """ + Callback type which writes all data into a file + :param device: device object + :param packet: data (String with binary data) + """ + data = ByteHelper.rawbytes(self.output.receive(packet)) + if data: + # only using every second byte, + # because only every second contains the useful information. + for x in data[1::2]: + self.file.write(str(x) + "\n") + + def __init__( + self, + driver="fx2lafw", + sample_rate=100_000, + sample_count=1_000_000, + debug_output=False, + used_datafeed=datafeed_changes, + fake=False, + ): + """ + + :param driver: Driver that should be used + :param sample_rate: The sample rate of the Logic analyzer + :param sample_count: The sample count of the Logic analyzer + :param debug_output: Should be true if output should be displayed to user + :param used_datafeed: one of the datafeeds above, user later as callback. + :param fake: + """ + super(SigrokAPIInterface, self).__init__(sample_rate, sample_count) + if fake: + raise NotImplementedError("Not implemented!") + self.used_datafeed = used_datafeed + + self.debug_output = debug_output + self.session = None + + def forceStopMeasure(self): + """ + Force stopping the measurement + :return: None + """ + self.session.stop() + + def runMeasure(self): + """ + Start the Measurement and set all settings + """ + context = sr.Context_create() + + devs = context.drivers[self.driver].scan() + # print(devs) + if len(devs) == 0: + raise RuntimeError("No device with that driver found!") + sigrokDevice = devs[0] + if len(devs) > 1: + raise Warning( + "Attention! Multiple devices with that driver found! Using ", + sigrokDevice.connection_id(), + ) + + sigrokDevice.open() + sigrokDevice.config_set(ConfigKey.LIMIT_SAMPLES, self.sample_count) + sigrokDevice.config_set(ConfigKey.SAMPLERATE, self.sample_rate) + + enabled_channels = ["D1"] + for channel in sigrokDevice.channels: + channel.enabled = channel.name in enabled_channels + + self.session = context.create_session() + self.session.add_device(sigrokDevice) + self.session.start() + + self.output = context.output_formats["binary"].create_output(sigrokDevice) + + print(context.output_formats) + self.all_data = b"" + + def datafeed(device, packet): + self.used_datafeed(self, device, packet) + + self.session.add_datafeed_callback(datafeed) + time_running = time.time() + self.session.run() + total_time = time.time() - time_running + print( + "Used time: ", + total_time * 1_000_000, + "µs | sample/s: ", + self.sample_count / (total_time), + "Hz ", + ) + self.session.stop() + + if self.debug_output: + # if self.used_datafeed == self.datafeed_in_change: + if True: + changes = [x / self.sample_rate for x in self.changes] + print(changes) + is_on = self.start == 0xFF + print("0", " - ", changes[0], " # Pin ", "HIGH" if is_on else "LOW") + for x in range(len(changes) - 1): + is_on = not is_on + print( + changes[x], + " - ", + changes[x + 1], + " / ", + changes[x + 1] - changes[x], + " # Pin ", + "HIGH" if is_on else "LOW", + ) + elif self.used_datafeed == self.datafeed_in_all: + print(self.all_data) diff --git a/lib/lennart/SigrokCLIInterface.py b/lib/lennart/SigrokCLIInterface.py new file mode 100644 index 0000000..873b226 --- /dev/null +++ b/lib/lennart/SigrokCLIInterface.py @@ -0,0 +1,78 @@ +import subprocess +import time + +from dfatool.lennart.SigrokInterface import SigrokInterface + + +class SigrokCLIInterface(SigrokInterface): + def __init__( + self, + bin_temp_file="temp/out.bin", + sample_rate=100_000, + sample_count=1_000_000, + fake=False, + ): + """ + creates SigrokCLIInterface object. Uses the CLI Interface (Command: sigrok-cli) + + :param bin_temp_file: temporary file for binary output + :param sample_rate: The sample rate of the Logic analyzer + :param sample_count: The sample count of the Logic analyzer + :param fake: if it should use existing data + """ + super(SigrokCLIInterface, self).__init__(sample_rate, sample_count) + self.fake = fake + self.bin_temp_file = bin_temp_file + self.sigrok_cli_thread = None + + def forceStopMeasure(self): + """ + Force stopping measure, sometimes needs pkill for killing definitly + :return: None + """ + self.sigrok_cli_thread.send_signal(subprocess.signal.SIGKILL) + stdout, stderr = self.sigrok_cli_thread.communicate(timeout=15) + time.sleep(5) + # TODO not nice solution, make better + import os + + os.system("pkill -f sigrok") + self.runOpenAnalyze() + + def runMeasure(self): + """ + starts the measurement, with waiting for done + """ + if not self.fake: + self.runMeasureAsynchronous() + self.waitForAsynchronousMeasure() + + def runMeasureAsynchronous(self): + """ + starts the measurement, not waiting for done + """ + shellcommand = ( + 'sigrok-cli --output-file %s --output-format binary --samples %s -d %s --config "samplerate=%s Hz"' + % (self.bin_temp_file, self.sample_count, self.driver, self.sample_rate) + ) + self.sigrok_cli_thread = subprocess.Popen(shellcommand, shell=True) + + def waitForAsynchronousMeasure(self): + """ + Wait until is command call is done + """ + if not self.fake: + self.sigrok_cli_thread.wait() + self.runOpenAnalyze() + + def runOpenAnalyze(self): + """ + Opens the generated binary file and parses it byte by byte + + """ + in_file = open(self.bin_temp_file, "rb") # opening for [r]eading as [b]inary + data = in_file.read() # if you only wanted to read 512 bytes, do .read(512) + in_file.close() + + for x in data: + self.analyzeData(x) diff --git a/lib/lennart/SigrokInterface.py b/lib/lennart/SigrokInterface.py new file mode 100644 index 0000000..555a25f --- /dev/null +++ b/lib/lennart/SigrokInterface.py @@ -0,0 +1,165 @@ +import json + +from dfatool.lennart.DataInterface import DataInterface + + +# Adding additional parsing functionality +class SigrokResult: + def __init__(self, timestamps, onbeforefirstchange): + """ + Creates SigrokResult object, struct for timestamps and onBeforeFirstChange. + + :param timestamps: list of changing timestamps + :param onbeforefirstchange: if the state before the first change is already on / should always be off, just to be data correct + """ + self.timestamps = timestamps + self.onBeforeFirstChange = onbeforefirstchange + + def __str__(self): + """ + :return: string representation of object + """ + return "" % ( + self.onBeforeFirstChange, + self.timestamps, + ) + + def getDict(self): + """ + :return: dict representation of object + """ + data = { + "onBeforeFirstChange": self.onBeforeFirstChange, + "timestamps": self.timestamps, + } + return data + + @classmethod + def fromFile(cls, path): + """ + Generates SigrokResult from json_file + + :param path: file path + :return: SigrokResult object + """ + with open(path) as json_file: + data = json.load(json_file) + return SigrokResult(data["timestamps"], data["onBeforeFirstChange"]) + pass + + @classmethod + def fromString(cls, string): + """ + Generates SigrokResult from string + + :param string: string + :return: SigrokResult object + """ + data = json.loads(string) + return SigrokResult(data["timestamps"], data["onBeforeFirstChange"]) + pass + + @classmethod + def fromTraces(cls, traces): + """ + Generates SigrokResult from ptalog.json traces + + :param traces: traces from dfatool ptalog.json + :return: SigrokResult object + """ + timestamps = [0] + for tr in traces: + for t in tr["trace"]: + # print(t['online_aggregates']['duration'][0]) + timestamps.append( + timestamps[-1] + (t["online_aggregates"]["duration"][0] * 10 ** -6) + ) + + # print(timestamps) + # prepend FAKE Sync point + t_neu = [0.0, 0.0000001, 1.0, 1.00000001] + for i, x in enumerate(timestamps): + t_neu.append( + round(float(x) + t_neu[3] + 0.20, 6) + ) # list(map(float, t_ist.split(",")[:i+1])) + + # append FAKE Sync point / eine überschneidung + # [30.403632, 30.403639, 31.407265, 31.407271] + # appendData = [29.144855,30.148495,30.148502,30.403632,30.403639,31.407265,31.407271,] + appendData = [0, 1.000001, 1.000002, 1.25, 1.2500001] + + # TODO future work here, why does the sync not work completely + t_neu[-1] = ( + t_neu[-2] + (t_neu[-1] - t_neu[-2]) * 0.9 + ) # Weird offset failure with UART stuff + + offset = t_neu[-1] - appendData[0] + for x in appendData: + t_neu.append(x + offset) + + # print(t_neu) + print(len(t_neu)) + return SigrokResult(t_neu, False) + + +class SigrokInterface(DataInterface): + def __init__( + self, sample_rate, sample_count, driver="fx2lafw", filename="temp/sigrok.log" + ): + """ + + :param sample_rate: Samplerate of the Logic Analyzer + :param sample_count: Count of samples + :param driver: for many Logic Analyzer from Saleae the "fx2lafw" should be working + :param filename: temporary file name + """ + # options + self.sample_count = sample_count + self.sample_rate = sample_rate + self.file = open(filename, "w+") + self.driver = driver + + # internal data + self.changes = [] + self.start = None + self.last_val = None + self.index = 0 + + def runMeasure(self): + """ + Not implemented because implemented in subclasses + :return: None + """ + raise NotImplementedError("The method not implemented") + + def forceStopMeasure(self): + """ + Not implemented because implemented in subclasses + :return: None + """ + raise NotImplementedError("The method not implemented") + + def getData(self): + """ + + :return: + """ + # return sigrok_energy_api_result(self.changes, True if self.start == 0xff else False) + return SigrokResult( + [x / self.sample_rate for x in self.changes], + True if self.start == 0xFF else False, + ) + + def analyzeData(self, byte): + """ + analyze one byte if it differs from the last byte, it will be appended to changes. + + :param byte: one byte to analyze + """ + if self.start is None: + self.start = byte + self.last_val = byte + if byte != self.last_val: + self.changes.append(self.index) + self.last_val = byte + self.index += 1 diff --git a/lib/lennart/__init__.py b/lib/lennart/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/loader.py b/lib/loader.py index fcd5490..568c98c 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1638,8 +1638,8 @@ class EnergyTraceWithLogicAnalyzer: self.errors = list() def load_data(self, log_data): - from data.timing.SigrokInterface import SigrokResult - from data.energy.EnergyInterface import EnergyInterface + from dfatool.lennart.SigrokInterface import SigrokResult + from dfatool.lennart.EnergyInterface import EnergyInterface # Daten laden self.sync_data = SigrokResult.fromString(log_data[0]) @@ -1677,7 +1677,7 @@ class EnergyTraceWithLogicAnalyzer: for state_or_transition in trace["trace"]: names.append(state_or_transition["name"]) # print(names[:15]) - from data.processing.DataProcessor import DataProcessor + from dfatool.lennart.DataProcessor import DataProcessor dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) dp.run() @@ -1741,8 +1741,8 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): super().__init__(voltage, state_duration, transition_names, with_traces) def load_data(self, log_data): - from data.timing.SigrokInterface import SigrokResult - from data.energy.EnergyInterface import EnergyInterface + from dfatool.lennart.SigrokInterface import SigrokResult + from dfatool.lennart.EnergyInterface import EnergyInterface # Daten laden self.sync_data = None @@ -1751,7 +1751,7 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): pass def analyze_states(self, traces, offline_index: int): - from data.timing.SigrokInterface import SigrokResult + from dfatool.lennart.SigrokInterface import SigrokResult self.sync_data = SigrokResult.fromTraces(traces) return super().analyze_states(traces, offline_index) diff --git a/lib/runner.py b/lib/runner.py index 96627cf..0f44e97 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -17,7 +17,7 @@ import serial.threaded import subprocess import sys import time -from data.timing.SigrokCLIInterface import SigrokCLIInterface +from dfatool.lennart.SigrokCLIInterface import SigrokCLIInterface class SerialReader(serial.threaded.Protocol): -- cgit v1.2.3 From 611d65e1b3b4556370d7089a4b5bd8be2250bc97 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 16 Oct 2020 15:18:18 +0200 Subject: loader: Do not plot sync=la traces --- lib/loader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index 568c98c..d2604b7 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1683,8 +1683,9 @@ class EnergyTraceWithLogicAnalyzer: dp.run() energy_trace_new = list() energy_trace_new.extend(dp.getStatesdfatool(state_sleep=self.state_duration)) - dp.plot() - # dp.plot(names) + # Uncomment to plot traces + # dp.plot() # <- plot traces with sync annotatons + # dp.plot(names) # <- plot annotated traces (with state/transition names) energy_trace_new = energy_trace_new[4:] energy_trace = list() -- cgit v1.2.3 From 5ed51c2a1e91109477f6c8726e45f1c638c2eb8f Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 19 Oct 2020 10:01:55 +0200 Subject: loader: improve error message when encountering invalid data --- lib/loader.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index d2604b7..e004fe1 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -689,7 +689,18 @@ class RawData: online_datapoints.append((run_idx, trace_part_idx)) for offline_idx, online_ref in enumerate(online_datapoints): online_run_idx, online_trace_part_idx = online_ref - offline_trace_part = measurement["energy_trace"][offline_idx] + try: + offline_trace_part = measurement["energy_trace"][offline_idx] + except IndexError: + logger.error( + f"While handling file #{measurement['fileno']} {measurement['info']}:" + ) + logger.error(f" offline energy_trace data is shorted than online data") + logger.error(f" len(online_datapoints) == {len(online_datapoints)}") + logger.error( + f" len(energy_trace) == {len(measurement['energy_trace'])}" + ) + raise online_trace_part = traces[online_run_idx]["trace"][online_trace_part_idx] if "offline" not in online_trace_part: @@ -1105,7 +1116,7 @@ def _add_trace_data_to_aggregate(aggregate, key, element): def pta_trace_to_aggregate(traces, ignore_trace_indexes=[]): - u""" + """ Convert preprocessed DFA traces from peripherals/drivers to by_name aggregate for PTAModel. arguments: @@ -1311,7 +1322,7 @@ class EnergyTraceWithBarcode: return self._ts_to_index(timestamp, mid_index, right_index) def analyze_states(self, traces, offline_index: int): - u""" + """ Split log data into states and transitions and return duration, energy, and mean power for each element. :param traces: expected traces, needed to synchronize with the measurement. @@ -1648,7 +1659,7 @@ class EnergyTraceWithLogicAnalyzer: pass def analyze_states(self, traces, offline_index: int): - u""" + """ Split log data into states and transitions and return duration, energy, and mean power for each element. :param traces: expected traces, needed to synchronize with the measurement. @@ -1789,7 +1800,7 @@ class MIMOSA: self.errors = list() def charge_to_current_nocal(self, charge): - u""" + """ Convert charge per 10µs (in pJ) to mean currents (in µA) without accounting for calibration. :param charge: numpy array of charges (pJ per 10µs) as returned by `load_data` or `load_file` @@ -1801,7 +1812,7 @@ class MIMOSA: return charge * ua_step def _load_tf(self, tf): - u""" + """ Load MIMOSA log data from an open `tarfile` instance. :param tf: `tarfile` instance @@ -1822,7 +1833,7 @@ class MIMOSA: return charges, triggers def load_data(self, raw_data): - u""" + """ Load MIMOSA log data from a MIMOSA log file passed as raw byte string :param raw_data: MIMOSA log file, passed as raw byte string @@ -1834,7 +1845,7 @@ class MIMOSA: return self._load_tf(tf) def load_file(self, filename): - u""" + """ Load MIMOSA log data from a MIMOSA log file :param filename: MIMOSA log file @@ -1845,7 +1856,7 @@ class MIMOSA: return self._load_tf(tf) def currents_nocal(self, charges): - u""" + """ Convert charges (pJ per 10µs) to mean currents without accounting for calibration. :param charges: numpy array of charges (pJ per 10µs) @@ -1902,7 +1913,7 @@ class MIMOSA: return trigidx def calibration_edges(self, currents): - u""" + """ Return start/stop indexes of calibration measurements. :param currents: uncalibrated currents as reported by MIMOSA. For best results, @@ -1939,7 +1950,7 @@ class MIMOSA: ) def calibration_function(self, charges, cal_edges): - u""" + """ Calculate calibration function from previously determined calibration edges. :param charges: raw charges from MIMOSA @@ -2029,7 +2040,7 @@ class MIMOSA: return calfunc, caldata def analyze_states(self, charges, trigidx, ua_func): - u""" + """ Split log data into states and transitions and return duration, energy, and mean power for each element. :param charges: raw charges (each element describes the charge in pJ transferred during 10 µs) -- cgit v1.2.3 From 675032e99a02a1d1157305eb4e200b373a196f2a Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 19 Oct 2020 11:47:04 +0200 Subject: Sigrok: Continuous measurement; fix sigrok-cli termination --- lib/lennart/SigrokAPIInterface.py | 9 ++------- lib/lennart/SigrokCLIInterface.py | 36 ++++++++++++++++++++++-------------- lib/lennart/SigrokInterface.py | 6 +----- lib/runner.py | 4 +--- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/lib/lennart/SigrokAPIInterface.py b/lib/lennart/SigrokAPIInterface.py index a8765ba..a2c087a 100644 --- a/lib/lennart/SigrokAPIInterface.py +++ b/lib/lennart/SigrokAPIInterface.py @@ -51,7 +51,6 @@ class SigrokAPIInterface(SigrokInterface): self, driver="fx2lafw", sample_rate=100_000, - sample_count=1_000_000, debug_output=False, used_datafeed=datafeed_changes, fake=False, @@ -60,12 +59,11 @@ class SigrokAPIInterface(SigrokInterface): :param driver: Driver that should be used :param sample_rate: The sample rate of the Logic analyzer - :param sample_count: The sample count of the Logic analyzer :param debug_output: Should be true if output should be displayed to user :param used_datafeed: one of the datafeeds above, user later as callback. :param fake: """ - super(SigrokAPIInterface, self).__init__(sample_rate, sample_count) + super(SigrokAPIInterface, self).__init__(sample_rate) if fake: raise NotImplementedError("Not implemented!") self.used_datafeed = used_datafeed @@ -98,7 +96,6 @@ class SigrokAPIInterface(SigrokInterface): ) sigrokDevice.open() - sigrokDevice.config_set(ConfigKey.LIMIT_SAMPLES, self.sample_count) sigrokDevice.config_set(ConfigKey.SAMPLERATE, self.sample_rate) enabled_channels = ["D1"] @@ -124,9 +121,7 @@ class SigrokAPIInterface(SigrokInterface): print( "Used time: ", total_time * 1_000_000, - "µs | sample/s: ", - self.sample_count / (total_time), - "Hz ", + "µs", ) self.session.stop() diff --git a/lib/lennart/SigrokCLIInterface.py b/lib/lennart/SigrokCLIInterface.py index 873b226..d7347ca 100644 --- a/lib/lennart/SigrokCLIInterface.py +++ b/lib/lennart/SigrokCLIInterface.py @@ -9,7 +9,6 @@ class SigrokCLIInterface(SigrokInterface): self, bin_temp_file="temp/out.bin", sample_rate=100_000, - sample_count=1_000_000, fake=False, ): """ @@ -17,10 +16,9 @@ class SigrokCLIInterface(SigrokInterface): :param bin_temp_file: temporary file for binary output :param sample_rate: The sample rate of the Logic analyzer - :param sample_count: The sample count of the Logic analyzer :param fake: if it should use existing data """ - super(SigrokCLIInterface, self).__init__(sample_rate, sample_count) + super(SigrokCLIInterface, self).__init__(sample_rate) self.fake = fake self.bin_temp_file = bin_temp_file self.sigrok_cli_thread = None @@ -30,13 +28,15 @@ class SigrokCLIInterface(SigrokInterface): Force stopping measure, sometimes needs pkill for killing definitly :return: None """ - self.sigrok_cli_thread.send_signal(subprocess.signal.SIGKILL) - stdout, stderr = self.sigrok_cli_thread.communicate(timeout=15) - time.sleep(5) - # TODO not nice solution, make better - import os + self.sigrok_cli_thread.terminate() - os.system("pkill -f sigrok") + try: + self.sigrok_cli_thread.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.warning("sigrok-cli has not stopped. Killing it.") + self.sigrok_cli_thread.kill() + + self.sigrok_cli_thread.communicate() self.runOpenAnalyze() def runMeasure(self): @@ -51,11 +51,19 @@ class SigrokCLIInterface(SigrokInterface): """ starts the measurement, not waiting for done """ - shellcommand = ( - 'sigrok-cli --output-file %s --output-format binary --samples %s -d %s --config "samplerate=%s Hz"' - % (self.bin_temp_file, self.sample_count, self.driver, self.sample_rate) - ) - self.sigrok_cli_thread = subprocess.Popen(shellcommand, shell=True) + shellcommand = [ + "sigrok-cli", + "--output-file", + self.bin_temp_file, + "--output-format", + "binary", + "--continuous", + "-d", + self.driver, + "--config", + f"samplerate={self.sample_rate} Hz", + ] + self.sigrok_cli_thread = subprocess.Popen(shellcommand) def waitForAsynchronousMeasure(self): """ diff --git a/lib/lennart/SigrokInterface.py b/lib/lennart/SigrokInterface.py index 555a25f..a5eaffc 100644 --- a/lib/lennart/SigrokInterface.py +++ b/lib/lennart/SigrokInterface.py @@ -103,18 +103,14 @@ class SigrokResult: class SigrokInterface(DataInterface): - def __init__( - self, sample_rate, sample_count, driver="fx2lafw", filename="temp/sigrok.log" - ): + def __init__(self, sample_rate, driver="fx2lafw", filename="temp/sigrok.log"): """ :param sample_rate: Samplerate of the Logic Analyzer - :param sample_count: Count of samples :param driver: for many Logic Analyzer from Saleae the "fx2lafw" should be working :param filename: temporary file name """ # options - self.sample_count = sample_count self.sample_rate = sample_rate self.file = open(filename, "w+") self.driver = driver diff --git a/lib/runner.py b/lib/runner.py index 0f44e97..72d222d 100644 --- a/lib/runner.py +++ b/lib/runner.py @@ -189,14 +189,12 @@ class EnergyTraceLogicAnalyzerMonitor(EnergyTraceMonitor): def __init__(self, port: str, baud: int, callback=None, voltage=3.3): super().__init__(port=port, baud=baud, callback=callback, voltage=voltage) - # TODO Max length - options = {"length": 90, "fake": False, "sample_rate": 1_000_000} + options = {"fake": False, "sample_rate": 1_000_000} self.log_file = "logic_output_log_%s.json" % (time.strftime("%Y%m%d-%H%M%S")) # Initialization of Interfaces self.sig = SigrokCLIInterface( sample_rate=options["sample_rate"], - sample_count=options["length"] * options["sample_rate"], fake=options["fake"], ) -- cgit v1.2.3 From f308a519edecd8ee92f2fe18552620a569f48d3b Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 19 Oct 2020 11:47:53 +0200 Subject: debug log --- lib/lennart/DataProcessor.py | 17 ++++++++++++++--- lib/lennart/EnergyInterface.py | 3 +++ lib/lennart/SigrokAPIInterface.py | 3 +++ lib/lennart/SigrokCLIInterface.py | 3 +++ lib/lennart/SigrokInterface.py | 3 +++ 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index df3f41c..58cc705 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -1,3 +1,8 @@ +import logging + +logger = logging.getLogger(__name__) + + class DataProcessor: def __init__(self, sync_data, energy_data): """ @@ -84,7 +89,7 @@ class DataProcessor: (sync_start / 1_000_000, pre_outliers_ts / 1_000_000) ) - # print("SYNC SPOTS: ", datasync_timestamps) + logger.debug(f"Synchronization areas: {datasync_timestamps}") # print(time_stamp_data[2]) start_offset = datasync_timestamps[0][1] - time_stamp_data[2] @@ -92,9 +97,15 @@ class DataProcessor: end_offset = datasync_timestamps[-2][0] - (time_stamp_data[-8] + start_offset) end_timestamp = datasync_timestamps[-2][0] - print(start_timestamp, end_timestamp) + logger.debug( + f"Measurement area: LA timestamp range [{start_timestamp}, {end_timestamp}]" + ) + logger.debug(f"Start/End offsets: {start_offset} / {end_offset}") - # print(start_offset, start_timestamp, end_offset, end_timestamp) + if end_offset > 10: + logger.warning( + f"synchronization end_offset == {end_offset}. It should be no more than a few seconds." + ) with_offset = self.addOffset(time_stamp_data, start_offset) diff --git a/lib/lennart/EnergyInterface.py b/lib/lennart/EnergyInterface.py index 2b23667..19aae84 100644 --- a/lib/lennart/EnergyInterface.py +++ b/lib/lennart/EnergyInterface.py @@ -2,6 +2,9 @@ import re import subprocess from dfatool.lennart.DataInterface import DataInterface +import logging + +logger = logging.getLogger(__name__) class EnergyInterface(DataInterface): diff --git a/lib/lennart/SigrokAPIInterface.py b/lib/lennart/SigrokAPIInterface.py index a2c087a..44da678 100644 --- a/lib/lennart/SigrokAPIInterface.py +++ b/lib/lennart/SigrokAPIInterface.py @@ -6,6 +6,9 @@ import sigrok.core as sr from sigrok.core.classes import * from util.ByteHelper import ByteHelper +import logging + +logger = logging.getLogger(__name__) class SigrokAPIInterface(SigrokInterface): diff --git a/lib/lennart/SigrokCLIInterface.py b/lib/lennart/SigrokCLIInterface.py index d7347ca..b28a8a9 100644 --- a/lib/lennart/SigrokCLIInterface.py +++ b/lib/lennart/SigrokCLIInterface.py @@ -2,6 +2,9 @@ import subprocess import time from dfatool.lennart.SigrokInterface import SigrokInterface +import logging + +logger = logging.getLogger(__name__) class SigrokCLIInterface(SigrokInterface): diff --git a/lib/lennart/SigrokInterface.py b/lib/lennart/SigrokInterface.py index a5eaffc..1733b68 100644 --- a/lib/lennart/SigrokInterface.py +++ b/lib/lennart/SigrokInterface.py @@ -1,6 +1,9 @@ import json from dfatool.lennart.DataInterface import DataInterface +import logging + +logger = logging.getLogger(__name__) # Adding additional parsing functionality -- cgit v1.2.3 From db41acafd2abeca153c9f552e82cd26968092cb3 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 19 Oct 2020 14:28:43 +0200 Subject: ET+LA: add --plot-traces support, actually calculate standard deviation. Jungejungejungejungejunge... --- lib/lennart/DataProcessor.py | 27 +++++++++++++++------------ lib/loader.py | 10 ++++++---- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 58cc705..27005b1 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -1,3 +1,4 @@ +import numpy as np import logging logger = logging.getLogger(__name__) @@ -248,13 +249,13 @@ class DataProcessor: def getPowerBetween(self, start, end, state_sleep): # 0.001469 """ - calculates the average powerusage in interval + calculates the powerusage in interval NOT SIDE EFFECT FREE, DON'T USE IT EVERYWHERE :param start: Start timestamp of interval :param end: End timestamp of interval :param state_sleep: Length in seconds of one state, needed for cutting out the UART Sending cycle - :return: float with average power usage + :return: power measurements in W """ first_index = 0 all_power = [] @@ -297,9 +298,9 @@ class DataProcessor: elif len(all_power) == 1: # print("OKAY") pass - return pre_fix_len, sum(all_power) / len(all_power) + return np.array(all_power) - def getStatesdfatool(self, state_sleep, algorithm=False): + def getStatesdfatool(self, state_sleep, with_traces=False, algorithm=False): """ Calculates the length and energy usage of the states @@ -325,14 +326,14 @@ class DataProcessor: start_transition_ts_timing = self.reduced_timestamps[ts_index * 2] if end_transition_ts is not None: - count_dp, power = self.getPowerBetween( + power = self.getPowerBetween( end_transition_ts, start_transition_ts, state_sleep ) # print("STATE", end_transition_ts * 10 ** 6, start_transition_ts * 10 ** 6, (start_transition_ts - end_transition_ts) * 10 ** 6, power) if ( (start_transition_ts - end_transition_ts) * 10 ** 6 > 900_000 - and power > self.power_sync_watt * 0.9 + and np.mean(power) > self.power_sync_watt * 0.9 and ts_index > 10 ): # remove last transition and stop (upcoming data only sync) @@ -342,8 +343,9 @@ class DataProcessor: state = { "isa": "state", - "W_mean": power, - "W_std": 0.0001, + "W_mean": np.mean(power), + "W_std": np.std(power), + "uW": power * 1e6, "s": ( start_transition_ts_timing - end_transition_ts_timing ), # * 10 ** 6, @@ -356,7 +358,7 @@ class DataProcessor: # get energy end_transition_ts end_transition_ts = self.modified_timestamps[ts_index * 2 + 1] - count_dp, power = self.getPowerBetween( + power = self.getPowerBetween( start_transition_ts, end_transition_ts, state_sleep ) @@ -365,12 +367,13 @@ class DataProcessor: transition = { "isa": "transition", - "W_mean": power, - "W_std": 0.0001, + "W_mean": np.mean(power), + "W_std": np.std(power), + "uW": power * 1e6, "s": ( end_transition_ts_timing - start_transition_ts_timing ), # * 10 ** 6, - "count_dp": count_dp, + "count_dp": len(power), } if (end_transition_ts - start_transition_ts) * 10 ** 6 > 2_000_000: diff --git a/lib/loader.py b/lib/loader.py index e004fe1..94a76b3 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1656,8 +1656,6 @@ class EnergyTraceWithLogicAnalyzer: self.sync_data = SigrokResult.fromString(log_data[0]) self.energy_data = EnergyInterface.getDataFromString(str(log_data[1])) - pass - def analyze_states(self, traces, offline_index: int): """ Split log data into states and transitions and return duration, energy, and mean power for each element. @@ -1693,9 +1691,13 @@ class EnergyTraceWithLogicAnalyzer: dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) dp.run() energy_trace_new = list() - energy_trace_new.extend(dp.getStatesdfatool(state_sleep=self.state_duration)) + energy_trace_new.extend( + dp.getStatesdfatool( + state_sleep=self.state_duration, with_traces=self.with_traces + ) + ) # Uncomment to plot traces - # dp.plot() # <- plot traces with sync annotatons + # dp.plot() # <- plot traces with sync annotatons # dp.plot(names) # <- plot annotated traces (with state/transition names) energy_trace_new = energy_trace_new[4:] -- cgit v1.2.3 From 7933d189e7aa6d1dfe857e1fd1bba70147380747 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Tue, 20 Oct 2020 11:39:35 +0200 Subject: DataProcessor: improve drift compensation it still isn't satisfactory --- lib/lennart/DataProcessor.py | 57 +++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 27005b1..44fef0a 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -51,11 +51,12 @@ class DataProcessor: sync_start = 0 outliers = 0 pre_outliers_ts = None + # TODO only consider the first few and the last few seconds for sync points for i, energytrace_dataset in enumerate(self.energy_data): usedtime = energytrace_dataset[0] - last_data[0] # in microseconds timestamp = energytrace_dataset[0] usedenergy = energytrace_dataset[3] - last_data[3] - power = usedenergy / usedtime * 10 ** -3 # in watts + power = usedenergy / usedtime * 1e-3 # in watts if power > 0: if power > self.power_sync_watt: if sync_start is None: @@ -81,7 +82,7 @@ class DataProcessor: last_data = energytrace_dataset - self.plot_data_x.append(energytrace_dataset[0] / 1_000_000) + self.plot_data_x.append(timestamp / 1_000_000) self.plot_data_y.append(power) if power > self.power_sync_watt: @@ -90,8 +91,11 @@ class DataProcessor: (sync_start / 1_000_000, pre_outliers_ts / 1_000_000) ) - logger.debug(f"Synchronization areas: {datasync_timestamps}") - # print(time_stamp_data[2]) + # print(datasync_timestamps) + + # time_stamp_data contains an entry for each level change on the Logic Analyzer input. + # So, time_stamp_data[0] is the first low-to-high transition, time_stamp_data[2] the second, etc. + # -> time_stamp_data[-8] is the low-to-high transition indicating the first after-measurement sync pulse start_offset = datasync_timestamps[0][1] - time_stamp_data[2] start_timestamp = datasync_timestamps[0][1] @@ -99,7 +103,10 @@ class DataProcessor: end_offset = datasync_timestamps[-2][0] - (time_stamp_data[-8] + start_offset) end_timestamp = datasync_timestamps[-2][0] logger.debug( - f"Measurement area: LA timestamp range [{start_timestamp}, {end_timestamp}]" + f"Measurement area: ET timestamp range [{start_timestamp}, {end_timestamp}]" + ) + logger.debug( + f"Measurement area: LA timestamp range [{time_stamp_data[2]}, {time_stamp_data[-8]}]" ) logger.debug(f"Start/End offsets: {start_offset} / {end_offset}") @@ -108,7 +115,7 @@ class DataProcessor: f"synchronization end_offset == {end_offset}. It should be no more than a few seconds." ) - with_offset = self.addOffset(time_stamp_data, start_offset) + with_offset = np.array(time_stamp_data) + start_offset with_drift = self.addDrift( with_offset, end_timestamp, end_offset, start_timestamp @@ -116,20 +123,6 @@ class DataProcessor: self.modified_timestamps = with_drift - def addOffset(self, input_timestamps, start_offset): - """ - Add begin offset at start - - :param input_timestamps: List of timestamps (float list) - :param start_offset: Timestamp of last EnergyTrace datapoint at the first sync point - :return: List of modified timestamps (float list) - """ - modified_timestamps_with_offset = [] - for x in input_timestamps: - if x + start_offset >= 0: - modified_timestamps_with_offset.append(x + start_offset) - return modified_timestamps_with_offset - def removeTooFarDatasets(self, input_timestamps): """ Removing datasets, that are to far away at ethe end @@ -151,20 +144,22 @@ class DataProcessor: Add drift to datapoints :param input_timestamps: List of timestamps (float list) - :param end_timestamp: Timestamp of first EnergyTrace datapoint at the second last sync point + :param end_timestamp: Timestamp of first EnergyTrace datapoint at the second-to-last sync point :param end_offset: the time between end_timestamp and the timestamp of synchronisation signal :param start_timestamp: Timestamp of last EnergyTrace datapoint at the first sync point :return: List of modified timestamps (float list) """ endFactor = (end_timestamp + end_offset - start_timestamp) / ( end_timestamp - start_timestamp - ) - modified_timestamps_with_drift = [] - for x in input_timestamps: - modified_timestamps_with_drift.append( - ((x - start_timestamp) * endFactor) + start_timestamp - ) - + ) + 0.0001 + # print( + # f"({end_timestamp} + {end_offset} - {start_timestamp}) / ({end_timestamp} - {start_timestamp}) == {endFactor}" + # ) + # Manuelles endFactor += 0.0001 macht es merklich besser + # print(f"endFactor = {endFactor}") + modified_timestamps_with_drift = ( + (input_timestamps - start_timestamp) * endFactor + ) + start_timestamp return modified_timestamps_with_drift def plot(self, annotateData=None): @@ -345,11 +340,12 @@ class DataProcessor: "isa": "state", "W_mean": np.mean(power), "W_std": np.std(power), - "uW": power * 1e6, "s": ( start_transition_ts_timing - end_transition_ts_timing ), # * 10 ** 6, } + if with_traces: + state["uW"] = power * 1e6 energy_trace_new.append(state) energy_trace_new[-2]["W_mean_delta_next"] = ( @@ -369,12 +365,13 @@ class DataProcessor: "isa": "transition", "W_mean": np.mean(power), "W_std": np.std(power), - "uW": power * 1e6, "s": ( end_transition_ts_timing - start_transition_ts_timing ), # * 10 ** 6, "count_dp": len(power), } + if with_traces: + transition["uW"] = power * 1e6 if (end_transition_ts - start_transition_ts) * 10 ** 6 > 2_000_000: # TODO Last data set corrupted? HOT FIX!!!!!!!!!!!! REMOVE LATER -- cgit v1.2.3 From 47e6378ad2b5862905a04b8f986404918a3c2a00 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Tue, 20 Oct 2020 14:55:35 +0200 Subject: analyze-archive: note that --plot-traces has a wrong X-axis on !MIMOSA --- bin/analyze-archive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index ca36745..bd1c824 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -223,14 +223,13 @@ def print_html_model_data(model, pm, pq, lm, lq, am, ai, aq): print("") print("") + def plot_traces(preprocessed_data, sot_name): traces = list() for trace in preprocessed_data: for state_or_transition in trace["trace"]: if state_or_transition["name"] == sot_name: - traces.extend( - map(lambda x: x["uW"], state_or_transition["offline"]) - ) + traces.extend(map(lambda x: x["uW"], state_or_transition["offline"])) if len(traces) == 0: print( f"""Did not find traces for state or transition {sot_name}. Abort.""", @@ -250,6 +249,7 @@ def plot_traces(preprocessed_data, sot_name): family=True, ) + if __name__ == "__main__": ignored_trace_indexes = [] @@ -295,7 +295,7 @@ if __name__ == "__main__": "--plot-traces", metavar="NAME", type=str, - help="Plot power trace for state or transition NAME", + help="Plot power trace for state or transition NAME. X axis is wrong for non-MIMOSA measurements", ) parser.add_argument( "--show-models", -- cgit v1.2.3 From 7382103823962305df09b7ed1913597602a175e2 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Tue, 20 Oct 2020 14:58:44 +0200 Subject: DataProcessor: comments --- lib/lennart/DataProcessor.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 44fef0a..8373e75 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -42,7 +42,7 @@ class DataProcessor: last_data = [0, 0, 0, 0] # clean timestamp data, if at the end strange ts got added somehow - time_stamp_data = self.removeTooFarDatasets(time_stamp_data) + # time_stamp_data = self.removeTooFarDatasets(time_stamp_data) self.reduced_timestamps = time_stamp_data @@ -95,13 +95,14 @@ class DataProcessor: # time_stamp_data contains an entry for each level change on the Logic Analyzer input. # So, time_stamp_data[0] is the first low-to-high transition, time_stamp_data[2] the second, etc. - # -> time_stamp_data[-8] is the low-to-high transition indicating the first after-measurement sync pulse + # -> time_stamp_data[2] is the low-to-high transition indicating the end of the first sync pulse + # -> time_stamp_data[-8] is the low-to-high transition indicating the start of the first after-measurement sync pulse - start_offset = datasync_timestamps[0][1] - time_stamp_data[2] start_timestamp = datasync_timestamps[0][1] + start_offset = start_timestamp - time_stamp_data[2] - end_offset = datasync_timestamps[-2][0] - (time_stamp_data[-8] + start_offset) end_timestamp = datasync_timestamps[-2][0] + end_offset = end_timestamp - (time_stamp_data[-8] + start_offset) logger.debug( f"Measurement area: ET timestamp range [{start_timestamp}, {end_timestamp}]" ) @@ -149,17 +150,19 @@ class DataProcessor: :param start_timestamp: Timestamp of last EnergyTrace datapoint at the first sync point :return: List of modified timestamps (float list) """ - endFactor = (end_timestamp + end_offset - start_timestamp) / ( - end_timestamp - start_timestamp - ) + 0.0001 + endFactor = 1 + (end_offset / (end_timestamp - start_timestamp)) # print( - # f"({end_timestamp} + {end_offset} - {start_timestamp}) / ({end_timestamp} - {start_timestamp}) == {endFactor}" + # f"({end_timestamp} + {end_offset} - {start_timestamp}) / ({end_timestamp} - {start_timestamp}) == {endFactor}" # ) # Manuelles endFactor += 0.0001 macht es merklich besser # print(f"endFactor = {endFactor}") + # endFactor assumes that the end of the first sync pulse is at timestamp 0. + # Then, timestamps with drift := timestamps * endFactor. + # As this is not the case (the first sync pulse ends at start_timestamp > 0), we shift the data by first + # removing start_timestamp, then multiplying with endFactor, and then re-adding the start_timestamp. modified_timestamps_with_drift = ( - (input_timestamps - start_timestamp) * endFactor - ) + start_timestamp + input_timestamps - start_timestamp + ) * endFactor + start_timestamp return modified_timestamps_with_drift def plot(self, annotateData=None): -- cgit v1.2.3 From 993a3bae7f5c560d8c9c601c9a6b423e9f507785 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Wed, 21 Oct 2020 12:35:59 +0200 Subject: --plot-traces: use the correct time base for each backend --- bin/analyze-archive.py | 21 +++++++++++++++------ lib/lennart/DataProcessor.py | 15 +++++++++------ lib/loader.py | 32 ++++++++++++++++++++++---------- lib/plotter.py | 6 +++++- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index bd1c824..5d4411b 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -226,10 +226,16 @@ def print_html_model_data(model, pm, pq, lm, lq, am, ai, aq): def plot_traces(preprocessed_data, sot_name): traces = list() + timestamps = list() for trace in preprocessed_data: for state_or_transition in trace["trace"]: if state_or_transition["name"] == sot_name: - traces.extend(map(lambda x: x["uW"], state_or_transition["offline"])) + timestamps.extend( + map(lambda x: x["plot"][0], state_or_transition["offline"]) + ) + traces.extend( + map(lambda x: x["plot"][1], state_or_transition["offline"]) + ) if len(traces) == 0: print( f"""Did not find traces for state or transition {sot_name}. Abort.""", @@ -239,12 +245,15 @@ def plot_traces(preprocessed_data, sot_name): if len(traces) > 40: print(f"""Truncating plot to 40 of {len(traces)} traces (random sample)""") - traces = random.sample(traces, 40) + indexes = random.sample(range(len(traces)), 40) + timestamps = [timestamps[i] for i in indexes] + traces = [traces[i] for i in indexes] - plotter.plot_y( + plotter.plot_xy( + timestamps, traces, - xlabel="t [1e-5 s]", - ylabel="P [uW]", + xlabel="t [s]", + ylabel="P [W]", title=sot_name, family=True, ) @@ -463,7 +472,7 @@ if __name__ == "__main__": if name not in uw_per_sot: uw_per_sot[name] = list() for elem in state_or_transition["offline"]: - elem["uW"] = list(elem["uW"]) + elem["plot"] = list(elem["plot"]) uw_per_sot[name].append(state_or_transition) for name, data in uw_per_sot.items(): target = f"{args.export_traces}/{name}.json" diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 8373e75..7546128 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -256,7 +256,8 @@ class DataProcessor: :return: power measurements in W """ first_index = 0 - all_power = [] + all_power = list() + all_ts = list() for ind in range(self.start_offset, len(self.plot_data_x)): first_index = ind if self.plot_data_x[ind] > start: @@ -272,6 +273,7 @@ class DataProcessor: self.start_offset = ind - 1 break all_power.append(self.plot_data_y[ind]) + all_ts.append(self.plot_data_x[ind]) # TODO Idea remove datapoints that are too far away def removeSD_Mean_Values(arr): @@ -293,10 +295,11 @@ class DataProcessor: if len(all_power) == 0: # print("PROBLEM") all_power.append(self.plot_data_y[nextIndAfterIndex]) + all_ts.append(0) elif len(all_power) == 1: # print("OKAY") pass - return np.array(all_power) + return np.array(all_power), np.array(all_ts) def getStatesdfatool(self, state_sleep, with_traces=False, algorithm=False): """ @@ -324,7 +327,7 @@ class DataProcessor: start_transition_ts_timing = self.reduced_timestamps[ts_index * 2] if end_transition_ts is not None: - power = self.getPowerBetween( + power, timestamps = self.getPowerBetween( end_transition_ts, start_transition_ts, state_sleep ) @@ -348,7 +351,7 @@ class DataProcessor: ), # * 10 ** 6, } if with_traces: - state["uW"] = power * 1e6 + state["plot"] = (timestamps - timestamps[0], power) energy_trace_new.append(state) energy_trace_new[-2]["W_mean_delta_next"] = ( @@ -357,7 +360,7 @@ class DataProcessor: # get energy end_transition_ts end_transition_ts = self.modified_timestamps[ts_index * 2 + 1] - power = self.getPowerBetween( + power, timestamps = self.getPowerBetween( start_transition_ts, end_transition_ts, state_sleep ) @@ -374,7 +377,7 @@ class DataProcessor: "count_dp": len(power), } if with_traces: - transition["uW"] = power * 1e6 + transition["plot"] = (timestamps - timestamps[0], power) if (end_transition_ts - start_transition_ts) * 10 ** 6 > 2_000_000: # TODO Last data set corrupted? HOT FIX!!!!!!!!!!!! REMOVE LATER diff --git a/lib/loader.py b/lib/loader.py index 94a76b3..c981ef7 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1431,7 +1431,13 @@ class EnergyTraceWithBarcode: } if self.with_traces: - transition["uW"] = transition_power_W * 1e6 + timestamps = ( + self.interval_start_timestamp[ + transition_start_index:transition_done_index + ] + - self.interval_start_timestamp[transition_start_index] + ) + transition["plot"] = (timestamps, transition_power_W) energy_trace.append(transition) @@ -1451,7 +1457,11 @@ class EnergyTraceWithBarcode: } if self.with_traces: - state["uW"] = state_power_W * 1e6 + timestamps = ( + self.interval_start_timestamp[state_start_index:state_done_index] + - self.interval_start_timestamp[state_start_index] + ) + state["plot"] = (timestamps, state_power_W) energy_trace.append(state) @@ -1690,15 +1700,14 @@ class EnergyTraceWithLogicAnalyzer: dp = DataProcessor(sync_data=self.sync_data, energy_data=self.energy_data) dp.run() - energy_trace_new = list() - energy_trace_new.extend( - dp.getStatesdfatool( - state_sleep=self.state_duration, with_traces=self.with_traces - ) + energy_trace_new = dp.getStatesdfatool( + state_sleep=self.state_duration, with_traces=self.with_traces ) # Uncomment to plot traces - # dp.plot() # <- plot traces with sync annotatons - # dp.plot(names) # <- plot annotated traces (with state/transition names) + if offline_index == 0: + # dp.plot() # <- plot traces with sync annotatons + # dp.plot(names) # <- plot annotated traces (with state/transition names) + pass energy_trace_new = energy_trace_new[4:] energy_trace = list() @@ -2091,7 +2100,10 @@ class MIMOSA: } if self.with_traces: - data["uW"] = range_ua * self.voltage + data["plot"] = ( + np.arange(len(range_ua)) * 1e-5, + range_ua * self.voltage * 1e-6, + ) if isa == "transition": # subtract average power of previous state diff --git a/lib/plotter.py b/lib/plotter.py index 16c0145..929ceb9 100755 --- a/lib/plotter.py +++ b/lib/plotter.py @@ -136,7 +136,11 @@ def plot_xy(X, Y, xlabel=None, ylabel=None, title=None, output=None, family=Fals if family: cm = plt.get_cmap("brg", len(Y)) for i, YY in enumerate(Y): - plt.plot(np.arange(len(YY)), YY, "-", markersize=2, color=cm(i)) + if X: + XX = X[i] + else: + XX = np.arange(len(YY)) + plt.plot(XX, YY, "-", markersize=2, color=cm(i)) else: plt.plot(X, Y, "bo", markersize=2) if output: -- cgit v1.2.3 From ac81f1a0d9ddcadc7cf5990919a03652e4bc04ee Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Wed, 21 Oct 2020 13:54:40 +0200 Subject: loader: Support new "files" format in MIMOSA files --- lib/loader.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/loader.py b/lib/loader.py index c981ef7..3f6b5ec 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -909,6 +909,12 @@ class RawData: } ) for repeat_id, mim_file in enumerate(ptalog["files"][j]): + # MIMOSA benchmarks always use a single .mim file per benchmark run. + # However, depending on the dfatool version used to run the + # benchmark, ptalog["files"][j] is either "foo.mim" (before Oct 2020) + # or ["foo.mim"] (from Oct 2020 onwards). + if type(mim_file) is list: + mim_file = mim_file[0] member = tf.getmember(mim_file) offline_data.append( { -- cgit v1.2.3 From 9f85fc2f2678c491d1ae8d42995fae0fb931f1fe Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 22 Oct 2020 11:18:05 +0200 Subject: unfuck LA<->ET drift calculation --- lib/lennart/DataProcessor.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 7546128..8d762e7 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -117,10 +117,16 @@ class DataProcessor: ) with_offset = np.array(time_stamp_data) + start_offset + logger.debug( + f"Measurement area with offset: LA timestamp range [{with_offset[2]}, {with_offset[-8]}]" + ) with_drift = self.addDrift( with_offset, end_timestamp, end_offset, start_timestamp ) + logger.debug( + f"Measurement area with drift: LA timestamp range [{with_drift[2]}, {with_drift[-8]}]" + ) self.modified_timestamps = with_drift @@ -150,7 +156,7 @@ class DataProcessor: :param start_timestamp: Timestamp of last EnergyTrace datapoint at the first sync point :return: List of modified timestamps (float list) """ - endFactor = 1 + (end_offset / (end_timestamp - start_timestamp)) + endFactor = 1 + (end_offset / ((end_timestamp - end_offset) - start_timestamp)) # print( # f"({end_timestamp} + {end_offset} - {start_timestamp}) / ({end_timestamp} - {start_timestamp}) == {endFactor}" # ) @@ -216,7 +222,7 @@ class DataProcessor: label="Synchronisationsignale mit Driftfaktor", ) - plt.xlabel("Zeit [s]") + plt.xlabel("Zeit von EnergyTrace [s]") plt.ylabel("Leistung [W]") leg = plt.legend() -- cgit v1.2.3 From b7e25c6ca7746e86ef201b9be7ecec57bf5b2d2a Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 22 Oct 2020 15:05:09 +0200 Subject: Improve sync=la timing restoration. There's still something fishy though... --- bin/generate-dfa-benchmark.py | 1 + lib/harness.py | 23 ++++++++++++++-------- lib/lennart/DataProcessor.py | 5 +++-- lib/lennart/SigrokInterface.py | 43 +----------------------------------------- lib/loader.py | 24 ++++++++++++++++++++++- 5 files changed, 43 insertions(+), 53 deletions(-) diff --git a/bin/generate-dfa-benchmark.py b/bin/generate-dfa-benchmark.py index c8681c5..98c3602 100755 --- a/bin/generate-dfa-benchmark.py +++ b/bin/generate-dfa-benchmark.py @@ -647,6 +647,7 @@ if __name__ == "__main__": log_return_values=need_return_values, repeat=1, energytrace_sync=energytrace_sync, + remove_nop_from_timings=False, # kein einfluss auf ungenauigkeiten ) elif "timing" in opt: harness = OnboardTimerHarness( diff --git a/lib/harness.py b/lib/harness.py index 51013e1..04b14eb 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -355,10 +355,14 @@ class OnboardTimerHarness(TransitionHarness): the dict `offline_aggregates` with the member `duration`. It contains a list of durations (in us) of the corresponding state/transition for each benchmark iteration. I.e. `.traces[*]['trace'][*]['offline_aggregates']['duration'] = [..., ...]` + :param remove_nop_from_timings: If true, remove the nop duration from reported timings + (i.e., reported timings reflect the estimated transition/state duration with the timer call overhea dremoved). + If false, do not remove nop durations, so the timings more accurately reflect the elapsed wall-clock time during the benchmark. """ - def __init__(self, counter_limits, **kwargs): + def __init__(self, counter_limits, remove_nop_from_timings=True, **kwargs): super().__init__(**kwargs) + self.remove_nop_from_timings = remove_nop_from_timings self.trace_length = 0 ( self.one_cycle_in_us, @@ -422,7 +426,6 @@ class OnboardTimerHarness(TransitionHarness): gpio.led_toggle(1); ptalog.stopTransition(); // ======================= LED SYNC ================================ - arch.sleep_ms(250); }\n\n""" return ret @@ -431,14 +434,17 @@ class OnboardTimerHarness(TransitionHarness): if self.energytrace_sync == "led": ret += "runLASync();\n" ret += "ptalog.passNop();\n" + if self.energytrace_sync == "led": + ret += "arch.sleep_ms(250);\n" ret += super().start_benchmark(benchmark_id) return ret def stop_benchmark(self): ret = "" + ret += super().stop_benchmark() if self.energytrace_sync == "led": ret += "runLASync();\n" - ret += super().stop_benchmark() + ret += "arch.sleep_ms(250);\n" return ret def pass_transition( @@ -498,8 +504,9 @@ class OnboardTimerHarness(TransitionHarness): prev_state_duration_us = ( prev_state_cycles * self.one_cycle_in_us + prev_state_overflow * self.one_overflow_in_us - - self.nop_cycles * self.one_cycle_in_us ) + if self.remove_nop_from_timings: + prev_state_duration_us -= self.nop_cycles * self.one_cycle_in_us final_state = self.traces[self.trace_id]["trace"][-1] if "offline_aggregates" not in final_state: final_state["offline_aggregates"] = {"duration": list()} @@ -561,15 +568,15 @@ class OnboardTimerHarness(TransitionHarness): ) ) duration_us = ( - cycles * self.one_cycle_in_us - + overflow * self.one_overflow_in_us - - self.nop_cycles * self.one_cycle_in_us + cycles * self.one_cycle_in_us + overflow * self.one_overflow_in_us ) prev_state_duration_us = ( prev_state_cycles * self.one_cycle_in_us + prev_state_overflow * self.one_overflow_in_us - - self.nop_cycles * self.one_cycle_in_us ) + if self.remove_nop_from_timings: + duration_us -= self.nop_cycles * self.one_cycle_in_us + prev_state_duration_us -= self.nop_cycles * self.one_cycle_in_us if duration_us < 0: duration_us = 0 # self.traces contains transitions and states, UART output only contains transitions -> use index * 2 diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 8d762e7..90cc54d 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -228,9 +228,10 @@ class DataProcessor: def getDataText(x): # print(x) + dl = len(annotateData) for i, xt in enumerate(self.modified_timestamps): - if xt > x: - return "Value: %s" % annotateData[i - 5] + if xt > x and i >= 4 and i - 5 < dl: + return f"SoT: {annotateData[i - 5]}" def update_annot(x, y, name): annot.xy = (x, y) diff --git a/lib/lennart/SigrokInterface.py b/lib/lennart/SigrokInterface.py index 1733b68..32e8fe2 100644 --- a/lib/lennart/SigrokInterface.py +++ b/lib/lennart/SigrokInterface.py @@ -1,4 +1,5 @@ import json +import numpy as np from dfatool.lennart.DataInterface import DataInterface import logging @@ -62,48 +63,6 @@ class SigrokResult: return SigrokResult(data["timestamps"], data["onBeforeFirstChange"]) pass - @classmethod - def fromTraces(cls, traces): - """ - Generates SigrokResult from ptalog.json traces - - :param traces: traces from dfatool ptalog.json - :return: SigrokResult object - """ - timestamps = [0] - for tr in traces: - for t in tr["trace"]: - # print(t['online_aggregates']['duration'][0]) - timestamps.append( - timestamps[-1] + (t["online_aggregates"]["duration"][0] * 10 ** -6) - ) - - # print(timestamps) - # prepend FAKE Sync point - t_neu = [0.0, 0.0000001, 1.0, 1.00000001] - for i, x in enumerate(timestamps): - t_neu.append( - round(float(x) + t_neu[3] + 0.20, 6) - ) # list(map(float, t_ist.split(",")[:i+1])) - - # append FAKE Sync point / eine überschneidung - # [30.403632, 30.403639, 31.407265, 31.407271] - # appendData = [29.144855,30.148495,30.148502,30.403632,30.403639,31.407265,31.407271,] - appendData = [0, 1.000001, 1.000002, 1.25, 1.2500001] - - # TODO future work here, why does the sync not work completely - t_neu[-1] = ( - t_neu[-2] + (t_neu[-1] - t_neu[-2]) * 0.9 - ) # Weird offset failure with UART stuff - - offset = t_neu[-1] - appendData[0] - for x in appendData: - t_neu.append(x + offset) - - # print(t_neu) - print(len(t_neu)) - return SigrokResult(t_neu, False) - class SigrokInterface(DataInterface): def __init__(self, sample_rate, driver="fx2lafw", filename="temp/sigrok.log"): diff --git a/lib/loader.py b/lib/loader.py index 3f6b5ec..2f8e603 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1780,9 +1780,31 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): pass def analyze_states(self, traces, offline_index: int): + + # Start "Synchronization pulse" + timestamps = [0, 10, 1e6, 1e6 + 10] + + # 250ms zwischen Ende der LASync und Beginn der Messungen + # (wegen sleep(250) in der generierten multipass-runLASync-Funktion) + timestamps.append(timestamps[-1] + 240e3) + for tr in traces: + for t in tr["trace"]: + # print(t['online_aggregates']['duration'][0]) + timestamps.append( + timestamps[-1] + t["online_aggregates"]["duration"][offline_index] + ) + + print(timestamps) + + # Stop "Synchronization pulses". The first one has already started. + timestamps.extend(np.array([10, 1e6, 1e6 + 10]) + timestamps[-1]) + timestamps.extend(np.array([0, 10, 1e6, 1e6 + 10]) + 250e3 + timestamps[-1]) + + timestamps = list(np.array(timestamps) * 1e-6) + from dfatool.lennart.SigrokInterface import SigrokResult - self.sync_data = SigrokResult.fromTraces(traces) + self.sync_data = SigrokResult(timestamps, False) return super().analyze_states(traces, offline_index) -- cgit v1.2.3 From 670e7f51f81b7052e17c453991f39018a5c9b008 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 22 Oct 2020 16:17:07 +0200 Subject: EnergyTraceWithTimer: turns out multipass was the issue. --- lib/loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/loader.py b/lib/loader.py index 2f8e603..71183c5 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1786,7 +1786,9 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): # 250ms zwischen Ende der LASync und Beginn der Messungen # (wegen sleep(250) in der generierten multipass-runLASync-Funktion) - timestamps.append(timestamps[-1] + 240e3) + # TODO die prevcycles der allerersten Transition werden vom Tooling nicht + # ausgewertet / weitergereicht, genau diese müssten hier hin. + timestamps.append(timestamps[-1] + 250e3) for tr in traces: for t in tr["trace"]: # print(t['online_aggregates']['duration'][0]) -- cgit v1.2.3 From 34b1b9466af7501e651cf3664d96e12c8126f116 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 23 Oct 2020 08:00:30 +0200 Subject: minor refactoring --- lib/lennart/DataProcessor.py | 2 +- lib/loader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 90cc54d..07d2a2b 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -188,10 +188,10 @@ class DataProcessor: data.append(ts) a = np.empty((len(data),)) + a[0::4] = min_value a[1::4] = max_value a[2::4] = max_value a[3::4] = min_value - a[4::4] = min_value return data, a # plotting by columns import matplotlib.pyplot as plt diff --git a/lib/loader.py b/lib/loader.py index 71183c5..0e46755 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -578,7 +578,7 @@ class RawData: # TODO es gibt next_transitions ohne 'plan' return True - def _merge_online_and_offline(self, measurement): + def _merge_online_and_mimosa(self, measurement): # Edits self.traces_by_fileno[measurement['fileno']][*]['trace'][*]['offline'] # and self.traces_by_fileno[measurement['fileno']][*]['trace'][*]['offline_aggregates'] in place # (appends data from measurement['energy_trace']) @@ -1050,7 +1050,7 @@ class RawData: if version == 0 or version == 1: if self._measurement_is_valid_01(measurement): - self._merge_online_and_offline(measurement) + self._merge_online_and_mimosa(measurement) num_valid += 1 else: logger.warning( -- cgit v1.2.3 From 0d387ea41e64afbb80a4ec5ac0f49327d07bd6d9 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 23 Oct 2020 12:33:59 +0200 Subject: ET+Timer: handle start offset between sync pulse and first trace looking pretty good now! --- lib/harness.py | 13 +++++++++++++ lib/loader.py | 11 ++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index 04b14eb..3074d25 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -384,6 +384,15 @@ class OnboardTimerHarness(TransitionHarness): new_harness.trace_id = self.trace_id return new_harness + def reset(self): + super().reset() + self.trace_length = 0 + + def set_trace_start_offset(self, start_offset): + if not "start_offset" in self.traces[0]: + self.traces[0]["start_offset"] = list() + self.traces[0]["start_offset"].append(start_offset) + def undo(self, undo_from): """ Undo all benchmark runs starting with index `undo_from`. @@ -402,6 +411,8 @@ class OnboardTimerHarness(TransitionHarness): ] = state_or_transition["offline_aggregates"]["duration"][ :undo_from ] + if "start_offset" in trace: + trace["start_offset"] = trace["start_offset"][:undo_from] def global_code(self): ret = "#define PTALOG_TIMING\n" @@ -591,6 +602,8 @@ class OnboardTimerHarness(TransitionHarness): elif self.current_transition_in_trace == 0 and self.trace_id > 0: prev_state_data = self.traces[self.trace_id - 1]["trace"][-1] else: + if self.current_transition_in_trace == 0 and self.trace_id == 0: + self.set_trace_start_offset(prev_state_duration_us) prev_state_data = None except IndexError: transition_name = None diff --git a/lib/loader.py b/lib/loader.py index 0e46755..649c032 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1784,19 +1784,16 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): # Start "Synchronization pulse" timestamps = [0, 10, 1e6, 1e6 + 10] - # 250ms zwischen Ende der LASync und Beginn der Messungen - # (wegen sleep(250) in der generierten multipass-runLASync-Funktion) - # TODO die prevcycles der allerersten Transition werden vom Tooling nicht - # ausgewertet / weitergereicht, genau diese müssten hier hin. - timestamps.append(timestamps[-1] + 250e3) + # The first trace doesn't start immediately, append offset saved by OnboarTimerHarness + timestamps.append(timestamps[-1] + traces[0]["start_offset"][offline_index]) for tr in traces: for t in tr["trace"]: - # print(t['online_aggregates']['duration'][0]) + # print(t["online_aggregates"]["duration"][offline_index]) timestamps.append( timestamps[-1] + t["online_aggregates"]["duration"][offline_index] ) - print(timestamps) + # print(timestamps) # Stop "Synchronization pulses". The first one has already started. timestamps.extend(np.array([10, 1e6, 1e6 + 10]) + timestamps[-1]) -- cgit v1.2.3 From 1809c309a7cbc2025161e805009d19d702749d2e Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 23 Oct 2020 14:23:52 +0200 Subject: sync=timer: fix end of measurement offset error --- lib/harness.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index 3074d25..92fb9b4 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -421,21 +421,19 @@ class OnboardTimerHarness(TransitionHarness): # TODO Make nicer ret += """\nvoid runLASync(){ // ======================= LED SYNC ================================ - ptalog.passTransition(0); - ptalog.startTransition(); - gpio.led_toggle(0); - gpio.led_toggle(1); - ptalog.stopTransition(); + gpio.write(PTALOG_GPIO, 1); + gpio.led_on(0); + gpio.led_on(1); + gpio.write(PTALOG_GPIO, 0); for (unsigned char i = 0; i < 4; i++) { arch.sleep_ms(250); } - ptalog.passTransition(0); - ptalog.startTransition(); - gpio.led_toggle(0); - gpio.led_toggle(1); - ptalog.stopTransition(); + gpio.write(PTALOG_GPIO, 1); + gpio.led_off(0); + gpio.led_off(1); + gpio.write(PTALOG_GPIO, 0); // ======================= LED SYNC ================================ }\n\n""" return ret @@ -452,9 +450,11 @@ class OnboardTimerHarness(TransitionHarness): def stop_benchmark(self): ret = "" - ret += super().stop_benchmark() if self.energytrace_sync == "led": + ret += "counter.stop();\n" ret += "runLASync();\n" + ret += super().stop_benchmark() + if self.energytrace_sync == "led": ret += "arch.sleep_ms(250);\n" return ret -- cgit v1.2.3 From 9868f9c5f30b05f2f4a37c6f083e29f3c7df5992 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 23 Oct 2020 14:24:13 +0200 Subject: assorted minor improvements --- lib/harness.py | 7 ++++--- lib/loader.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index 92fb9b4..de70ace 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -287,8 +287,8 @@ class TransitionHarness: if transition.name != log_data_target["name"]: self.abort = True raise RuntimeError( - "Log mismatch: Expected transition {:s}, got transition {:s} -- may have been caused by preceding malformed UART output".format( - log_data_target["name"], transition.name + "Log mismatch: Expected transition {:s}, got transition {:s}\nMay have been caused by preceding malformed UART output\nOffending line: {:s}".format( + log_data_target["name"], transition.name, line ) ) if self.log_return_values and len(transition.return_value_handlers): @@ -373,6 +373,7 @@ class OnboardTimerHarness(TransitionHarness): def copy(self): new_harness = __class__( (self.one_cycle_in_us, self.one_overflow_in_us, self.counter_max_overflow), + remove_nop_from_timings=self.remove_nop_from_timings, gpio_pin=self.gpio_pin, gpio_mode=self.gpio_mode, pta=self.pta, @@ -647,7 +648,7 @@ class OnboardTimerHarness(TransitionHarness): if transition.name != log_data_target["name"]: self.abort = True raise RuntimeError( - "Log mismatch in benchmark id={:d} trace={:d}: transition #{:d} (ID {:d}): Expected transition {:s}, got transition {:s} -- may have been caused by preceding maformed UART output".format( + "Log mismatch in benchmark id={:d} trace={:d}: transition #{:d} (ID {:d}): Expected transition {:s}, got transition {:s}\nMay have been caused by preceding maformed UART output\nOffending line: {:s}".format( 0, self.trace_id, self.current_transition_in_trace, diff --git a/lib/loader.py b/lib/loader.py index 649c032..a319c94 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1004,7 +1004,7 @@ class RawData: "info": members[0], "setup": self.setup_by_fileno[j], "repeat_id": repeat_id, - "expected_trace": ptalog["traces"][j], + "expected_trace": traces, "with_traces": self.with_traces, "transition_names": list( map( -- cgit v1.2.3 From 8494b500712d06be15b53aebaed730c88e5c2b20 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 26 Oct 2020 10:26:07 +0100 Subject: lint_python: Exclude external code --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0b6c96..d1eae2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ lint_python: script: - apt-get update -qy - apt-get install -y black - - black --check --diff bin + - black --check --diff bin/*.py lib/*.py rules: - if: '$CI_COMMIT_BRANCH == "master"' -- cgit v1.2.3 From 506a49fd49e028055378af2e35d6022e6bb99b9c Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 26 Oct 2020 10:35:50 +0100 Subject: blacken code; do not run lint check on external Python libraries --- .gitlab-ci.yml | 2 +- lib/functions.py | 2 +- lib/parameters.py | 7 ++++++- lib/protocol_benchmarks.py | 30 ++++++++++++++++++++---------- lib/sly/docparse.py | 4 ++-- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1eae2c..498f371 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ lint_python: script: - apt-get update -qy - apt-get install -y black - - black --check --diff bin/*.py lib/*.py + - find bin lib -type f -name '*.py' | xargs black --check --diff rules: - if: '$CI_COMMIT_BRANCH == "master"' diff --git a/lib/functions.py b/lib/functions.py index 94b1aaf..9d799c7 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -394,7 +394,7 @@ class analytic: :param safe_functions_enabled: Include "safe" variants of functions with limited argument range, e.g. a safe inverse which returns 1 when dividing by 0. - + Returns a dict of functions which are typical for energy/timing behaviour of embedded hardware, e.g. linear, exponential or inverse dependency on a configuration setting/runtime variable. diff --git a/lib/parameters.py b/lib/parameters.py index 5c6b978..fa966a3 100644 --- a/lib/parameters.py +++ b/lib/parameters.py @@ -434,7 +434,12 @@ class ParamStats: """ def __init__( - self, by_name, by_param, parameter_names, arg_count, use_corrcoef=False, + self, + by_name, + by_param, + parameter_names, + arg_count, + use_corrcoef=False, ): """ Compute standard deviation and correlation coefficient on parameterized data partitions. diff --git a/lib/protocol_benchmarks.py b/lib/protocol_benchmarks.py index d41979f..7f3e2f2 100755 --- a/lib/protocol_benchmarks.py +++ b/lib/protocol_benchmarks.py @@ -328,8 +328,10 @@ class ArduinoJSON(DummyProtocol): child = enc_node + "l" while child in self.children: child += "_" - self.enc_buf += "ArduinoJson::JsonArray& {} = {}.createNestedArray();\n".format( - child, enc_node + self.enc_buf += ( + "ArduinoJson::JsonArray& {} = {}.createNestedArray();\n".format( + child, enc_node + ) ) self.children.add(child) self.from_json(value, child) @@ -338,8 +340,10 @@ class ArduinoJSON(DummyProtocol): child = enc_node + "o" while child in self.children: child += "_" - self.enc_buf += "ArduinoJson::JsonObject& {} = {}.createNestedObject();\n".format( - child, enc_node + self.enc_buf += ( + "ArduinoJson::JsonObject& {} = {}.createNestedObject();\n".format( + child, enc_node + ) ) self.children.add(child) self.from_json(value, child) @@ -616,11 +620,15 @@ class CapnProtoC(DummyProtocol): [len(value)], ) for i, elem in enumerate(value): - self.enc_buf += "capn_set{:d}({}.{}, {:d}, capn_from_f{:d}({:f}));\n".format( - self.float_bits, self.name, key, i, self.float_bits, elem + self.enc_buf += ( + "capn_set{:d}({}.{}, {:d}, capn_from_f{:d}({:f}));\n".format( + self.float_bits, self.name, key, i, self.float_bits, elem + ) ) - self.dec_buf += "kout << capn_to_f{:d}(capn_get{:d}({}.{}, {:d}));\n".format( - self.float_bits, self.float_bits, self.name, key, i + self.dec_buf += ( + "kout << capn_to_f{:d}(capn_get{:d}({}.{}, {:d}));\n".format( + self.float_bits, self.float_bits, self.name, key, i + ) ) self.assign_and_kout( self.float_type, @@ -1196,8 +1204,10 @@ class NanoPB(DummyProtocol): self.cc_encoders += ( "if (!pb_encode_tag_for_field(stream, field)) return false;\n" ) - self.cc_encoders += 'return pb_encode_string(stream, (uint8_t*)"{}", {:d});\n'.format( - value, len(value) + self.cc_encoders += ( + 'return pb_encode_string(stream, (uint8_t*)"{}", {:d});\n'.format( + value, len(value) + ) ) self.cc_encoders += "}\n" self.enc_buf += "msg.{}{}.funcs.encode = encode_{};\n".format( diff --git a/lib/sly/docparse.py b/lib/sly/docparse.py index 0f35c97..6a60eaf 100644 --- a/lib/sly/docparse.py +++ b/lib/sly/docparse.py @@ -9,7 +9,7 @@ class DocParseMeta(type): ''' Metaclass that processes the class docstring through a parser and incorporates the result into the resulting class definition. This - allows Python classes to be defined with alternative syntax. + allows Python classes to be defined with alternative syntax. To use this class, you first need to define a lexer and parser: from sly import Lexer, Parser @@ -39,7 +39,7 @@ class DocParseMeta(type): ... """ - It is expected that the MyParser() class would return a dictionary. + It is expected that the MyParser() class would return a dictionary. This dictionary is used to create the final class Spam in this example. ''' -- cgit v1.2.3 From 48e34b2dbde909cad61f618b2ea8a2ea286012af Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 29 Oct 2020 14:01:21 +0100 Subject: EnergyTraceWithTimer: Gracefully handle errors in a single measurement run --- lib/loader.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index a319c94..8aecbe0 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1789,9 +1789,16 @@ class EnergyTraceWithTimer(EnergyTraceWithLogicAnalyzer): for tr in traces: for t in tr["trace"]: # print(t["online_aggregates"]["duration"][offline_index]) - timestamps.append( - timestamps[-1] + t["online_aggregates"]["duration"][offline_index] - ) + try: + timestamps.append( + timestamps[-1] + + t["online_aggregates"]["duration"][offline_index] + ) + except IndexError: + self.errors.append( + f"""offline_index {offline_index} missing in trace {tr["id"]}""" + ) + return list() # print(timestamps) -- cgit v1.2.3 From 395cec8d03d150f31363be7ef2ec800f6775bff9 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 29 Oct 2020 14:01:42 +0100 Subject: Set DFATOOL_PLOT_LASYNC to plot ET+LA/ET+Timer sync data --- lib/loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/loader.py b/lib/loader.py index 8aecbe0..0e1e8c9 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -1710,8 +1710,8 @@ class EnergyTraceWithLogicAnalyzer: state_sleep=self.state_duration, with_traces=self.with_traces ) # Uncomment to plot traces - if offline_index == 0: - # dp.plot() # <- plot traces with sync annotatons + if offline_index == 0 and os.getenv("DFATOOL_PLOT_LASYNC") is not None: + dp.plot() # <- plot traces with sync annotatons # dp.plot(names) # <- plot annotated traces (with state/transition names) pass energy_trace_new = energy_trace_new[4:] -- cgit v1.2.3 From c2858f6c1070d7baa87d2f433746adec5042531e Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 29 Oct 2020 14:43:45 +0100 Subject: Gitlab CI: Remove Python Black check, as its behaviour differs between versions --- .gitlab-ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 498f371..f524a29 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,15 +3,6 @@ image: debian:bullseye stages: - test -lint_python: - stage: test - script: - - apt-get update -qy - - apt-get install -y black - - find bin lib -type f -name '*.py' | xargs black --check --diff - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - run_tests: stage: test script: -- cgit v1.2.3 From df04f15d9132ec6b2781edfccc5ad8d33dd3cdd9 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 30 Oct 2020 11:15:45 +0100 Subject: Add DFATOOL_EXPORT_LASYNC variable for ET+LA / ET+Timer sync eval --- lib/lennart/DataProcessor.py | 17 +++++++++++++++++ lib/loader.py | 7 ++++++- lib/utils.py | 13 +++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/lennart/DataProcessor.py b/lib/lennart/DataProcessor.py index 07d2a2b..b46315a 100644 --- a/lib/lennart/DataProcessor.py +++ b/lib/lennart/DataProcessor.py @@ -171,6 +171,23 @@ class DataProcessor: ) * endFactor + start_timestamp return modified_timestamps_with_drift + def export_sync(self): + # [1st trans start, 1st trans stop, 2nd trans start, 2nd trans stop, ...] + sync_timestamps = list() + + for i in range(4, len(self.modified_timestamps) - 8, 2): + sync_timestamps.append( + (self.modified_timestamps[i], self.modified_timestamps[i + 1]) + ) + + # EnergyTrace timestamps + timestamps = self.plot_data_x + + # EnergyTrace power values + power = self.plot_data_y + + return {"sync": sync_timestamps, "timestamps": timestamps, "power": power} + def plot(self, annotateData=None): """ Plots the power usage and the timestamps by logic analyzer diff --git a/lib/loader.py b/lib/loader.py index 0e1e8c9..0c7ad91 100644 --- a/lib/loader.py +++ b/lib/loader.py @@ -12,7 +12,7 @@ import tarfile import hashlib from multiprocessing import Pool -from .utils import running_mean, soft_cast_int +from .utils import NpEncoder, running_mean, soft_cast_int logger = logging.getLogger(__name__) @@ -1714,6 +1714,11 @@ class EnergyTraceWithLogicAnalyzer: dp.plot() # <- plot traces with sync annotatons # dp.plot(names) # <- plot annotated traces (with state/transition names) pass + if os.getenv("DFATOOL_EXPORT_LASYNC") is not None: + filename = os.getenv("DFATOOL_EXPORT_LASYNC") + f"_{offline_index}.json" + with open(filename, "w") as f: + json.dump(dp.export_sync(), f, cls=NpEncoder) + logger.info("Exported data and LA sync timestamps to {filename}") energy_trace_new = energy_trace_new[4:] energy_trace = list() diff --git a/lib/utils.py b/lib/utils.py index d28ecda..adcb534 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -1,3 +1,4 @@ +import json import numpy as np import re import logging @@ -6,6 +7,18 @@ arg_support_enabled = True logger = logging.getLogger(__name__) +class NpEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, np.integer): + return int(obj) + elif isinstance(obj, np.floating): + return float(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + else: + return super(NpEncoder, self).default(obj) + + def running_mean(x: np.ndarray, N: int) -> np.ndarray: """ Compute `N` elements wide running average over `x`. -- cgit v1.2.3 From dd33d9b36dd071d04ccba5a000e9562c2b6a4a31 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 2 Nov 2020 10:10:43 +0100 Subject: harness: redo benchmark when running out of sync --- lib/harness.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/harness.py b/lib/harness.py index de70ace..d1440db 100644 --- a/lib/harness.py +++ b/lib/harness.py @@ -264,17 +264,17 @@ class TransitionHarness: transition_name = None if self.pta: transition_name = self.pta.transitions[transition_id].name - print( - "[HARNESS] benchmark id={:d} trace={:d}: transition #{:d} (ID {:d}, name {}) is out of bounds".format( + self.abort = True + raise RuntimeError( + "Benchmark id={:d} trace={:d}: transition #{:d} (ID {:d}, name {}) is out of bounds. Offending line: {}".format( 0, self.trace_id, self.current_transition_in_trace, transition_id, transition_name, + line, ) ) - print(" Offending line: {}".format(line)) - return if log_data_target["isa"] != "transition": self.abort = True raise RuntimeError( -- cgit v1.2.3