diff options
Diffstat (limited to 'lib/loader')
| -rw-r--r-- | lib/loader/__init__.py | 3 | ||||
| -rw-r--r-- | lib/loader/dlog.py | 293 | ||||
| -rw-r--r-- | lib/loader/keysight.py | 291 |
3 files changed, 290 insertions, 297 deletions
diff --git a/lib/loader/__init__.py b/lib/loader/__init__.py index 4ac91b8..141c7ca 100644 --- a/lib/loader/__init__.py +++ b/lib/loader/__init__.py @@ -19,8 +19,7 @@ from .energytrace import ( EnergyTraceWithLogicAnalyzer, EnergyTraceWithTimer, ) -from .dlog import DLog -from .keysight import KeysightCSV +from .keysight import DLog, KeysightCSV from .mimosa import MIMOSA logger = logging.getLogger(__name__) diff --git a/lib/loader/dlog.py b/lib/loader/dlog.py deleted file mode 100644 index b87a6bb..0000000 --- a/lib/loader/dlog.py +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env python3 - -import io -import numpy as np -import struct -import xml.etree.ElementTree as ET - -from bisect import bisect_right - - -class DLogChannel: - def __init__(self, desc_tuple): - self.slot = desc_tuple[0] - self.smu = desc_tuple[1] - self.unit = desc_tuple[2] - self.data = None - - def __repr__(self): - return f"""<DLogChannel(slot={self.slot}, smu="{self.smu}", unit="{self.unit}", data={self.data})>""" - - -class DLog: - def __init__( - self, - voltage: float, - state_duration: int, - with_traces=False, - skip_duration=None, - limit_duration=None, - ): - self.voltage = voltage - self.state_duration = state_duration - self.with_traces = with_traces - self.skip_duration = skip_duration - self.limit_duration = limit_duration - self.errors = list() - - self.sync_min_duration = 0.7 - # TODO auto-detect - self.sync_power = 10e-3 - - def load_data(self, content): - lines = [] - line = "" - with io.BytesIO(content) as f: - while line != "</dlog>\n": - line = f.readline().decode() - lines.append(line) - xml_header = "".join(lines) - raw_header = f.read(8) - data_offset = f.tell() - raw_data = f.read() - - xml_header = xml_header.replace("1ua>", "X1ua>") - xml_header = xml_header.replace("2ua>", "X2ua>") - dlog = ET.fromstring(xml_header) - channels = [] - - for channel in dlog.findall("channel"): - channel_id = int(channel.get("id")) - sense_curr = channel.find("sense_curr").text - sense_volt = channel.find("sense_volt").text - model = channel.find("ident").find("model").text - if sense_volt == "1": - channels.append((channel_id, model, "V")) - if sense_curr == "1": - channels.append((channel_id, model, "A")) - - num_channels = len(channels) - - self.channels = list(map(DLogChannel, channels)) - self.interval = float(dlog.find("frame").find("tint").text) - self.sense_minmax = int(dlog.find("frame").find("sense_minmax").text) - self.planned_duration = int(dlog.find("frame").find("time").text) - self.observed_duration = self.interval * int(len(raw_data) / (4 * num_channels)) - - if self.sense_minmax: - raise RuntimeError( - "DLog files with 'Log Min/Max' enabled are not supported yet" - ) - - self.timestamps = np.linspace( - 0, self.observed_duration, num=int(len(raw_data) / (4 * num_channels)) - ) - - if ( - self.skip_duration is not None - and self.observed_duration >= self.skip_duration - ): - start_offset = 0 - for i, ts in enumerate(self.timestamps): - if ts >= self.skip_duration: - start_offset = i - break - self.timestamps = self.timestamps[start_offset:] - raw_data = raw_data[start_offset * 4 * num_channels :] - - if ( - self.limit_duration is not None - and self.observed_duration > self.limit_duration - ): - stop_offset = len(self.timestamps) - 1 - for i, ts in enumerate(self.timestamps): - if ts > self.limit_duration: - stop_offset = i - break - self.timestamps = self.timestamps[:stop_offset] - self.observed_duration = self.timestamps[-1] - raw_data = raw_data[: stop_offset * 4 * num_channels] - - self.data = np.ndarray( - shape=(num_channels, int(len(raw_data) / (4 * num_channels))), - dtype=np.float32, - ) - - iterator = struct.iter_unpack(">f", raw_data) - channel_offset = 0 - measurement_offset = 0 - for value in iterator: - if value[0] < -1e6 or value[0] > 1e6: - print( - f"Invalid data value {value[0]} at channel {channel_offset}, measurement {measurement_offset}. Replacing with 0." - ) - self.data[channel_offset, measurement_offset] = 0 - else: - self.data[channel_offset, measurement_offset] = value[0] - if channel_offset + 1 == num_channels: - channel_offset = 0 - measurement_offset += 1 - else: - channel_offset += 1 - - # An SMU has four slots - self.slots = [dict(), dict(), dict(), dict()] - - for i, channel in enumerate(self.channels): - channel.data = self.data[i] - self.slots[channel.slot - 1][channel.unit] = channel - - assert "A" in self.slots[0] - self.data = self.slots[0]["A"].data - - def observed_duration_equals_expectation(self): - return int(self.observed_duration) == self.planned_duration - - # very similar to DataProcessor.getStatesdfatool - def analyze_states(self, expected_trace, repeat_id): - sync_start = None - sync_timestamps = list() - above_count = 0 - below_count = 0 - for i, timestamp in enumerate(self.timestamps): - power = self.voltage * self.data[i] - if power > self.sync_power: - above_count += 1 - below_count = 0 - else: - above_count = 0 - below_count += 1 - - if above_count > 2 and sync_start is None: - sync_start = timestamp - elif below_count > 2 and sync_start is not None: - if timestamp - sync_start > self.sync_min_duration: - sync_end = timestamp - sync_timestamps.append((sync_start, sync_end)) - sync_start = None - print(sync_timestamps) - - if len(sync_timestamps) != 3: - self.errors.append( - f"Found {len(sync_timestamps)} synchronization pulses, expected three." - ) - self.errors.append(f"Synchronization pulses == {sync_timestamps}") - return list() - - start_ts = sync_timestamps[0][1] - end_ts = sync_timestamps[1][0] - - # start and end of first state - online_timestamps = [0, expected_trace[0]["start_offset"][repeat_id]] - - # remaining events from the end of the first transition (start of second state) to the end of the last observed state - for trace in expected_trace: - for word in trace["trace"]: - online_timestamps.append( - online_timestamps[-1] - + word["online_aggregates"]["duration"][repeat_id] - ) - - online_timestamps = np.array(online_timestamps) * 1e-6 - online_timestamps = ( - online_timestamps * ((end_ts - start_ts) / online_timestamps[-1]) + start_ts - ) - - trigger_edges = list() - for ts in online_timestamps: - trigger_edges.append(bisect_right(self.timestamps, ts)) - - energy_trace = list() - - for i in range(2, len(online_timestamps), 2): - prev_state_start_index = trigger_edges[i - 2] - prev_state_stop_index = trigger_edges[i - 1] - transition_start_index = trigger_edges[i - 1] - transition_stop_index = trigger_edges[i] - state_start_index = trigger_edges[i] - state_stop_index = trigger_edges[i + 1] - - # If a transition takes less time than the measurement interval, its start and stop index may be the same. - # In this case, self.data[transition_start_index] is the only data point affected by the transition. - # We use the self.data slice [transition_start_index, transition_stop_index) to determine the mean power, so we need - # to increment transition_stop_index by 1 to end at self.data[transition_start_index] - # (self.data[transition_start_index : transition_start_index+1 ] == [self.data[transition_start_index]) - if transition_stop_index == transition_start_index: - transition_stop_index += 1 - - prev_state_duration = online_timestamps[i + 1] - online_timestamps[i] - transition_duration = online_timestamps[i] - online_timestamps[i - 1] - state_duration = online_timestamps[i + 1] - online_timestamps[i] - - # some states are followed by a UART dump of log data. This causes an increase in CPU energy - # consumption and is not part of the peripheral behaviour, so it should not be part of the benchmark results. - # If a case is followed by a UART dump, its duration is longer than the sleep duration between two transitions. - # In this case, we re-calculate the stop index, and calculate the state duration from coarse energytrace data - # instead of high-precision sync data - if ( - self.timestamps[prev_state_stop_index] - - self.timestamps[prev_state_start_index] - > self.state_duration - ): - prev_state_stop_index = bisect_right( - self.timestamps, - self.timestamps[prev_state_start_index] + self.state_duration, - ) - prev_state_duration = ( - self.timestamps[prev_state_stop_index] - - self.timestamps[prev_state_start_index] - ) - - if ( - self.timestamps[state_stop_index] - self.timestamps[state_start_index] - > self.state_duration - ): - state_stop_index = bisect_right( - self.timestamps, - self.timestamps[state_start_index] + self.state_duration, - ) - state_duration = ( - self.timestamps[state_stop_index] - - self.timestamps[state_start_index] - ) - - prev_state_power = self.data[prev_state_start_index:prev_state_stop_index] - - transition_timestamps = self.timestamps[ - transition_start_index:transition_stop_index - ] - transition_power = self.data[transition_start_index:transition_stop_index] - - state_timestamps = self.timestamps[state_start_index:state_stop_index] - state_power = self.data[state_start_index:state_stop_index] - - transition = { - "isa": "transition", - "W_mean": np.mean(transition_power), - "W_std": np.std(transition_power), - "s": transition_duration, - } - if self.with_traces: - transition["plot"] = ( - transition_timestamps - transition_timestamps[0], - transition_power, - ) - - state = { - "isa": "state", - "W_mean": np.mean(state_power), - "W_std": np.std(state_power), - "s": state_duration, - } - if self.with_traces: - state["plot"] = (state_timestamps - state_timestamps[0], state_power) - - transition["W_mean_delta_prev"] = transition["W_mean"] - np.mean( - prev_state_power - ) - transition["W_mean_delta_next"] = transition["W_mean"] - state["W_mean"] - - energy_trace.append(transition) - energy_trace.append(state) - - return energy_trace diff --git a/lib/loader/keysight.py b/lib/loader/keysight.py index b9b298d..9f9623a 100644 --- a/lib/loader/keysight.py +++ b/lib/loader/keysight.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 import csv -import logging +import io import numpy as np +import struct +import xml.etree.ElementTree as ET -logger = logging.getLogger(__name__) +from bisect import bisect_right class KeysightCSV: @@ -34,3 +36,288 @@ class KeysightCSV: timestamps[i] = float(row[0]) currents[i] = float(row[2]) * -1 return timestamps, currents + + +class DLogChannel: + def __init__(self, desc_tuple): + self.slot = desc_tuple[0] + self.smu = desc_tuple[1] + self.unit = desc_tuple[2] + self.data = None + + def __repr__(self): + return f"""<DLogChannel(slot={self.slot}, smu="{self.smu}", unit="{self.unit}", data={self.data})>""" + + +class DLog: + def __init__( + self, + voltage: float, + state_duration: int, + with_traces=False, + skip_duration=None, + limit_duration=None, + ): + self.voltage = voltage + self.state_duration = state_duration + self.with_traces = with_traces + self.skip_duration = skip_duration + self.limit_duration = limit_duration + self.errors = list() + + self.sync_min_duration = 0.7 + # TODO auto-detect + self.sync_power = 10e-3 + + def load_data(self, content): + lines = [] + line = "" + with io.BytesIO(content) as f: + while line != "</dlog>\n": + line = f.readline().decode() + lines.append(line) + xml_header = "".join(lines) + raw_header = f.read(8) + data_offset = f.tell() + raw_data = f.read() + + xml_header = xml_header.replace("1ua>", "X1ua>") + xml_header = xml_header.replace("2ua>", "X2ua>") + dlog = ET.fromstring(xml_header) + channels = [] + + for channel in dlog.findall("channel"): + channel_id = int(channel.get("id")) + sense_curr = channel.find("sense_curr").text + sense_volt = channel.find("sense_volt").text + model = channel.find("ident").find("model").text + if sense_volt == "1": + channels.append((channel_id, model, "V")) + if sense_curr == "1": + channels.append((channel_id, model, "A")) + + num_channels = len(channels) + + self.channels = list(map(DLogChannel, channels)) + self.interval = float(dlog.find("frame").find("tint").text) + self.sense_minmax = int(dlog.find("frame").find("sense_minmax").text) + self.planned_duration = int(dlog.find("frame").find("time").text) + self.observed_duration = self.interval * int(len(raw_data) / (4 * num_channels)) + + if self.sense_minmax: + raise RuntimeError( + "DLog files with 'Log Min/Max' enabled are not supported yet" + ) + + self.timestamps = np.linspace( + 0, self.observed_duration, num=int(len(raw_data) / (4 * num_channels)) + ) + + if ( + self.skip_duration is not None + and self.observed_duration >= self.skip_duration + ): + start_offset = 0 + for i, ts in enumerate(self.timestamps): + if ts >= self.skip_duration: + start_offset = i + break + self.timestamps = self.timestamps[start_offset:] + raw_data = raw_data[start_offset * 4 * num_channels :] + + if ( + self.limit_duration is not None + and self.observed_duration > self.limit_duration + ): + stop_offset = len(self.timestamps) - 1 + for i, ts in enumerate(self.timestamps): + if ts > self.limit_duration: + stop_offset = i + break + self.timestamps = self.timestamps[:stop_offset] + self.observed_duration = self.timestamps[-1] + raw_data = raw_data[: stop_offset * 4 * num_channels] + + self.data = np.ndarray( + shape=(num_channels, int(len(raw_data) / (4 * num_channels))), + dtype=np.float32, + ) + + iterator = struct.iter_unpack(">f", raw_data) + channel_offset = 0 + measurement_offset = 0 + for value in iterator: + if value[0] < -1e6 or value[0] > 1e6: + print( + f"Invalid data value {value[0]} at channel {channel_offset}, measurement {measurement_offset}. Replacing with 0." + ) + self.data[channel_offset, measurement_offset] = 0 + else: + self.data[channel_offset, measurement_offset] = value[0] + if channel_offset + 1 == num_channels: + channel_offset = 0 + measurement_offset += 1 + else: + channel_offset += 1 + + # An SMU has four slots + self.slots = [dict(), dict(), dict(), dict()] + + for i, channel in enumerate(self.channels): + channel.data = self.data[i] + self.slots[channel.slot - 1][channel.unit] = channel + + assert "A" in self.slots[0] + self.data = self.slots[0]["A"].data + + def observed_duration_equals_expectation(self): + return int(self.observed_duration) == self.planned_duration + + # very similar to DataProcessor.getStatesdfatool + def analyze_states(self, expected_trace, repeat_id): + sync_start = None + sync_timestamps = list() + above_count = 0 + below_count = 0 + for i, timestamp in enumerate(self.timestamps): + power = self.voltage * self.data[i] + if power > self.sync_power: + above_count += 1 + below_count = 0 + else: + above_count = 0 + below_count += 1 + + if above_count > 2 and sync_start is None: + sync_start = timestamp + elif below_count > 2 and sync_start is not None: + if timestamp - sync_start > self.sync_min_duration: + sync_end = timestamp + sync_timestamps.append((sync_start, sync_end)) + sync_start = None + print(sync_timestamps) + + if len(sync_timestamps) != 3: + self.errors.append( + f"Found {len(sync_timestamps)} synchronization pulses, expected three." + ) + self.errors.append(f"Synchronization pulses == {sync_timestamps}") + return list() + + start_ts = sync_timestamps[0][1] + end_ts = sync_timestamps[1][0] + + # start and end of first state + online_timestamps = [0, expected_trace[0]["start_offset"][repeat_id]] + + # remaining events from the end of the first transition (start of second state) to the end of the last observed state + for trace in expected_trace: + for word in trace["trace"]: + online_timestamps.append( + online_timestamps[-1] + + word["online_aggregates"]["duration"][repeat_id] + ) + + online_timestamps = np.array(online_timestamps) * 1e-6 + online_timestamps = ( + online_timestamps * ((end_ts - start_ts) / online_timestamps[-1]) + start_ts + ) + + trigger_edges = list() + for ts in online_timestamps: + trigger_edges.append(bisect_right(self.timestamps, ts)) + + energy_trace = list() + + for i in range(2, len(online_timestamps), 2): + prev_state_start_index = trigger_edges[i - 2] + prev_state_stop_index = trigger_edges[i - 1] + transition_start_index = trigger_edges[i - 1] + transition_stop_index = trigger_edges[i] + state_start_index = trigger_edges[i] + state_stop_index = trigger_edges[i + 1] + + # If a transition takes less time than the measurement interval, its start and stop index may be the same. + # In this case, self.data[transition_start_index] is the only data point affected by the transition. + # We use the self.data slice [transition_start_index, transition_stop_index) to determine the mean power, so we need + # to increment transition_stop_index by 1 to end at self.data[transition_start_index] + # (self.data[transition_start_index : transition_start_index+1 ] == [self.data[transition_start_index]) + if transition_stop_index == transition_start_index: + transition_stop_index += 1 + + prev_state_duration = online_timestamps[i + 1] - online_timestamps[i] + transition_duration = online_timestamps[i] - online_timestamps[i - 1] + state_duration = online_timestamps[i + 1] - online_timestamps[i] + + # some states are followed by a UART dump of log data. This causes an increase in CPU energy + # consumption and is not part of the peripheral behaviour, so it should not be part of the benchmark results. + # If a case is followed by a UART dump, its duration is longer than the sleep duration between two transitions. + # In this case, we re-calculate the stop index, and calculate the state duration from coarse energytrace data + # instead of high-precision sync data + if ( + self.timestamps[prev_state_stop_index] + - self.timestamps[prev_state_start_index] + > self.state_duration + ): + prev_state_stop_index = bisect_right( + self.timestamps, + self.timestamps[prev_state_start_index] + self.state_duration, + ) + prev_state_duration = ( + self.timestamps[prev_state_stop_index] + - self.timestamps[prev_state_start_index] + ) + + if ( + self.timestamps[state_stop_index] - self.timestamps[state_start_index] + > self.state_duration + ): + state_stop_index = bisect_right( + self.timestamps, + self.timestamps[state_start_index] + self.state_duration, + ) + state_duration = ( + self.timestamps[state_stop_index] + - self.timestamps[state_start_index] + ) + + prev_state_power = self.data[prev_state_start_index:prev_state_stop_index] + + transition_timestamps = self.timestamps[ + transition_start_index:transition_stop_index + ] + transition_power = self.data[transition_start_index:transition_stop_index] + + state_timestamps = self.timestamps[state_start_index:state_stop_index] + state_power = self.data[state_start_index:state_stop_index] + + transition = { + "isa": "transition", + "W_mean": np.mean(transition_power), + "W_std": np.std(transition_power), + "s": transition_duration, + } + if self.with_traces: + transition["plot"] = ( + transition_timestamps - transition_timestamps[0], + transition_power, + ) + + state = { + "isa": "state", + "W_mean": np.mean(state_power), + "W_std": np.std(state_power), + "s": state_duration, + } + if self.with_traces: + state["plot"] = (state_timestamps - state_timestamps[0], state_power) + + transition["W_mean_delta_prev"] = transition["W_mean"] - np.mean( + prev_state_power + ) + transition["W_mean_delta_next"] = transition["W_mean"] - state["W_mean"] + + energy_trace.append(transition) + energy_trace.append(state) + + return energy_trace |