From feafeb9f619d426201b98d05e4feb77c8b1cf4a3 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Thu, 2 Jul 2020 09:29:01 +0200 Subject: Use logging module for debug output --- lib/functions.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) (limited to 'lib/functions.py') diff --git a/lib/functions.py b/lib/functions.py index 6d8daa4..359c8d7 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -5,12 +5,14 @@ This module provides classes and helper functions useful for least-squares regression and general handling of model functions. """ from itertools import chain, combinations +import logging import numpy as np import re from scipy import optimize -from .utils import is_numeric, vprint +from .utils import is_numeric arg_support_enabled = True +logger = logging.getLogger(__name__) def powerset(iterable): @@ -118,9 +120,7 @@ class AnalyticFunction: packet length. """ - def __init__( - self, function_str, parameters, num_args, verbose=True, regression_args=None - ): + def __init__(self, function_str, parameters, num_args, regression_args=None): """ Create a new AnalyticFunction object from a function string. @@ -135,7 +135,6 @@ class AnalyticFunction: :param num_args: number of local function arguments, if any. Set to 0 if the model attribute does not belong to a function or if function arguments are not included in the model. - :param verbose: complain about odd events :param regression_args: Initial regression variable values, both for function usage and least squares optimization. If unset, defaults to [1, 1, 1, ...] @@ -146,7 +145,6 @@ class AnalyticFunction: rawfunction = function_str self._dependson = [False] * (len(parameters) + num_args) self.fit_success = False - self.verbose = verbose if type(function_str) == str: num_vars_re = re.compile(r"regression_arg\(([0-9]+)\)") @@ -231,9 +229,8 @@ class AnalyticFunction: else: X[i].extend([np.nan] * len(val[model_attribute])) elif key[0] == state_or_tran and len(key[1]) != dimension: - vprint( - self.verbose, - "[W] Invalid parameter key length while gathering fit data for {}/{}. is {}, want {}.".format( + logging.warning( + "Invalid parameter key length while gathering fit data for {}/{}. is {}, want {}.".format( state_or_tran, model_attribute, len(key[1]), dimension ), ) @@ -266,9 +263,8 @@ class AnalyticFunction: error_function, self._regression_args, args=(X, Y), xtol=2e-15 ) except ValueError as err: - vprint( - self.verbose, - "[W] Fit failed for {}/{}: {} (function: {})".format( + logging.warning( + "Fit failed for {}/{}: {} (function: {})".format( state_or_tran, model_attribute, err, self._model_str ), ) @@ -277,16 +273,14 @@ class AnalyticFunction: self._regression_args = res.x self.fit_success = True else: - vprint( - self.verbose, - "[W] Fit failed for {}/{}: {} (function: {})".format( + logging.warning( + "Fit failed for {}/{}: {} (function: {})".format( state_or_tran, model_attribute, res.message, self._model_str ), ) else: - vprint( - self.verbose, - "[W] Insufficient amount of valid parameter keys, cannot fit {}/{}".format( + logging.warning( + "Insufficient amount of valid parameter keys, cannot fit {}/{}".format( state_or_tran, model_attribute ), ) -- cgit v1.2.3 From e42f3541d9e16264f79e090ddd87b864f5c2a837 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 3 Jul 2020 09:27:46 +0200 Subject: more consistent logging; use logger. instead of logging. where appropriate --- lib/automata.py | 11 +++++++---- lib/data_parameters.py | 17 +++++++++------- lib/dfatool.py | 42 +++++++++++++++++++--------------------- lib/functions.py | 8 ++++---- lib/lex.py | 9 ++++++--- lib/parameters.py | 48 +++++++++++++++++++++++++--------------------- lib/protocol_benchmarks.py | 7 +++++-- lib/utils.py | 4 +++- 8 files changed, 81 insertions(+), 65 deletions(-) (limited to 'lib/functions.py') diff --git a/lib/automata.py b/lib/automata.py index b3318e0..69b3969 100755 --- a/lib/automata.py +++ b/lib/automata.py @@ -3,11 +3,14 @@ from .functions import AnalyticFunction, NormalizationFunction from .utils import is_numeric import itertools +import logging import numpy as np import json import queue import yaml +logger = logging.getLogger(__name__) + def _dict_to_list(input_dict: dict) -> list: return [input_dict[x] for x in sorted(input_dict.keys())] @@ -1305,8 +1308,8 @@ class PTA: "power" ] except KeyError: - print( - "[W] skipping model update of state {} due to missing data".format( + logger.warning( + "skipping model update of state {} due to missing data".format( state.name ) ) @@ -1353,8 +1356,8 @@ class PTA: "timeout" ] except KeyError: - print( - "[W] skipping model update of transition {} due to missing data".format( + logger.warning( + "skipping model update of transition {} due to missing data".format( transition.name ) ) diff --git a/lib/data_parameters.py b/lib/data_parameters.py index 1150b71..84eacfd 100644 --- a/lib/data_parameters.py +++ b/lib/data_parameters.py @@ -7,9 +7,12 @@ length of lists, ane more. from .protocol_benchmarks import codegen_for_lib from . import cycles_to_energy, size_to_radio_energy, utils +import logging import numpy as np import ubjson +logger = logging.getLogger(__name__) + def _string_value_length(json): if type(json) == str: @@ -224,7 +227,7 @@ class Protolog: except KeyError: pass except TypeError as e: - print( + logger.error( "TypeError in {} {} {} {}: {} -> {}".format( arch_lib, benchmark, @@ -395,7 +398,7 @@ class Protolog: except KeyError: pass except ValueError: - print( + logger.warning( "cycles_enc is NaN for {} -> {} -> {}".format( arch, lib, key ) @@ -410,7 +413,7 @@ class Protolog: except KeyError: pass except ValueError: - print( + logger.warning( "cycles_ser is NaN for {} -> {} -> {}".format( arch, lib, key ) @@ -425,7 +428,7 @@ class Protolog: except KeyError: pass except ValueError: - print( + logger.warning( "cycles_encser is NaN for {} -> {} -> {}".format( arch, lib, key ) @@ -440,7 +443,7 @@ class Protolog: except KeyError: pass except ValueError: - print( + logger.warning( "cycles_des is NaN for {} -> {} -> {}".format( arch, lib, key ) @@ -455,7 +458,7 @@ class Protolog: except KeyError: pass except ValueError: - print( + logger.warning( "cycles_dec is NaN for {} -> {} -> {}".format( arch, lib, key ) @@ -470,7 +473,7 @@ class Protolog: except KeyError: pass except ValueError: - print( + logger.warning( "cycles_desdec is NaN for {} -> {} -> {}".format( arch, lib, key ) diff --git a/lib/dfatool.py b/lib/dfatool.py index ef3bac7..1e38907 100644 --- a/lib/dfatool.py +++ b/lib/dfatool.py @@ -75,7 +75,7 @@ def gplearn_to_function(function_str: str): arg_list.append("X{:d}".format(i)) eval_str = "lambda {}, *whatever: {}".format(",".join(arg_list), function_str) - print(eval_str) + logger.debug(eval_str) return eval(eval_str, eval_globals) @@ -1371,7 +1371,7 @@ class RawData: for measurement in measurements: if "energy_trace" not in measurement: - logging.warning( + logger.warning( "Skipping {ar:s}/{m:s}: {e:s}".format( ar=self.filenames[measurement["fileno"]], m=measurement["info"].name, @@ -1393,7 +1393,7 @@ class RawData: self._merge_online_and_offline(measurement) num_valid += 1 else: - logging.warning( + logger.warning( "Skipping {ar:s}/{m:s}: {e:s}".format( ar=self.filenames[measurement["fileno"]], m=measurement["info"].name, @@ -1405,14 +1405,14 @@ class RawData: self._merge_online_and_etlog(measurement) num_valid += 1 else: - logging.warning( + logger.warning( "Skipping {ar:s}/{m:s}: {e:s}".format( ar=self.filenames[measurement["fileno"]], m=measurement["info"].name, e=measurement["error"], ), ) - logging.info( + logger.info( "{num_valid:d}/{num_total:d} measurements are valid".format( num_valid=num_valid, num_total=len(measurements) ), @@ -1834,9 +1834,9 @@ class AnalyticModel: try: model[name][key] = model_function(elem[key]) except RuntimeWarning: - logging.warning("Got no data for {} {}".format(name, key)) + logger.warning("Got no data for {} {}".format(name, key)) except FloatingPointError as fpe: - logging.warning("Got no data for {} {}: {}".format(name, key, fpe),) + logger.warning("Got no data for {} {}: {}".format(name, key, fpe),) return model def param_index(self, param_name): @@ -2230,9 +2230,9 @@ class PTAModel: try: model[name][key] = model_function(elem[key]) except RuntimeWarning: - logging.warning("Got no data for {} {}".format(name, key)) + logger.warning("Got no data for {} {}".format(name, key)) except FloatingPointError as fpe: - logging.warning("Got no data for {} {}: {}".format(name, key, fpe),) + logger.warning("Got no data for {} {}: {}".format(name, key, fpe),) return model def get_static(self, use_mean=False): @@ -2730,7 +2730,7 @@ class EnergyTraceLog: self.sample_rate = data_count / (m_duration_us * 1e-6) - logging.debug( + logger.debug( "got {} samples with {} seconds of log data ({} Hz)".format( data_count, m_duration_us * 1e-6, self.sample_rate ), @@ -2837,19 +2837,17 @@ class EnergyTraceLog: for name, duration in expected_transitions: bc, start, stop, end = self.find_barcode(next_barcode) if bc is None: - print('[!!!] did not find transition "{}"'.format(name)) + logger.error('did not find transition "{}"'.format(name)) break next_barcode = end + self.state_duration + duration - logging.debug( + logger.debug( '{} barcode "{}" area: {:0.2f} .. {:0.2f} / {:0.2f} seconds'.format( offline_index, bc, start, stop, end ), ) if bc != name: - logging.debug( - '[!!!] mismatch: expected "{}", got "{}"'.format(name, bc), - ) - logging.debug( + logger.error('mismatch: expected "{}", got "{}"'.format(name, bc),) + logger.debug( "{} estimated transition area: {:0.3f} .. {:0.3f} seconds".format( offline_index, end, end + duration ), @@ -2862,7 +2860,7 @@ class EnergyTraceLog: self.ts_to_index(end + duration + self.state_duration) + 1 ) - logging.debug( + logger.debug( "{} estimated transitionindex: {:0.3f} .. {:0.3f} seconds".format( offline_index, transition_start_index / self.sample_rate, @@ -2962,7 +2960,7 @@ class EnergyTraceLog: + self.led_power / 3 ) - logging.debug( + logger.debug( "looking for barcode starting at {:0.2f} s, threshold is {:0.1f} mW".format( start_ts, sync_threshold_power * 1e3 ), @@ -2996,7 +2994,7 @@ class EnergyTraceLog: barcode_data = self.interval_power[sync_area_start:sync_area_end] - logging.debug( + logger.debug( "barcode search area: {:0.2f} .. {:0.2f} seconds ({} samples)".format( sync_start_ts, sync_end_ts, len(barcode_data) ), @@ -3074,7 +3072,7 @@ class EnergyTraceLog: return content, sym_start, sym_end, padding_bits else: - logging.warning("unable to find barcode") + logger.warning("unable to find barcode") return None, None, None, None @@ -3300,7 +3298,7 @@ class MIMOSA: if cal_r2_mean > cal_0_mean: b_lower = (ua_r2 - 0) / (cal_r2_mean - cal_0_mean) else: - logging.warning("0 uA == %.f uA during calibration" % (ua_r2)) + logger.warning("0 uA == %.f uA during calibration" % (ua_r2)) b_lower = 0 b_upper = (ua_r1 - ua_r2) / (cal_r1_mean - cal_r2_mean) @@ -3472,7 +3470,7 @@ class MIMOSA: data["substates"] = substates ssum = np.sum(list(map(lambda x: x["duration"], substates["states"]))) if ssum != data["us"]: - logging.warning("duration %d vs %d" % (data["us"], ssum)) + logger.warning("duration %d vs %d" % (data["us"], ssum)) if isa == "transition": # subtract average power of previous state diff --git a/lib/functions.py b/lib/functions.py index 359c8d7..0b849bd 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -229,7 +229,7 @@ class AnalyticFunction: else: X[i].extend([np.nan] * len(val[model_attribute])) elif key[0] == state_or_tran and len(key[1]) != dimension: - logging.warning( + logger.warning( "Invalid parameter key length while gathering fit data for {}/{}. is {}, want {}.".format( state_or_tran, model_attribute, len(key[1]), dimension ), @@ -263,7 +263,7 @@ class AnalyticFunction: error_function, self._regression_args, args=(X, Y), xtol=2e-15 ) except ValueError as err: - logging.warning( + logger.warning( "Fit failed for {}/{}: {} (function: {})".format( state_or_tran, model_attribute, err, self._model_str ), @@ -273,13 +273,13 @@ class AnalyticFunction: self._regression_args = res.x self.fit_success = True else: - logging.warning( + logger.warning( "Fit failed for {}/{}: {} (function: {})".format( state_or_tran, model_attribute, res.message, self._model_str ), ) else: - logging.warning( + logger.warning( "Insufficient amount of valid parameter keys, cannot fit {}/{}".format( state_or_tran, model_attribute ), diff --git a/lib/lex.py b/lib/lex.py index 7bb3760..f698e8c 100644 --- a/lib/lex.py +++ b/lib/lex.py @@ -1,4 +1,7 @@ from .sly import Lexer, Parser +import logging + +logger = logging.getLogger(__name__) class TimedWordLexer(Lexer): @@ -38,7 +41,7 @@ class TimedSequenceLexer(Lexer): FUNCTIONSEP = r";" def error(self, t): - print("Illegal character '%s'" % t.value[0]) + logger.error("Illegal character '%s'" % t.value[0]) if t.value[0] == "{" and t.value.find("}"): self.index += 1 + t.value.find("}") else: @@ -153,11 +156,11 @@ class TimedSequenceParser(Parser): def error(self, p): if p: - print("Syntax error at token", p.type) + logger.error("Syntax error at token", p.type) # Just discard the token and tell the parser it's okay. self.errok() else: - print("Syntax error at EOF") + logger.error("Syntax error at EOF") class TimedWord: diff --git a/lib/parameters.py b/lib/parameters.py index bd67cc1..79543a6 100644 --- a/lib/parameters.py +++ b/lib/parameters.py @@ -1,6 +1,7 @@ import itertools import logging import numpy as np +import warnings from collections import OrderedDict from copy import deepcopy from multiprocessing import Pool @@ -163,12 +164,11 @@ def _std_by_param(by_param, all_param_values, state_or_tran, attribute, param_in # vprint(verbose, '[W] parameter value partition for {} is empty'.format(param_value)) if np.all(np.isnan(stddev_matrix)): - print( - "[W] {}/{} parameter #{} has no data partitions -- how did this even happen?".format( - state_or_tran, attribute, param_index + warnings.warn( + "{}/{} parameter #{} has no data partitions. stddev_matrix = {}".format( + state_or_tran, attribute, param_index, stddev_matrix ) ) - print("stddev_matrix = {}".format(stddev_matrix)) return stddev_matrix, 0.0 return ( @@ -203,13 +203,13 @@ def _corr_by_param(by_name, state_or_trans, attribute, param_index): # -> assume no correlation return 0.0 except ValueError: - print( - "[!] Exception in _corr_by_param(by_name, state_or_trans={}, attribute={}, param_index={})".format( + logger.error( + "ValueError in _corr_by_param(by_name, state_or_trans={}, attribute={}, param_index={})".format( state_or_trans, attribute, param_index ) ) - print( - "[!] while executing np.corrcoef(by_name[{}][{}]={}, {}))".format( + logger.error( + "while executing np.corrcoef(by_name[{}][{}]={}, {}))".format( state_or_trans, attribute, by_name[state_or_trans][attribute], @@ -443,8 +443,8 @@ def prune_dependent_parameters(by_name, parameter_names, correlation_threshold=0 correlation != np.nan and np.abs(correlation) > correlation_threshold ): - print( - "[!] Parameters {} <-> {} are correlated with coefficcient {}".format( + logger.debug( + "Parameters {} <-> {} are correlated with coefficcient {}".format( parameter_names[index_1], parameter_names[index_2], correlation, @@ -454,7 +454,7 @@ def prune_dependent_parameters(by_name, parameter_names, correlation_threshold=0 index_to_remove = index_1 else: index_to_remove = index_2 - print( + logger.debug( " Removing parameter {}".format( parameter_names[index_to_remove] ) @@ -581,15 +581,17 @@ class ParamStats: ) > 2 ): - print( - key, - param, - list( - filter( - lambda n: is_numeric(n), - self.distinct_values[key][param], - ) - ), + logger.debug( + "{} can be fitted for param {} on {}".format( + key, + param, + list( + filter( + lambda n: is_numeric(n), + self.distinct_values[key][param], + ) + ), + ) ) return True return False @@ -646,13 +648,15 @@ class ParamStats: depends_on_a_parameter = False for param in self._parameter_names: if self.stats[state_or_tran][attribute]["depends_on_param"][param]: - print("{}/{} depends on {}".format(state_or_tran, attribute, param)) + logger.debug( + "{}/{} depends on {}".format(state_or_tran, attribute, param) + ) depends_on_a_parameter = True if ( len(self.codependent_parameters(state_or_tran, attribute, param)) == 0 ): - print("has no codependent parameters") + logger.debug("... and has no codependent parameters") # Always depends on this parameter, regardless of other parameters' values return False return depends_on_a_parameter diff --git a/lib/protocol_benchmarks.py b/lib/protocol_benchmarks.py index b42e821..d41979f 100755 --- a/lib/protocol_benchmarks.py +++ b/lib/protocol_benchmarks.py @@ -16,8 +16,11 @@ import io import os import re import time +import logging from filelock import FileLock +logger = logging.getLogger(__name__) + class DummyProtocol: def __init__(self): @@ -1838,14 +1841,14 @@ class Benchmark: this_result["data"] = data if value != None: this_result[key] = {"v": value, "ts": int(time.time())} - print( + logger.debug( "{} {} {} ({}) :: {} -> {}".format( libkey, bench_name, bench_index, data, key, value ) ) else: this_result[key] = {"e": error, "ts": int(time.time())} - print( + logger.debug( "{} {} {} ({}) :: {} -> [E] {}".format( libkey, bench_name, bench_index, data, key, error[:500] ) diff --git a/lib/utils.py b/lib/utils.py index 8186ee7..d28ecda 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -1,7 +1,9 @@ import numpy as np import re +import logging arg_support_enabled = True +logger = logging.getLogger(__name__) def running_mean(x: np.ndarray, N: int) -> np.ndarray: @@ -212,7 +214,7 @@ def filter_aggregate_by_param(aggregate, parameters, parameter_filter): ) ) if len(indices_to_keep) == 0: - print("??? {}->{}".format(parameter_filter, name)) + logger.debug("??? {}->{}".format(parameter_filter, name)) names_to_remove.add(name) else: for attribute in aggregate[name]["attributes"]: -- cgit v1.2.3 From adaa03cf0247b065e6b3863cf16ad88ee24f5169 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Fri, 3 Jul 2020 12:26:50 +0200 Subject: AnalyticFunction: Remove _ prefix from public attributes --- bin/analyze-archive.py | 8 ++++---- bin/analyze-timing.py | 4 ++-- bin/eval-outlier-removal.py | 16 ++++++++-------- lib/automata.py | 6 +++--- lib/functions.py | 20 ++++++++++---------- test/test_parameters.py | 2 +- test/test_ptamodel.py | 18 +++++++++--------- test/test_timingharness.py | 32 ++++++++++++++++---------------- 8 files changed, 53 insertions(+), 53 deletions(-) (limited to 'lib/functions.py') diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index bf3ab64..e23fb9e 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -645,13 +645,13 @@ if __name__ == "__main__": if param_info(state, attribute): print( "{:10s}: {}".format( - state, param_info(state, attribute)["function"]._model_str + state, param_info(state, attribute)["function"].model_function ) ) print( "{:10s} {}".format( "", - param_info(state, attribute)["function"]._regression_args, + param_info(state, attribute)["function"].model_args, ) ) for trans in model.transitions(): @@ -661,14 +661,14 @@ if __name__ == "__main__": "{:10s}: {:10s}: {}".format( trans, attribute, - param_info(trans, attribute)["function"]._model_str, + param_info(trans, attribute)["function"].model_function, ) ) print( "{:10s} {:10s} {}".format( "", "", - param_info(trans, attribute)["function"]._regression_args, + param_info(trans, attribute)["function"].model_args, ) ) diff --git a/bin/analyze-timing.py b/bin/analyze-timing.py index 4039f45..924388d 100755 --- a/bin/analyze-timing.py +++ b/bin/analyze-timing.py @@ -423,14 +423,14 @@ if __name__ == "__main__": "{:10s}: {:10s}: {}".format( trans, attribute, - param_info(trans, attribute)["function"]._model_str, + param_info(trans, attribute)["function"].model_function, ) ) print( "{:10s} {:10s} {}".format( "", "", - param_info(trans, attribute)["function"]._regression_args, + param_info(trans, attribute)["function"].model_args, ) ) diff --git a/bin/eval-outlier-removal.py b/bin/eval-outlier-removal.py index 14f0e60..b091ea4 100755 --- a/bin/eval-outlier-removal.py +++ b/bin/eval-outlier-removal.py @@ -141,12 +141,12 @@ if __name__ == "__main__": if param_i1(state, attribute): print( "{:10s}: {}".format( - state, param_i1(state, attribute)["function"]._model_str + state, param_i1(state, attribute)["function"].model_function ) ) print( "{:10s} {}".format( - "", param_i1(state, attribute)["function"]._regression_args + "", param_i1(state, attribute)["function"].model_args ) ) for trans in m1.transitions(): @@ -162,12 +162,12 @@ if __name__ == "__main__": "{:10s}: {:10s}: {}".format( trans, attribute, - param_i1(trans, attribute)["function"]._model_str, + param_i1(trans, attribute)["function"].model_function, ) ) print( "{:10s} {:10s} {}".format( - "", "", param_i1(trans, attribute)["function"]._regression_args + "", "", param_i1(trans, attribute)["function"].model_args ) ) param_m2, param_i2 = m2.get_fitted() @@ -176,12 +176,12 @@ if __name__ == "__main__": if param_i2(state, attribute): print( "{:10s}: {}".format( - state, param_i2(state, attribute)["function"]._model_str + state, param_i2(state, attribute)["function"].model_function ) ) print( "{:10s} {}".format( - "", param_i2(state, attribute)["function"]._regression_args + "", param_i2(state, attribute)["function"].model_args ) ) for trans in m2.transitions(): @@ -197,12 +197,12 @@ if __name__ == "__main__": "{:10s}: {:10s}: {}".format( trans, attribute, - param_i2(trans, attribute)["function"]._model_str, + param_i2(trans, attribute)["function"].model_function, ) ) print( "{:10s} {:10s} {}".format( - "", "", param_i2(trans, attribute)["function"]._regression_args + "", "", param_i2(trans, attribute)["function"].model_args ) ) diff --git a/lib/automata.py b/lib/automata.py index 69b3969..ebe1871 100755 --- a/lib/automata.py +++ b/lib/automata.py @@ -103,7 +103,7 @@ class PTAAttribute: def __repr__(self): if self.function is not None: return "PTAATtribute<{:.0f}, {}>".format( - self.value, self.function._model_str + self.value, self.function.model_function ) return "PTAATtribute<{:.0f}, None>".format(self.value) @@ -137,8 +137,8 @@ class PTAAttribute: } if self.function: ret["function"] = { - "raw": self.function._model_str, - "regression_args": list(self.function._regression_args), + "raw": self.function.model_function, + "regression_args": list(self.function.model_args), } ret["function_error"] = self.function_error return ret diff --git a/lib/functions.py b/lib/functions.py index 0b849bd..99ba17d 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -141,7 +141,7 @@ class AnalyticFunction: """ self._parameter_names = parameters self._num_args = num_args - self._model_str = function_str + self.model_function = function_str rawfunction = function_str self._dependson = [False] * (len(parameters) + num_args) self.fit_success = False @@ -174,12 +174,12 @@ class AnalyticFunction: self._function = function_str if regression_args: - self._regression_args = regression_args.copy() + self.model_args = regression_args.copy() self._fit_success = True elif type(function_str) == str: - self._regression_args = list(np.ones((num_vars))) + self.model_args = list(np.ones((num_vars))) else: - self._regression_args = [] + self.model_args = [] def get_fit_data(self, by_param, state_or_tran, model_attribute): """ @@ -260,22 +260,22 @@ class AnalyticFunction: error_function = lambda P, X, y: self._function(P, X) - y try: res = optimize.least_squares( - error_function, self._regression_args, args=(X, Y), xtol=2e-15 + error_function, self.model_args, args=(X, Y), xtol=2e-15 ) except ValueError as err: logger.warning( "Fit failed for {}/{}: {} (function: {})".format( - state_or_tran, model_attribute, err, self._model_str + state_or_tran, model_attribute, err, self.model_function ), ) return if res.status > 0: - self._regression_args = res.x + self.model_args = res.x self.fit_success = True else: logger.warning( "Fit failed for {}/{}: {} (function: {})".format( - state_or_tran, model_attribute, res.message, self._model_str + state_or_tran, model_attribute, res.message, self.model_function ), ) else: @@ -308,9 +308,9 @@ class AnalyticFunction: corresponds to lexically first parameter, etc. :param arg_list: argument values (list of float), if arguments are used. """ - if len(self._regression_args) == 0: + if len(self.model_args) == 0: return self._function(param_list, arg_list) - return self._function(self._regression_args, param_list) + return self._function(self.model_args, param_list) class analytic: diff --git a/test/test_parameters.py b/test/test_parameters.py index 5d7ec84..baf1c99 100755 --- a/test/test_parameters.py +++ b/test/test_parameters.py @@ -63,7 +63,7 @@ class TestModels(unittest.TestCase): combined_fit = analytic.function_powerset(fit_result, parameter_names, 0) self.assertEqual( - combined_fit._model_str, + combined_fit.model_function, "0 + regression_arg(0) + regression_arg(1) * parameter(p_linear)", ) self.assertEqual( diff --git a/test/test_ptamodel.py b/test/test_ptamodel.py index 3237450..9abe3c0 100755 --- a/test/test_ptamodel.py +++ b/test/test_ptamodel.py @@ -134,26 +134,26 @@ class TestModels(unittest.TestCase): param_model, param_info = model.get_fitted() self.assertEqual(param_info("POWERDOWN", "power"), None) self.assertEqual( - param_info("RX", "power")["function"]._model_str, + param_info("RX", "power")["function"].model_function, "0 + regression_arg(0) + regression_arg(1) * np.sqrt(parameter(datarate))", ) self.assertAlmostEqual( - param_info("RX", "power")["function"]._regression_args[0], 48530.7, places=0 + param_info("RX", "power")["function"].model_args[0], 48530.7, places=0 ) self.assertAlmostEqual( - param_info("RX", "power")["function"]._regression_args[1], 117, places=0 + param_info("RX", "power")["function"].model_args[1], 117, places=0 ) self.assertEqual(param_info("STANDBY1", "power"), None) self.assertEqual( - param_info("TX", "power")["function"]._model_str, + param_info("TX", "power")["function"].model_function, "0 + regression_arg(0) + regression_arg(1) * 1/(parameter(datarate)) + regression_arg(2) * parameter(txpower) + regression_arg(3) * 1/(parameter(datarate)) * parameter(txpower)", ) self.assertEqual( - param_info("epilogue", "timeout")["function"]._model_str, + param_info("epilogue", "timeout")["function"].model_function, "0 + regression_arg(0) + regression_arg(1) * 1/(parameter(datarate))", ) self.assertEqual( - param_info("stopListening", "duration")["function"]._model_str, + param_info("stopListening", "duration")["function"].model_function, "0 + regression_arg(0) + regression_arg(1) * 1/(parameter(datarate))", ) @@ -371,7 +371,7 @@ class TestModels(unittest.TestCase): param_model, param_info = model.get_fitted() self.assertEqual(param_info("IDLE", "power"), None) self.assertEqual( - param_info("RX", "power")["function"]._model_str, + param_info("RX", "power")["function"].model_function, "0 + regression_arg(0) + regression_arg(1) * np.log(parameter(symbolrate) + 1)", ) self.assertEqual(param_info("SLEEP", "power"), None) @@ -380,10 +380,10 @@ class TestModels(unittest.TestCase): self.assertEqual(param_info("XOFF", "power"), None) self.assertAlmostEqual( - param_info("RX", "power")["function"]._regression_args[0], 84415, places=0 + param_info("RX", "power")["function"].model_args[0], 84415, places=0 ) self.assertAlmostEqual( - param_info("RX", "power")["function"]._regression_args[1], 206, places=0 + param_info("RX", "power")["function"].model_args[1], 206, places=0 ) diff --git a/test/test_timingharness.py b/test/test_timingharness.py index 29e21f8..13289ea 100755 --- a/test/test_timingharness.py +++ b/test/test_timingharness.py @@ -30,25 +30,25 @@ class TestModels(unittest.TestCase): self.assertEqual(param_info("setRetries", "duration"), None) self.assertEqual(param_info("setup", "duration"), None) self.assertEqual( - param_info("write", "duration")["function"]._model_str, + param_info("write", "duration")["function"].model_function, "0 + regression_arg(0) + regression_arg(1) * parameter(max_retry_count) + regression_arg(2) * parameter(retry_delay) + regression_arg(3) * parameter(max_retry_count) * parameter(retry_delay)", ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[0], + param_info("write", "duration")["function"].model_args[0], 1163, places=0, ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[1], + param_info("write", "duration")["function"].model_args[1], 464, places=0, ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[2], 1, places=0 + param_info("write", "duration")["function"].model_args[2], 1, places=0 ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[3], 1, places=0 + param_info("write", "duration")["function"].model_args[3], 1, places=0 ) def test_dependent_parameter_pruning(self): @@ -80,25 +80,25 @@ class TestModels(unittest.TestCase): self.assertEqual(param_info("setRetries", "duration"), None) self.assertEqual(param_info("setup", "duration"), None) self.assertEqual( - param_info("write", "duration")["function"]._model_str, + param_info("write", "duration")["function"].model_function, "0 + regression_arg(0) + regression_arg(1) * parameter(max_retry_count) + regression_arg(2) * parameter(retry_delay) + regression_arg(3) * parameter(max_retry_count) * parameter(retry_delay)", ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[0], + param_info("write", "duration")["function"].model_args[0], 1163, places=0, ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[1], + param_info("write", "duration")["function"].model_args[1], 464, places=0, ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[2], 1, places=0 + param_info("write", "duration")["function"].model_args[2], 1, places=0 ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[3], 1, places=0 + param_info("write", "duration")["function"].model_args[3], 1, places=0 ) def test_function_override(self): @@ -139,28 +139,28 @@ class TestModels(unittest.TestCase): self.assertEqual(param_info("setRetries", "duration"), None) self.assertEqual(param_info("setup", "duration"), None) self.assertEqual( - param_info("write", "duration")["function"]._model_str, + param_info("write", "duration")["function"].model_function, "(parameter(auto_ack!) * (regression_arg(0) + regression_arg(1) * parameter(max_retry_count) + regression_arg(2) * parameter(retry_delay) + regression_arg(3) * parameter(max_retry_count) * parameter(retry_delay))) + ((1 - parameter(auto_ack!)) * regression_arg(4))", ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[0], + param_info("write", "duration")["function"].model_args[0], 1162, places=0, ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[1], + param_info("write", "duration")["function"].model_args[1], 464, places=0, ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[2], 1, places=0 + param_info("write", "duration")["function"].model_args[2], 1, places=0 ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[3], 1, places=0 + param_info("write", "duration")["function"].model_args[3], 1, places=0 ) self.assertAlmostEqual( - param_info("write", "duration")["function"]._regression_args[4], + param_info("write", "duration")["function"].model_args[4], 1086, places=0, ) -- cgit v1.2.3 From 2a4ee78fd4c8b57f759135e068d85cf730b2e268 Mon Sep 17 00:00:00 2001 From: Daniel Friesel Date: Mon, 6 Jul 2020 15:28:07 +0200 Subject: move gplearn_to_function to functions module --- bin/analyze-archive.py | 2 +- bin/analyze-timing.py | 2 +- bin/test_corrcoef.py | 2 +- lib/dfatool.py | 47 ----------------------------------------------- lib/functions.py | 41 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 44 insertions(+), 50 deletions(-) (limited to 'lib/functions.py') diff --git a/bin/analyze-archive.py b/bin/analyze-archive.py index a18c1e1..7cac27a 100755 --- a/bin/analyze-archive.py +++ b/bin/analyze-archive.py @@ -114,7 +114,7 @@ import re import sys from dfatool import plotter from dfatool.dfatool import RawData, pta_trace_to_aggregate -from dfatool.dfatool import gplearn_to_function +from dfatool.functions import gplearn_to_function from dfatool.model import PTAModel from dfatool.validation import CrossValidator from dfatool.utils import filter_aggregate_by_param diff --git a/bin/analyze-timing.py b/bin/analyze-timing.py index e27acbf..3503279 100755 --- a/bin/analyze-timing.py +++ b/bin/analyze-timing.py @@ -80,7 +80,7 @@ import re import sys from dfatool import plotter from dfatool.dfatool import TimingData, pta_trace_to_aggregate -from dfatool.dfatool import gplearn_to_function +from dfatool.functions import gplearn_to_function from dfatool.model import AnalyticModel from dfatool.validation import CrossValidator from dfatool.utils import filter_aggregate_by_param diff --git a/bin/test_corrcoef.py b/bin/test_corrcoef.py index 75b5d0d..fef0b24 100755 --- a/bin/test_corrcoef.py +++ b/bin/test_corrcoef.py @@ -5,7 +5,7 @@ import re import sys from dfatool import plotter from dfatool.dfatool import RawData, pta_trace_to_aggregate -from dfatool.dfatool import gplearn_to_function +from dfatool.functions import gplearn_to_function from dfatool.model import PTAModel opt = dict() diff --git a/lib/dfatool.py b/lib/dfatool.py index 07aa7b3..47ce24e 100644 --- a/lib/dfatool.py +++ b/lib/dfatool.py @@ -27,53 +27,6 @@ except ImportError: arg_support_enabled = True -def gplearn_to_function(function_str: str): - """ - Convert gplearn-style function string to Python function. - - Takes a function string like "mul(add(X0, X1), X2)" and returns - a Python function implementing the specified behaviour, - e.g. "lambda x, y, z: (x + y) * z". - - Supported functions: - add -- x + y - sub -- x - y - mul -- x * y - div -- x / y if |y| > 0.001, otherwise 1 - sqrt -- sqrt(|x|) - log -- log(|x|) if |x| > 0.001, otherwise 0 - inv -- 1 / x if |x| > 0.001, otherwise 0 - """ - eval_globals = { - "add": lambda x, y: x + y, - "sub": lambda x, y: x - y, - "mul": lambda x, y: x * y, - "div": lambda x, y: np.divide(x, y) if np.abs(y) > 0.001 else 1.0, - "sqrt": lambda x: np.sqrt(np.abs(x)), - "log": lambda x: np.log(np.abs(x)) if np.abs(x) > 0.001 else 0.0, - "inv": lambda x: 1.0 / x if np.abs(x) > 0.001 else 0.0, - } - - last_arg_index = 0 - for i in range(0, 100): - if function_str.find("X{:d}".format(i)) >= 0: - last_arg_index = i - - arg_list = [] - for i in range(0, last_arg_index + 1): - arg_list.append("X{:d}".format(i)) - - eval_str = "lambda {}, *whatever: {}".format(",".join(arg_list), function_str) - logger.debug(eval_str) - return eval(eval_str, eval_globals) - - -def append_if_set(aggregate: dict, data: dict, key: str): - """Append data[key] to aggregate if key in data.""" - if key in data: - aggregate.append(data[key]) - - def mean_or_none(arr): """ Compute mean of NumPy array `arr`, return -1 if empty. diff --git a/lib/functions.py b/lib/functions.py index 99ba17d..94b1aaf 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -25,6 +25,47 @@ def powerset(iterable): return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) +def gplearn_to_function(function_str: str): + """ + Convert gplearn-style function string to Python function. + + Takes a function string like "mul(add(X0, X1), X2)" and returns + a Python function implementing the specified behaviour, + e.g. "lambda x, y, z: (x + y) * z". + + Supported functions: + add -- x + y + sub -- x - y + mul -- x * y + div -- x / y if |y| > 0.001, otherwise 1 + sqrt -- sqrt(|x|) + log -- log(|x|) if |x| > 0.001, otherwise 0 + inv -- 1 / x if |x| > 0.001, otherwise 0 + """ + eval_globals = { + "add": lambda x, y: x + y, + "sub": lambda x, y: x - y, + "mul": lambda x, y: x * y, + "div": lambda x, y: np.divide(x, y) if np.abs(y) > 0.001 else 1.0, + "sqrt": lambda x: np.sqrt(np.abs(x)), + "log": lambda x: np.log(np.abs(x)) if np.abs(x) > 0.001 else 0.0, + "inv": lambda x: 1.0 / x if np.abs(x) > 0.001 else 0.0, + } + + last_arg_index = 0 + for i in range(0, 100): + if function_str.find("X{:d}".format(i)) >= 0: + last_arg_index = i + + arg_list = [] + for i in range(0, last_arg_index + 1): + arg_list.append("X{:d}".format(i)) + + eval_str = "lambda {}, *whatever: {}".format(",".join(arg_list), function_str) + logger.debug(eval_str) + return eval(eval_str, eval_globals) + + class ParamFunction: """ A one-dimensional model function, ready for least squares optimization and similar. -- cgit v1.2.3