#!/usr/bin/env python3 """analyze-kconfig - Generate a model for KConfig selections analyze-kconfig builds a model determining system attributes (e.g. ROM or RAM usage) based on KConfig configuration variables. Only boolean variables are supported at the moment. """ import argparse import hashlib import json import kconfiglib import logging import os import sys import time import numpy as np import dfatool.cli import dfatool.plotter import dfatool.utils import dfatool.functions as df from dfatool.loader.kconfig import KConfigAttributes from dfatool.model import AnalyticModel from dfatool.validation import CrossValidator def write_csv(f, model, attr, precision=None): model_attr = model.attr_by_name[attr] attributes = sorted(model_attr.keys()) print(", ".join(model.parameters) + ", " + ", ".join(attributes), file=f) if precision is not None: data_wrapper = lambda x: f"{x:.{precision}f}" else: data_wrapper = str # by convention, model_attr[attr].param_values is the same regardless of 'attr' for param_tuple in model_attr[attributes[0]].param_values: param_data = map( lambda a: model_attr[a].by_param.get(tuple(param_tuple), list()), attributes ) print( ", ".join(map(str, param_tuple)) + ", " + ", ".join(map(data_wrapper, map(np.mean, param_data))), file=f, ) def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__ ) dfatool.cli.add_standard_arguments(parser) parser.add_argument( "--boolean-parameters", action="store_true", help="Use boolean (not categorial) parameters when building the NFP model", ) parser.add_argument( "--show-failing-symbols", action="store_true", help="Show Kconfig symbols related to build failures. Must be used with an experiment result directory.", ) parser.add_argument( "--show-nop-symbols", action="store_true", help="Show Kconfig symbols which are only present in a single configuration. Must be used with an experiment result directory.", ) parser.add_argument( "--force-tree", action="store_true", help="Build decision tree without checking for analytic functions first. Use this for large kconfig files.", ) parser.add_argument( "--skip-param-stats", action="store_true", help="Do not compute param stats that are required for RMT. Use this for large kconfig files.", ) parser.add_argument( "--max-std", type=str, metavar="VALUE_OR_MAP", help="Specify desired maximum standard deviation for RMT generation, either as float (global) or /=[,/=,...]. Has no effect when using CART, LMT or XGBoost.", ) parser.add_argument( "--csv-precision", type=int, metavar="NDIGITS", help="Precision (number of decimal digits) for CSV export", ) parser.add_argument( "--export-csv", type=str, metavar="FILE", help="Export observations aggregated by parameter to FILE", ) parser.add_argument( "--export-csv-only", action="store_true", help="Exit after exporting observations to CSV file", ) parser.add_argument( "--export-aggregate", type=str, metavar="FILE.json.xz", help="Export aggregated observations (intermediate and generic benchmark data representation) to FILE. Exported observations are affected by --param-shift and --ignore-param.", ) parser.add_argument( "--export-aggregate-only", action="store_true", help="Exit after exporting aggregated observations", ) parser.add_argument( "--export-observations", type=str, metavar="FILE.json.xz", help="Export observations (intermediate and generic benchmark data representation) to FILE", ) parser.add_argument( "--export-observations-only", action="store_true", help="Exit after exporting observations", ) parser.add_argument( "--export-webconf", type=str, help="Export kconfig-webconf NFP model to file", metavar="FILE", ) parser.add_argument( "--config", type=str, help="Show model results for symbols in .config file", metavar="FILE", ) parser.add_argument( "--sample-size", type=int, help="Restrict model generation to N random samples", metavar="N", ) parser.add_argument( "--show-quality", choices=["table"], action="append", default=list(), help="table: show LUT, model, and static prediction error for each key and attribute.", ) parser.add_argument("kconfig_path", type=str, help="Path to Kconfig file") parser.add_argument( "model", type=str, help="Path to experiment results directory or observations.json.xz file", ) args = parser.parse_args() dfatool.cli.sanity_check(args) if args.skip_param_stats and not args.force_tree: print("--skip-param-stats requires --force-tree", file=sys.stderr) sys.exit(1) if args.log_level: numeric_level = getattr(logging, args.log_level.upper(), None) if not isinstance(numeric_level, int): print(f"Invalid log level: {args.log_level}", file=sys.stderr) sys.exit(1) logging.basicConfig(level=numeric_level) if args.export_dref: dref = dict() if os.path.isdir(args.model): attributes = KConfigAttributes(args.kconfig_path, args.model) if args.export_dref: dref.update(attributes.to_dref()) if args.show_failing_symbols: show_failing_symbols(attributes) if args.show_nop_symbols: show_nop_symbols(attributes) observations = list() for param, attr in attributes.data: for key, value in attr.items(): observations.append( { "name": key, "param": param, "attribute": value, } ) if args.sample_size: shuffled_data_indices = np.random.permutation(np.arange(len(observations))) sample_indices = shuffled_data_indices[: args.sample_size] new_observations = list() for sample_index in sample_indices: new_observations.append(observations[sample_index]) observations = new_observations if args.export_observations: import lzma print( f"Exporting {len(observations)} observations to {args.export_observations}" ) with lzma.open(args.export_observations, "wt") as f: json.dump(observations, f) if args.export_observations_only: return else: # show-failing-symbols, show-nop-symbols, DFATOOL_KCONF_WITH_CHOICE_NODES have no effect # in this branch. if os.path.exists(args.kconfig_path): attributes = KConfigAttributes(args.kconfig_path, None) if args.export_dref: dref.update(attributes.to_dref()) if args.model.endswith("xz"): import lzma with lzma.open(args.model, "rt") as f: observations = json.load(f) elif args.model.endswith("ubjson"): import ubjson with open(args.model, "rb") as f: observations = ubjson.load(f) else: with open(args.model, "r") as f: observations = json.load(f) if bool(int(os.getenv("DFATOOL_KCONF_IGNORE_STRING", 0))) or bool( int(os.getenv("DFATOOL_KCONF_IGNORE_NUMERIC", 0)) ): attributes = KConfigAttributes(args.kconfig_path, None) if type(observations) is dict: ignore_index = dict() new_param_names = list() for i, param in enumerate(observations["param_names"]): if param in attributes.param_names: new_param_names.append(param) else: ignore_index[i] = True observations["param_names"] = new_param_names for data in observations["by_name"].values(): for i in range(len(data["param"])): for j in sorted(ignore_index.keys(), reverse=True): data["param"][i].pop(j) else: for observation in observations: to_remove = list() for param in observation["param"].keys(): if param not in attributes.param_names: to_remove.append(param) for param in to_remove: observation["param"].pop(param) if args.boolean_parameters: if type(observations) is list: logging.warning("--boolean-parameters is deprecated") dfatool.utils.observations_enum_to_bool(observations, kconfig=True) else: logging.error( "--boolean-parameters is only supported with legacy observations data" ) sys.exit(1) function_override = dict() if args.function_override: for function_desc in args.function_override.split(";"): state_or_tran, attribute, *function_str = function_desc.split(" ") function_override[(state_or_tran, attribute)] = " ".join(function_str) by_name, parameter_names = dfatool.utils.observations_to_by_name(observations) if args.ignore_param: args.ignore_param = args.ignore_param.split(",") dfatool.utils.ignore_param(by_name, parameter_names, args.ignore_param) if args.param_shift: param_shift = dfatool.cli.parse_param_shift(args.param_shift) dfatool.utils.shift_param_in_aggregate(by_name, parameter_names, param_shift) if args.normalize_nfp: norm = dfatool.cli.parse_nfp_normalization(args.normalize_nfp) dfatool.utils.normalize_nfp_in_aggregate(by_name, norm) if args.export_aggregate: import lzma print(f"Exporting aggregate to {args.export_aggregate}") with lzma.open(args.export_aggregate, "wt") as f: json.dump( {"by_name": by_name, "param_names": parameter_names}, f, cls=dfatool.utils.NpEncoder, ) if args.export_aggregate_only: return # Release memory del observations if args.filter_param: args.filter_param = list( map(lambda x: x.split("="), args.filter_param.split(",")) ) dfatool.utils.filter_aggregate_by_param( by_name, parameter_names, args.filter_param ) if args.max_std: max_std = dict() if "=" in args.max_std: for kkv in args.max_std.split(","): kk, v = kkv.split("=") key, attr = kk.split("/") if key not in max_std: max_std[key] = dict() max_std[key][attr] = float(v) else: for key in by_name.keys(): max_std[key] = dict() for attr in by_name[key]["attributes"]: max_std[key][attr] = float(args.max_std) else: max_std = None constructor_start = time.time() model = AnalyticModel( by_name, parameter_names, force_tree=args.force_tree, max_std=max_std, compute_stats=not args.skip_param_stats, function_override=function_override, ) constructor_duration = time.time() - constructor_start logging.debug(f"AnalyticModel(...) took {constructor_duration : 7.1f} seconds") if not model.names: logging.error( f"Model contains no names. Is --filter-param={args.filter_param} set too restrictive?" ) sys.exit(1) if args.info: dfatool.cli.print_info_by_name(model, by_name) if args.export_pgf_unparam: dfatool.cli.export_pgf_unparam(model, args.export_pgf_unparam) if args.export_json_unparam: dfatool.cli.export_json_unparam(model, args.export_json_unparam) if args.plot_unparam: for kv in args.plot_unparam.split(";"): state_or_trans, attribute, ylabel = kv.split(":") fname = "param_y_{}_{}.pdf".format(state_or_trans, attribute) dfatool.plotter.plot_y( model.by_name[state_or_trans][attribute], xlabel="measurement #", ylabel=ylabel, # output=fname, show=not args.non_interactive, ) if args.boxplot_unparam: title = None if args.filter_param: title = "filter: " + ", ".join( map(lambda kv: f"{kv[0]}={kv[1]}", args.filter_param) ) for name in model.names: attr_names = sorted(model.attributes(name)) dfatool.plotter.boxplot( attr_names, [model.by_name[name][attr] for attr in attr_names], xlabel="Attribute", output=f"{args.boxplot_unparam}{name}.pdf", title=title, show=not args.non_interactive, ) for attribute in attr_names: dfatool.plotter.boxplot( [attribute], [model.by_name[name][attribute]], output=f"{args.boxplot_unparam}{name}-{attribute}.pdf", title=title, show=not args.non_interactive, ) if args.boxplot_param: dfatool.cli.boxplot_param(args, model) if args.plot_param: for kv in args.plot_param.split(";"): try: state_or_trans, attribute, param_name, *function = kv.split(":") except ValueError: print( "Usage: --plot-param='state_or_trans attribute param_name [additional function spec]'", file=sys.stderr, ) sys.exit(1) if len(function): function = gplearn_to_function(" ".join(function)) else: function = None dfatool.plotter.plot_param( model, state_or_trans, attribute, model.param_index(param_name), extra_function=function, output=f"{state_or_trans}-{attribute}-{param_name}.pdf", show=not args.non_interactive, ) if args.cross_validate: xv_method, xv_count = args.cross_validate.split(":") xv_count = int(xv_count) xv = CrossValidator( AnalyticModel, by_name, parameter_names, force_tree=args.force_tree, max_std=max_std, compute_stats=not args.skip_param_stats, ) xv.parameter_aware = args.parameter_aware_cross_validation else: xv_method = None static_model = model.get_static() try: lut_model = model.get_param_lut() lut_quality = model.assess(lut_model) except RuntimeError as e: if args.force_tree: # this is to be expected logging.debug(f"Skipping LUT model: {e}") else: logging.warning(f"Skipping LUT model: {e}") lut_model = None lut_quality = None if args.export_csv: for name in model.names: target = f"{args.export_csv}-{name}.csv" print(f"Exporting aggregated data to {target}") with open(target, "w") as f: write_csv(f, model, name, args.csv_precision) if args.export_csv_only: return fit_start_time = time.time() param_model, param_info = model.get_fitted() fit_duration = time.time() - fit_start_time logging.debug(f"model.get_fitted(...) took {fit_duration : 7.1f} seconds") if xv_method == "montecarlo": static_quality, _ = xv.montecarlo( lambda m: m.get_static(), xv_count, static=True ) xv.export_filename = args.export_xv analytic_quality, xv_analytic_models = xv.montecarlo( lambda m: m.get_fitted()[0], xv_count ) elif xv_method == "kfold": static_quality, _ = xv.kfold(lambda m: m.get_static(), xv_count, static=True) xv.export_filename = args.export_xv analytic_quality, xv_analytic_models = xv.kfold( lambda m: m.get_fitted()[0], xv_count ) else: assess_start = time.time() static_quality = model.assess(static_model) assess_duration = time.time() - assess_start logging.debug(f"model.assess(static) took {assess_duration : 7.1f} seconds") if args.export_raw_predictions: analytic_quality, raw_results = model.assess(param_model, return_raw=True) with open(args.export_raw_predictions, "w") as f: json.dump(raw_results, f, cls=dfatool.utils.NpEncoder) else: assess_start = time.time() analytic_quality = model.assess(param_model) assess_duration = time.time() - assess_start logging.debug(f"model.assess(param) took {assess_duration : 7.1f} seconds") xv_analytic_models = None if lut_model: assess_start = time.time() lut_quality = model.assess(lut_model) assess_duration = time.time() - assess_start logging.debug(f"model.assess(lut) took {assess_duration : 7.1f} seconds") else: lut_quality = None if "static" in args.show_model or "all" in args.show_model: print("--- static model ---") for name in model.names: for attribute in model.attributes(name): dfatool.cli.print_static( model, static_model, name, attribute, with_dependence="all" in args.show_model, ) if "param" in args.show_model or "all" in args.show_model: print("--- param model ---") for name in model.names: for attribute in model.attributes(name): info = param_info(name, attribute) dfatool.cli.print_model( f"{name:20s} {attribute:15s}", info, model.parameters ) if "table" in args.show_quality or "all" in args.show_quality: if xv_method is not None: print( f"Model error ({args.error_metric}) after cross validation ({xv_method}, {xv_count}):" ) else: print(f"Model error ({args.error_metric}) on training data:") dfatool.cli.model_quality_table( lut=lut_quality, model=analytic_quality, static=static_quality, model_info=param_info, xv_method=xv_method, error_metric=args.error_metric, ) if not args.show_quality: if xv_method is not None: print(f"Model Error after Cross Validation ({xv_method}, {xv_count}):") else: print("Model Error on Training Data:") for name in sorted(model.names): for attribute, error in sorted( analytic_quality[name].items(), key=lambda kv: kv[0] ): mae = error["mae"] mape = error["mape"] smape = error["smape"] if mape is not None: print(f"{name:15s} {attribute:20s} ± {mae:10.2} / {mape:5.1f}%") else: print( f"{name:15s} {attribute:20s} ± {mae:10.2} / {smape:5.1f}% SMAPE" ) if args.show_model_size: dfatool.cli.print_model_size(model) if args.export_webconf: attributes = KConfigAttributes(args.kconfig_path, None) try: with open(f"{attributes.kconfig_root}/nfpkeys.json", "r") as f: nfpkeys = json.load(f) except FileNotFoundError: logging.error( f"{attributes.kconfig_root}/nfpkeys.json is missing, webconf model will be incomplete" ) nfpkeys = None kconfig_hasher = hashlib.sha256() with open(args.kconfig_path, "rb") as f: kconfig_data = f.read() while len(kconfig_data) > 0: kconfig_hasher.update(kconfig_data) kconfig_data = f.read() kconfig_hash = str(kconfig_hasher.hexdigest()) complete_json_model = model.to_json( with_param_name=True, param_names=parameter_names ) json_model = dict() for name, attribute_data in complete_json_model["name"].items(): for attribute, data in attribute_data.items(): json_model[attribute] = data.copy() if nfpkeys: json_model[attribute].update(nfpkeys[name][attribute]) out_model = { "model": json_model, "modelType": "dfatool-kconfig", "project": "tbd", "kconfigHash": kconfig_hash, "symbols": attributes.symbol_names, "choices": attributes.choice_names, } with open(args.export_webconf, "w") as f: json.dump(out_model, f, sort_keys=True, cls=dfatool.utils.NpEncoder) if args.export_dot: dfatool.cli.export_dot(model, args.export_dot) if args.export_dref: dref.update( model.to_dref( static_quality, lut_quality, analytic_quality, xv_models=xv_analytic_models, ) ) dref["constructor duration"] = (constructor_duration, r"\second") dref["regression duration"] = (fit_duration, r"\second") dfatool.cli.export_dataref( args.export_dref, dref, precision=args.dref_precision ) if args.export_json: with open(args.export_json, "w") as f: json.dump(model.to_json(), f, sort_keys=True, cls=dfatool.utils.NpEncoder) if args.config: kconf = kconfiglib.Kconfig(args.kconfig_path) kconf.load_config(args.config) print(f"Model result for .config: {model.value_for_config(kconf)}") for symbol in model.symbols: kconf2 = kconfiglib.Kconfig(args.kconfig_path) kconf2.load_config(args.config) kconf_sym = kconf2.syms[symbol] if kconf_sym.tri_value == 0 and 2 in kconf_sym.assignable: kconf_sym.set_value(2) elif kconf_sym.tri_value == 2 and 0 in kconf_sym.assignable: kconf_sym.set_value(0) else: continue # specific to multipass: # Do not suggest changes which affect the application skip = False num_changes = 0 changed_symbols = list() for i, csymbol in enumerate(model.symbols): if kconf.syms[csymbol].tri_value != kconf2.syms[csymbol].tri_value: num_changes += 1 changed_symbols.append(csymbol) if ( csymbol.startswith("app_") and kconf.syms[csymbol].tri_value != kconf2.syms[csymbol].tri_value ): skip = True break if skip: continue try: model_diff = model.value_for_config(kconf2) - model.value_for_config( kconf ) if kconf_sym.choice: print( f"Setting {kconf_sym.choice.name} to {kconf_sym.name} changes {num_changes:2d} symbols, model change: {model_diff:+5.0f}" ) else: print( f"Setting {symbol} to {kconf_sym.str_value} changes {num_changes:2d} symbols, model change: {model_diff:+5.0f}" ) except TypeError: if kconf_sym.choice: print( f"Setting {kconf_sym.choice.name} to {kconf_sym.name} changes {num_changes:2d} symbols, model is undefined" ) else: print( f"Setting {symbol} to {kconf_sym.str_value} changes {num_changes:2d} symbols, model is undefined" ) for changed_symbol in changed_symbols: print( f" {changed_symbol:30s} -> {kconf2.syms[changed_symbol].str_value}" ) def show_failing_symbols(data): for symbol in data.param_names: unique_values = list(set(map(lambda p: p[symbol], data.failures))) for value in unique_values: fail_count = len(list(filter(lambda p: p[symbol] == value, data.failures))) success_count = len( list(filter(lambda p: p[0][symbol] == value, data.data)) ) if success_count == 0 and fail_count > 0: print( f"Setting {symbol} to '{value}' reliably causes the build to fail (count = {fail_count})" ) def show_nop_symbols(data): for symbol in data.symbol_names: true_count = len( list(filter(lambda config: config[symbol] == True, data.failures)) ) + len(list(filter(lambda config: config[0][symbol] == True, data.data))) false_count = len( list(filter(lambda config: config[symbol] == False, data.failures)) ) + len(list(filter(lambda config: config[0][symbol] == False, data.data))) if false_count == 0: print(f"Symbol {symbol} is never n") if true_count == 0: print(f"Symbol {symbol} is never y") pass if __name__ == "__main__": main()