path: root/bin/analyze-kconfig.py

#!/usr/bin/env python3

"""analyze-kconfig - Generate a model for KConfig selections

analyze-kconfig builds a model determining system attributes
(e.g. ROM or RAM usage) based on KConfig configuration variables.
Only boolean variables are supported at the moment.
"""

import argparse
import hashlib
import json
import kconfiglib
import logging
import os
import sys
import time

import numpy as np

import dfatool.cli
import dfatool.plotter
import dfatool.utils
import dfatool.functions as df
from dfatool.loader.kconfig import KConfigAttributes
from dfatool.model import AnalyticModel
from dfatool.validation import CrossValidator


def write_csv(f, model, attr, precision=None):
    model_attr = model.attr_by_name[attr]
    attributes = sorted(model_attr.keys())
    print(", ".join(model.parameters) + ",     " + ", ".join(attributes), file=f)

    if precision is not None:
        data_wrapper = lambda x: f"{x:.{precision}f}"
    else:
        data_wrapper = str

    # by convention, model_attr[attr].param_values is the same regardless of 'attr'
    for param_tuple in model_attr[attributes[0]].param_values:
        param_data = map(
            lambda a: model_attr[a].by_param.get(tuple(param_tuple), list()), attributes
        )
        print(
            ", ".join(map(str, param_tuple))
            + ",     "
            + ", ".join(map(data_wrapper, map(np.mean, param_data))),
            file=f,
        )


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__
    )
    dfatool.cli.add_standard_arguments(parser)
    parser.add_argument(
        "--boolean-parameters",
        action="store_true",
        help="Use boolean (not categorial) parameters when building the NFP model",
    )
    parser.add_argument(
        "--show-failing-symbols",
        action="store_true",
        help="Show Kconfig symbols related to build failures. Must be used with an experiment result directory.",
    )
    parser.add_argument(
        "--show-nop-symbols",
        action="store_true",
        help="Show Kconfig symbols which are only present in a single configuration. Must be used with an experiment result directory.",
    )
    parser.add_argument(
        "--force-tree",
        action="store_true",
        help="Build decision tree without checking for analytic functions first. Use this for large kconfig files.",
    )
    parser.add_argument(
        "--skip-param-stats",
        action="store_true",
        help="Do not compute param stats that are required for RMT. Use this for large kconfig files.",
    )
    parser.add_argument(
        "--max-std",
        type=str,
        metavar="VALUE_OR_MAP",
        help="Specify desired maximum standard deviation for RMT generation, either as float (global) or <key>/<attribute>=<value>[,<key>/<attribute>=<value>,...]. Has no effect when using CART, LMT or XGBoost.",
    )
    parser.add_argument(
        "--csv-precision",
        type=int,
        metavar="NDIGITS",
        help="Precision (number of decimal digits) for CSV export",
    )
    parser.add_argument(
        "--export-csv",
        type=str,
        metavar="FILE",
        help="Export observations aggregated by parameter to FILE",
    )
    parser.add_argument(
        "--export-csv-only",
        action="store_true",
        help="Exit after exporting observations to CSV file",
    )
    parser.add_argument(
        "--export-aggregate",
        type=str,
        metavar="FILE.json.xz",
        help="Export aggregated observations (intermediate and generic benchmark data representation) to FILE. Exported observations are affected by --param-shift and --ignore-param.",
    )
    parser.add_argument(
        "--export-aggregate-only",
        action="store_true",
        help="Exit after exporting aggregated observations",
    )
    parser.add_argument(
        "--export-observations",
        type=str,
        metavar="FILE.json.xz",
        help="Export observations (intermediate and generic benchmark data representation) to FILE",
    )
    parser.add_argument(
        "--export-observations-only",
        action="store_true",
        help="Exit after exporting observations",
    )
    parser.add_argument(
        "--export-webconf",
        type=str,
        help="Export kconfig-webconf NFP model to file",
        metavar="FILE",
    )
    parser.add_argument(
        "--config",
        type=str,
        help="Show model results for symbols in .config file",
        metavar="FILE",
    )
    parser.add_argument(
        "--sample-size",
        type=int,
        help="Restrict model generation to N random samples",
        metavar="N",
    )
    parser.add_argument(
        "--show-quality",
        choices=["table"],
        action="append",
        default=list(),
        help="table: show LUT, model, and static prediction error for each key and attribute.",
    )
    parser.add_argument(
        "--plot-param",
        metavar="<name>:<attribute>:<parameter>[:gplearn function][;<name>:<attribute>:<parameter>:[function];...])",
        type=str,
        help="Plot measurements for <name> <attribute> by <parameter>. "
        "X axis is parameter value. "
        "Plots the model function as one solid line for each combination of non-<parameter> parameters. "
        "Also plots the corresponding measurements. "
        "If gplearn function is set, it is plotted using dashed lines.",
    )
    parser.add_argument("kconfig_path", type=str, help="Path to Kconfig file")
    parser.add_argument(
        "model",
        type=str,
        help="Path to experiment results directory or observations.json.xz file",
    )

    args = parser.parse_args()

    if args.skip_param_stats and not args.force_tree:
        print("--skip-param-stats requires --force-tree", file=sys.stderr)
        sys.exit(1)

    if args.log_level:
        numeric_level = getattr(logging, args.log_level.upper(), None)
        if not isinstance(numeric_level, int):
            print(f"Invalid log level: {args.log_level}", file=sys.stderr)
            sys.exit(1)
        logging.basicConfig(level=numeric_level)

    if args.export_dref:
        dref = dict()

    if os.path.isdir(args.model):
        attributes = KConfigAttributes(args.kconfig_path, args.model)
        if args.export_dref:
            dref.update(attributes.to_dref())

        if args.show_failing_symbols:
            show_failing_symbols(attributes)
        if args.show_nop_symbols:
            show_nop_symbols(attributes)

        observations = list()

        for param, attr in attributes.data:
            for key, value in attr.items():
                observations.append(
                    {
                        "name": key,
                        "param": param,
                        "attribute": value,
                    }
                )

        if args.sample_size:
            shuffled_data_indices = np.random.permutation(np.arange(len(observations)))
            sample_indices = shuffled_data_indices[: args.sample_size]
            new_observations = list()
            for sample_index in sample_indices:
                new_observations.append(observations[sample_index])
            observations = new_observations

        if args.export_observations:
            import lzma

            print(
                f"Exporting {len(observations)} observations to {args.export_observations}"
            )
            with lzma.open(args.export_observations, "wt") as f:
                json.dump(observations, f)
            if args.export_observations_only:
                return
    else:
        # show-failing-symbols, show-nop-symbols, DFATOOL_KCONF_WITH_CHOICE_NODES have no effect
        # in this branch.

        if os.path.exists(args.kconfig_path):
            attributes = KConfigAttributes(args.kconfig_path, None)
            if args.export_dref:
                dref.update(attributes.to_dref())

        if args.model.endswith("xz"):
            import lzma

            with lzma.open(args.model, "rt") as f:
                observations = json.load(f)
        elif args.model.endswith("ubjson"):
            import ubjson

            with open(args.model, "rb") as f:
                observations = ubjson.load(f)
        else:
            with open(args.model, "r") as f:
                observations = json.load(f)

        if bool(int(os.getenv("DFATOOL_KCONF_IGNORE_STRING", 0))) or bool(
            int(os.getenv("DFATOOL_KCONF_IGNORE_NUMERIC", 0))
        ):
            attributes = KConfigAttributes(args.kconfig_path, None)
            if type(observations) is dict:
                ignore_index = dict()
                new_param_names = list()
                for i, param in enumerate(observations["param_names"]):
                    if param in attributes.param_names:
                        new_param_names.append(param)
                    else:
                        ignore_index[i] = True
                observations["param_names"] = new_param_names
                for data in observations["by_name"].values():
                    for i in range(len(data["param"])):
                        for j in sorted(ignore_index.keys(), reverse=True):
                            data["param"][i].pop(j)
            else:
                for observation in observations:
                    to_remove = list()
                    for param in observation["param"].keys():
                        if param not in attributes.param_names:
                            to_remove.append(param)
                    for param in to_remove:
                        observation["param"].pop(param)

    if args.boolean_parameters:
        if type(observations) is list:
            logging.warning("--boolean-parameters is deprecated")
            dfatool.utils.observations_enum_to_bool(observations, kconfig=True)
        else:
            logging.error(
                "--boolean-parameters is only supported with legacy observations data"
            )
            sys.exit(1)

    function_override = dict()
    if args.function_override:
        for function_desc in args.function_override.split(";"):
            state_or_tran, attribute, *function_str = function_desc.split(" ")
            function_override[(state_or_tran, attribute)] = " ".join(function_str)

    by_name, parameter_names = dfatool.utils.observations_to_by_name(observations)

    if args.ignore_param:
        args.ignore_param = args.ignore_param.split(",")
        dfatool.utils.ignore_param(by_name, parameter_names, args.ignore_param)

    if args.param_shift:
        param_shift = dfatool.cli.parse_param_shift(args.param_shift)
        dfatool.utils.shift_param_in_aggregate(by_name, parameter_names, param_shift)

    if args.normalize_nfp:
        norm = dfatool.cli.parse_nfp_normalization(args.normalize_nfp)
        dfatool.utils.normalize_nfp_in_aggregate(by_name, norm)

    if args.export_aggregate:
        import lzma

        print(f"Exporting aggregate to {args.export_aggregate}")
        with lzma.open(args.export_aggregate, "wt") as f:
            json.dump(
                {"by_name": by_name, "param_names": parameter_names},
                f,
                cls=dfatool.utils.NpEncoder,
            )
        if args.export_aggregate_only:
            return

    # Release memory
    del observations

    if args.filter_param:
        args.filter_param = list(
            map(lambda x: x.split("="), args.filter_param.split(","))
        )
        dfatool.utils.filter_aggregate_by_param(
            by_name, parameter_names, args.filter_param
        )

    if args.max_std:
        max_std = dict()
        if "=" in args.max_std:
            for kkv in args.max_std.split(","):
                kk, v = kkv.split("=")
                key, attr = kk.split("/")
                if key not in max_std:
                    max_std[key] = dict()
                max_std[key][attr] = float(v)
        else:
            for key in by_name.keys():
                max_std[key] = dict()
                for attr in by_name[key]["attributes"]:
                    max_std[key][attr] = float(args.max_std)
    else:
        max_std = None

    constructor_start = time.time()
    model = AnalyticModel(
        by_name,
        parameter_names,
        force_tree=args.force_tree,
        max_std=max_std,
        compute_stats=not args.skip_param_stats,
        function_override=function_override,
    )
    constructor_duration = time.time() - constructor_start
    logging.debug(f"AnalyticModel(...) took {constructor_duration : 7.1f} seconds")

    if not model.names:
        logging.error(
            f"Model contains no names. Is --filter-param={args.filter_param} set too restrictive?"
        )
        sys.exit(1)

    if args.info:
        dfatool.cli.print_info_by_name(model, by_name)

    if args.export_pgf_unparam:
        dfatool.cli.export_pgf_unparam(model, args.export_pgf_unparam)

    if args.export_json_unparam:
        dfatool.cli.export_json_unparam(model, args.export_json_unparam)

    if args.plot_unparam:
        for kv in args.plot_unparam.split(";"):
            state_or_trans, attribute, ylabel = kv.split(":")
            fname = "param_y_{}_{}.pdf".format(state_or_trans, attribute)
            dfatool.plotter.plot_y(
                model.by_name[state_or_trans][attribute],
                xlabel="measurement #",
                ylabel=ylabel,
                # output=fname,
                show=not args.non_interactive,
            )

    if args.boxplot_unparam:
        title = None
        if args.filter_param:
            title = "filter: " + ", ".join(
                map(lambda kv: f"{kv[0]}={kv[1]}", args.filter_param)
            )
        for name in model.names:
            attr_names = sorted(model.attributes(name))
            dfatool.plotter.boxplot(
                attr_names,
                [model.by_name[name][attr] for attr in attr_names],
                xlabel="Attribute",
                output=f"{args.boxplot_unparam}{name}.pdf",
                title=title,
                show=not args.non_interactive,
            )
            for attribute in attr_names:
                dfatool.plotter.boxplot(
                    [attribute],
                    [model.by_name[name][attribute]],
                    output=f"{args.boxplot_unparam}{name}-{attribute}.pdf",
                    title=title,
                    show=not args.non_interactive,
                )

    if args.boxplot_param:
        dfatool.cli.boxplot_param(args, model)

    if args.plot_param:
        for kv in args.plot_param.split(";"):
            try:
                state_or_trans, attribute, param_name, *function = kv.split(":")
            except ValueError:
                print(
                    "Usage: --plot-param='state_or_trans attribute param_name [additional function spec]'",
                    file=sys.stderr,
                )
                sys.exit(1)
            if len(function):
                function = gplearn_to_function(" ".join(function))
            else:
                function = None
            dfatool.plotter.plot_param(
                model,
                state_or_trans,
                attribute,
                model.param_index(param_name),
                extra_function=function,
            )

    if args.cross_validate:
        xv_method, xv_count = args.cross_validate.split(":")
        xv_count = int(xv_count)
        xv = CrossValidator(
            AnalyticModel,
            by_name,
            parameter_names,
            force_tree=args.force_tree,
            max_std=max_std,
            compute_stats=not args.skip_param_stats,
        )
        xv.parameter_aware = args.parameter_aware_cross_validation
    else:
        xv_method = None

    static_model = model.get_static()
    try:
        lut_model = model.get_param_lut()
        lut_quality = model.assess(lut_model)
    except RuntimeError as e:
        if args.force_tree:
            # this is to be expected
            logging.debug(f"Skipping LUT model: {e}")
        else:
            logging.warning(f"Skipping LUT model: {e}")
        lut_model = None
        lut_quality = None

    if args.export_csv:
        for name in model.names:
            target = f"{args.export_csv}-{name}.csv"
            print(f"Exporting aggregated data to {target}")
            with open(target, "w") as f:
                write_csv(f, model, name, args.csv_precision)
        if args.export_csv_only:
            return

    fit_start_time = time.time()
    param_model, param_info = model.get_fitted()
    fit_duration = time.time() - fit_start_time
    logging.debug(f"model.get_fitted(...) took {fit_duration : 7.1f} seconds")

    if xv_method == "montecarlo":
        static_quality, _ = xv.montecarlo(
            lambda m: m.get_static(), xv_count, static=True
        )
        xv.export_filename = args.export_xv
        analytic_quality, xv_analytic_models = xv.montecarlo(
            lambda m: m.get_fitted()[0], xv_count
        )
    elif xv_method == "kfold":
        static_quality, _ = xv.kfold(lambda m: m.get_static(), xv_count, static=True)
        xv.export_filename = args.export_xv
        analytic_quality, xv_analytic_models = xv.kfold(
            lambda m: m.get_fitted()[0], xv_count
        )
    else:
        assess_start = time.time()
        static_quality = model.assess(static_model)
        assess_duration = time.time() - assess_start
        logging.debug(f"model.assess(static) took {assess_duration : 7.1f} seconds")
        if args.export_raw_predictions:
            analytic_quality, raw_results = model.assess(param_model, return_raw=True)
            with open(args.export_raw_predictions, "w") as f:
                json.dump(raw_results, f, cls=dfatool.utils.NpEncoder)
        else:
            assess_start = time.time()
            analytic_quality = model.assess(param_model)
            assess_duration = time.time() - assess_start
            logging.debug(f"model.assess(param) took {assess_duration : 7.1f} seconds")
        xv_analytic_models = None
        if lut_model:
            assess_start = time.time()
            lut_quality = model.assess(lut_model)
            assess_duration = time.time() - assess_start
            logging.debug(f"model.assess(lut) took {assess_duration : 7.1f} seconds")
        else:
            lut_quality = None

    if "static" in args.show_model or "all" in args.show_model:
        print("--- static model ---")
        for name in model.names:
            for attribute in model.attributes(name):
                dfatool.cli.print_static(
                    model,
                    static_model,
                    name,
                    attribute,
                    with_dependence="all" in args.show_model,
                )

    if "param" in args.show_model or "all" in args.show_model:
        print("--- param model ---")
        for name in model.names:
            for attribute in model.attributes(name):
                info = param_info(name, attribute)
                if type(info) is df.AnalyticFunction:
                    dfatool.cli.print_analyticinfo(f"{name:20s} {attribute:15s}", info)
                elif type(info) is df.CARTFunction:
                    dfatool.cli.print_cartinfo(
                        f"{name:20s} {attribute:15s}", info, model.parameters
                    )
                elif type(info) is df.FOLFunction:
                    dfatool.cli.print_analyticinfo(f"{name:20s} {attribute:15s}", info)
                elif type(info) is df.SplitFunction:
                    dfatool.cli.print_splitinfo(
                        model.parameters, info, f"{name:20s} {attribute:15s}"
                    )
                elif type(info) is df.StaticFunction:
                    dfatool.cli.print_staticinfo(f"{name:10s} {attribute:15s}", info)

    if "table" in args.show_quality or "all" in args.show_quality:
        if xv_method is not None:
            print(
                f"Model error ({args.error_metric}) after cross validation ({xv_method}, {xv_count}):"
            )
        else:
            print(f"Model error ({args.error_metric}) on training data:")
        dfatool.cli.model_quality_table(
            lut=lut_quality,
            model=analytic_quality,
            static=static_quality,
            model_info=param_info,
            xv_method=xv_method,
            error_metric=args.error_metric,
        )

    if not args.show_quality:
        if xv_method is not None:
            print(f"Model Error after Cross Validation ({xv_method}, {xv_count}):")
        else:
            print("Model Error on Training Data:")
        for name in sorted(model.names):
            for attribute, error in sorted(
                analytic_quality[name].items(), key=lambda kv: kv[0]
            ):
                mae = error["mae"]
                mape = error["mape"]
                smape = error["smape"]
                if mape is not None:
                    print(f"{name:15s} {attribute:20s}  ± {mae:10.2}  /  {mape:5.1f}%")
                else:
                    print(
                        f"{name:15s} {attribute:20s}  ± {mae:10.2}  /  {smape:5.1f}% SMAPE"
                    )

    if args.show_model_size:
        dfatool.cli.print_model_size(model)

    if args.export_webconf:
        attributes = KConfigAttributes(args.kconfig_path, None)
        try:
            with open(f"{attributes.kconfig_root}/nfpkeys.json", "r") as f:
                nfpkeys = json.load(f)
        except FileNotFoundError:
            logging.error(
                f"{attributes.kconfig_root}/nfpkeys.json is missing, webconf model will be incomplete"
            )
            nfpkeys = None
        kconfig_hasher = hashlib.sha256()
        with open(args.kconfig_path, "rb") as f:
            kconfig_data = f.read()
            while len(kconfig_data) > 0:
                kconfig_hasher.update(kconfig_data)
                kconfig_data = f.read()
        kconfig_hash = str(kconfig_hasher.hexdigest())
        complete_json_model = model.to_json(
            with_param_name=True, param_names=parameter_names
        )
        json_model = dict()
        for name, attribute_data in complete_json_model["name"].items():
            for attribute, data in attribute_data.items():
                json_model[attribute] = data.copy()
                if nfpkeys:
                    json_model[attribute].update(nfpkeys[name][attribute])
        out_model = {
            "model": json_model,
            "modelType": "dfatool-kconfig",
            "project": "tbd",
            "kconfigHash": kconfig_hash,
            "symbols": attributes.symbol_names,
            "choices": attributes.choice_names,
        }
        with open(args.export_webconf, "w") as f:
            json.dump(out_model, f, sort_keys=True, cls=dfatool.utils.NpEncoder)

    if args.export_dot:
        dfatool.cli.export_dot(model, args.export_dot)

    if args.export_dref:
        dref.update(
            model.to_dref(
                static_quality,
                lut_quality,
                analytic_quality,
                xv_models=xv_analytic_models,
            )
        )
        dref["constructor duration"] = (constructor_duration, r"\second")
        dref["regression duration"] = (fit_duration, r"\second")
        dfatool.cli.export_dataref(
            args.export_dref, dref, precision=args.dref_precision
        )

    if args.config:
        kconf = kconfiglib.Kconfig(args.kconfig_path)
        kconf.load_config(args.config)
        print(f"Model result for .config: {model.value_for_config(kconf)}")

        for symbol in model.symbols:
            kconf2 = kconfiglib.Kconfig(args.kconfig_path)
            kconf2.load_config(args.config)
            kconf_sym = kconf2.syms[symbol]
            if kconf_sym.tri_value == 0 and 2 in kconf_sym.assignable:
                kconf_sym.set_value(2)
            elif kconf_sym.tri_value == 2 and 0 in kconf_sym.assignable:
                kconf_sym.set_value(0)
            else:
                continue

            # specific to multipass:
            # Do not suggest changes which affect the application
            skip = False
            num_changes = 0
            changed_symbols = list()
            for i, csymbol in enumerate(model.symbols):
                if kconf.syms[csymbol].tri_value != kconf2.syms[csymbol].tri_value:
                    num_changes += 1
                    changed_symbols.append(csymbol)
                    if (
                        csymbol.startswith("app_")
                        and kconf.syms[csymbol].tri_value
                        != kconf2.syms[csymbol].tri_value
                    ):
                        skip = True
                        break
            if skip:
                continue

            try:
                model_diff = model.value_for_config(kconf2) - model.value_for_config(
                    kconf
                )
                if kconf_sym.choice:
                    print(
                        f"Setting {kconf_sym.choice.name} to {kconf_sym.name} changes {num_changes:2d} symbols, model change: {model_diff:+5.0f}"
                    )
                else:
                    print(
                        f"Setting {symbol} to {kconf_sym.str_value} changes {num_changes:2d} symbols, model change: {model_diff:+5.0f}"
                    )
            except TypeError:
                if kconf_sym.choice:
                    print(
                        f"Setting {kconf_sym.choice.name} to {kconf_sym.name} changes {num_changes:2d} symbols, model is undefined"
                    )
                else:
                    print(
                        f"Setting {symbol} to {kconf_sym.str_value} changes {num_changes:2d} symbols, model is undefined"
                    )
            for changed_symbol in changed_symbols:
                print(
                    f"    {changed_symbol:30s} -> {kconf2.syms[changed_symbol].str_value}"
                )


def show_failing_symbols(data):
    for symbol in data.param_names:
        unique_values = list(set(map(lambda p: p[symbol], data.failures)))
        for value in unique_values:
            fail_count = len(list(filter(lambda p: p[symbol] == value, data.failures)))
            success_count = len(
                list(filter(lambda p: p[0][symbol] == value, data.data))
            )
            if success_count == 0 and fail_count > 0:
                print(
                    f"Setting {symbol} to '{value}' reliably causes the build to fail (count = {fail_count})"
                )


def show_nop_symbols(data):
    for symbol in data.symbol_names:
        true_count = len(
            list(filter(lambda config: config[symbol] == True, data.failures))
        ) + len(list(filter(lambda config: config[0][symbol] == True, data.data)))
        false_count = len(
            list(filter(lambda config: config[symbol] == False, data.failures))
        ) + len(list(filter(lambda config: config[0][symbol] == False, data.data)))
        if false_count == 0:
            print(f"Symbol {symbol} is never n")
        if true_count == 0:
            print(f"Symbol {symbol} is never y")
    pass


if __name__ == "__main__":
    main()