#!/usr/bin/env python3

"""eval-kconfig - tbd
"""

import argparse
import json
import kconfiglib
import logging
import os
import sys

import numpy as np

from dfatool import kconfig, validation
from dfatool.loader import KConfigAttributes
from dfatool.model import KConfigModel

from versuchung.experiment import Experiment
from versuchung.types import String, Bool, Integer
from versuchung.files import File, Directory


def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__
    )
    parser.add_argument(
        "--log-level",
        default=logging.INFO,
        type=lambda level: getattr(logging, level.upper()),
        help="Set log level",
    )
    parser.add_argument(
        "--attribute", choices=["rom", "ram"], default="rom", help="Model attribute"
    )
    parser.add_argument(
        "--with-choice-node", action="store_true", help="Use non-binary Choice Nodes"
    )
    parser.add_argument(
        "--max-loss",
        type=float,
        help="Maximum acceptable model loss for DecisionTree Leaves",
        default=10,
    )
    # Falls die population exhaustive ist, kann man nun den generalization error berechnen
    parser.add_argument(
        "--sample-size",
        type=int,
        help="Perform model generation and validation with N random samples from the population",
        metavar="N",
    )
    parser.add_argument("kconfig_path", type=str, help="Path to Kconfig file")
    parser.add_argument(
        "experiment_root", type=str, help="Experiment results directory"
    )
    parser.add_argument("model", type=str, help="JSON model", nargs="?")

    args = parser.parse_args()

    if isinstance(args.log_level, int):
        logging.basicConfig(level=args.log_level)
    else:
        print(f"Invalid log level. Setting log level to INFO.", file=sys.stderr)

    data = KConfigAttributes(args.kconfig_path, args.experiment_root)

    k = 10

    if args.sample_size:
        shuffled_data_indices = np.random.permutation(np.arange(len(data.data)))
        sample_indices = shuffled_data_indices[: args.sample_size]
        nonsample_indices = shuffled_data_indices[args.sample_size :]
        partition_pairs = validation._xv_partitions_kfold(args.sample_size, k)
        partition_pairs = list(
            map(
                lambda tv: (shuffled_data_indices[tv[0]], shuffled_data_indices[tv[1]]),
                partition_pairs,
            )
        )
    else:
        partition_pairs = validation._xv_partitions_kfold(len(data.data), k)

    measures = list()
    for training_set, validation_set in partition_pairs:
        model = KConfigModel.from_benchmark(data, args.attribute, indices=training_set)
        model.with_choice_node = args.with_choice_node
        if args.max_loss:
            model.max_loss = args.max_loss
        model.build_tree()
        measures.append(model.assess_benchmark(data, indices=validation_set))

    aggregate = dict()
    for measure in measures[0].keys():
        aggregate[measure] = np.mean(list(map(lambda m: m[measure], measures)))
    aggregate["unpredictable_count"] = np.sum(
        list(map(lambda m: m["unpredictable_count"], measures))
    )

    print("10-fold Cross Validation:")
    print(f"MAE: {aggregate['mae']:.0f} B")
    print(f"SMAPE: {aggregate['smape']:.1f} %")
    print(f"Unpredictable Configurations: {aggregate['unpredictable_count']}")

    print(aggregate)

    if args.sample_size:
        print("Estimated Generalization Error")
        model = KConfigModel.from_benchmark(
            data, args.attribute, indices=sample_indices
        )
        model.with_choice_node = args.with_choice_node
        if args.max_loss:
            model.max_loss = args.max_loss
        model.build_tree()
        generalization_measure = model.assess_benchmark(data, indices=nonsample_indices)
        print(f"MAE: {generalization_measure['mae']:.0f} B")
        print(f"SMAPE: {generalization_measure['smape']:.1f} %")
        print(
            f"Unpredictable Configurations: {generalization_measure['unpredictable_count']}"
        )

    """
    if args.model:
        with open(args.model, "r") as f:
            model = KConfigModel.from_json(json.load(f))
    else:
        model = KConfigModel.from_benchmark(data, args.attribute)
        model.build_tree()
    """


if __name__ == "__main__":
    main()