#!/usr/bin/env python3
# Copyright (c) 2021 Marco Cerliani, MIT License <https://github.com/cerlymarco/linear-tree>
import numpy as np
from sklearn.base import ClassifierMixin, RegressorMixin
from sklearn.utils import check_array
from sklearn.utils.validation import check_is_fitted, _check_sample_weight
from ._classes import _predict_branch
from ._classes import _LinearTree, _LinearBoosting, _LinearForest
class LinearTreeRegressor(_LinearTree, RegressorMixin):
"""A Linear Tree Regressor.
A Linear Tree Regressor is a meta-estimator that combine the learning
ability of Decision Tree and the predictive power of Linear Models.
Like in tree-based algorithms, the received data are splitted according
simple decision rules. The goodness of slits is evaluated in gain terms
fitting linear models in each node. This implies that the models in the
leaves are linear instead of constant approximations like in classical
Decision Tree.
Parameters
----------
base_estimator : object
The base estimator to fit on dataset splits.
The base estimator must be a sklearn.linear_model.
criterion : {"mse", "rmse", "mae", "poisson"}, default="mse"
The function to measure the quality of a split. "poisson"
requires ``y >= 0``.
max_depth : int, default=5
The maximum depth of the tree considering only the splitting nodes.
A higher value implies a higher training time.
min_samples_split : int or float, default=6
The minimum number of samples required to split an internal node.
The minimum valid number of samples in each node is 6.
A lower value implies a higher training time.
- If int, then consider `min_samples_split` as the minimum number.
- If float, then `min_samples_split` is a fraction and
`ceil(min_samples_split * n_samples)` are the minimum
number of samples for each split.
min_samples_leaf : int or float, default=0.1
The minimum number of samples required to be at a leaf node.
A split point at any depth will only be considered if it leaves at
least `min_samples_leaf` training samples in each of the left and
right branches.
The minimum valid number of samples in each leaf is 3.
A lower value implies a higher training time.
- If int, then consider `min_samples_leaf` as the minimum number.
- If float, then `min_samples_leaf` is a fraction and
`ceil(min_samples_leaf * n_samples)` are the minimum
number of samples for each node.
max_bins : int, default=25
The maximum number of bins to use to search the optimal split in each
feature. Features with a small number of unique values may use less than
``max_bins`` bins. Must be lower than 120 and larger than 10.
A higher value implies a higher training time.
categorical_features : int or array-like of int, default=None
Indicates the categorical features.
All categorical indices must be in `[0, n_features)`.
Categorical features are used for splits but are not used in
model fitting.
More categorical features imply a higher training time.
- None : no feature will be considered categorical.
- integer array-like : integer indices indicating categorical
features.
- integer : integer index indicating a categorical
feature.
split_features : int or array-like of int, default=None
Defines which features can be used to split on.
All split feature indices must be in `[0, n_features)`.
- None : All features will be used for splitting.
- integer array-like : integer indices indicating splitting features.
- integer : integer index indicating a single splitting feature.
linear_features : int or array-like of int, default=None
Defines which features are used for the linear model in the leaves.
All linear feature indices must be in `[0, n_features)`.
- None : All features except those in `categorical_features`
will be used in the leaf models.
- integer array-like : integer indices indicating features to
be used in the leaf models.
- integer : integer index indicating a single feature to be used
in the leaf models.
n_jobs : int, default=None
The number of jobs to run in parallel for model fitting.
``None`` means 1 using one processor. ``-1`` means using all
processors.
Attributes
----------
n_features_in_ : int
The number of features when :meth:`fit` is performed.
feature_importances_ : ndarray of shape (n_features, )
Normalized total reduction of criteria by splitting features.
n_targets_ : int
The number of targets when :meth:`fit` is performed.
Examples
--------
>>> from sklearn.linear_model import LinearRegression
>>> from lineartree import LinearTreeRegressor
>>> from sklearn.datasets import make_regression
>>> X, y = make_regression(n_samples=100, n_features=4,
... n_informative=2, n_targets=1,
... random_state=0, shuffle=False)
>>> regr = LinearTreeRegressor(base_estimator=LinearRegression())
>>> regr.fit(X, y)
>>> regr.predict([[0, 0, 0, 0]])
array([8.8817842e-16])
"""
def __init__(
self,
base_estimator,
*,
criterion="mse",
max_depth=5,
min_samples_split=6,
min_samples_leaf=0.1,
max_bins=25,
categorical_features=None,
split_features=None,
linear_features=None,
n_jobs=None
):
self.base_estimator = base_estimator
self.criterion = criterion
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.max_bins = max_bins
self.categorical_features = categorical_features
self.split_features = split_features
self.linear_features = linear_features
self.n_jobs = n_jobs
def fit(self, X, y, sample_weight=None):
"""Build a Linear Tree of a linear estimator from the training
set (X, y).
Parameters
----------
X : array-like of shape (n_samples, n_features)
The training input samples.
y : array-like of shape (n_samples, ) or (n_samples, n_targets)
Target values.
sample_weight : array-like of shape (n_samples, ), default=None
Sample weights. If None, then samples are equally weighted.
Note that if the base estimator does not support sample weighting,
the sample weights are still used to evaluate the splits.
Returns
-------
self : object
"""
reg_criterions = ("mse", "rmse", "mae", "poisson")
if self.criterion not in reg_criterions:
raise ValueError(
"Regression tasks support only criterion in {}, "
"got '{}'.".format(reg_criterions, self.criterion)
)
# Convert data (X is required to be 2d and indexable)
X, y = self._validate_data(
X,
y,
accept_sparse=False,
dtype=None,
force_all_finite=False,
multi_output=True,
)
if sample_weight is not None:
sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
y_shape = np.shape(y)
self.n_targets_ = y_shape[1] if len(y_shape) > 1 else 1
if self.n_targets_ < 2:
y = y.ravel()
self._fit(X, y, sample_weight)
return self
def predict(self, X):
"""Predict regression target for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, ) or also (n_samples, n_targets) if
multitarget regression.
The predicted values.
"""
check_is_fitted(self, attributes="_nodes")
X = check_array(X, accept_sparse=False, dtype=None, force_all_finite=False)
self._check_n_features(X, reset=False)
if self.n_targets_ > 1:
pred = np.zeros((X.shape[0], self.n_targets_))
else:
pred = np.zeros(X.shape[0])
for L in self._leaves.values():
mask = _predict_branch(X, L.threshold)
if (~mask).all():
continue
pred[mask] = L.model.predict(X[np.ix_(mask, self._linear_features)])
return pred
class LinearTreeClassifier(_LinearTree, ClassifierMixin):
"""A Linear Tree Classifier.
A Linear Tree Classifier is a meta-estimator that combine the learning
ability of Decision Tree and the predictive power of Linear Models.
Like in tree-based algorithms, the received data are splitted according
simple decision rules. The goodness of slits is evaluated in gain terms
fitting linear models in each node. This implies that the models in the
leaves are linear instead of constant approximations like in classical
Decision Tree.
Parameters
----------
base_estimator : object
The base estimator to fit on dataset splits.
The base estimator must be a sklearn.linear_model.
The selected base estimator is automatically substituted by a
`~sklearn.dummy.DummyClassifier` when a dataset split
is composed of unique labels.
criterion : {"hamming", "crossentropy"}, default="hamming"
The function to measure the quality of a split. `"crossentropy"`
can be used only if `base_estimator` has `predict_proba` method.
max_depth : int, default=5
The maximum depth of the tree considering only the splitting nodes.
A higher value implies a higher training time.
min_samples_split : int or float, default=6
The minimum number of samples required to split an internal node.
The minimum valid number of samples in each node is 6.
A lower value implies a higher training time.
- If int, then consider `min_samples_split` as the minimum number.
- If float, then `min_samples_split` is a fraction and
`ceil(min_samples_split * n_samples)` are the minimum
number of samples for each split.
min_samples_leaf : int or float, default=0.1
The minimum number of samples required to be at a leaf node.
A split point at any depth will only be considered if it leaves at
least `min_samples_leaf` training samples in each of the left and
right branches.
The minimum valid number of samples in each leaf is 3.
A lower value implies a higher training time.
- If int, then consider `min_samples_leaf` as the minimum number.
- If float, then `min_samples_leaf` is a fraction and
`ceil(min_samples_leaf * n_samples)` are the minimum
number of samples for each node.
max_bins : int, default=25
The maximum number of bins to use to search the optimal split in each
feature. Features with a small number of unique values may use less than
``max_bins`` bins. Must be lower than 120 and larger than 10.
A higher value implies a higher training time.
categorical_features : int or array-like of int, default=None
Indicates the categorical features.
All categorical indices must be in `[0, n_features)`.
Categorical features are used for splits but are not used in
model fitting.
More categorical features imply a higher training time.
- None : no feature will be considered categorical.
- integer array-like : integer indices indicating categorical
features.
- integer : integer index indicating a categorical
feature.
split_features : int or array-like of int, default=None
Defines which features can be used to split on.
All split feature indices must be in `[0, n_features)`.
- None : All features will be used for splitting.
- integer array-like : integer indices indicating splitting features.
- integer : integer index indicating a single splitting feature.
linear_features : int or array-like of int, default=None
Defines which features are used for the linear model in the leaves.
All linear feature indices must be in `[0, n_features)`.
- None : All features except those in `categorical_features`
will be used in the leaf models.
- integer array-like : integer indices indicating features to
be used in the leaf models.
- integer : integer index indicating a single feature to be used
in the leaf models.
n_jobs : int, default=None
The number of jobs to run in parallel for model fitting.
``None`` means 1 using one processor. ``-1`` means using all
processors.
Attributes
----------
n_features_in_ : int
The number of features when :meth:`fit` is performed.
feature_importances_ : ndarray of shape (n_features, )
Normalized total reduction of criteria by splitting features.
classes_ : ndarray of shape (n_classes, )
A list of class labels known to the classifier.
Examples
--------
>>> from sklearn.linear_model import RidgeClassifier
>>> from lineartree import LinearTreeClassifier
>>> from sklearn.datasets import make_classification
>>> X, y = make_classification(n_samples=100, n_features=4,
... n_informative=2, n_redundant=0,
... random_state=0, shuffle=False)
>>> clf = LinearTreeClassifier(base_estimator=RidgeClassifier())
>>> clf.fit(X, y)
>>> clf.predict([[0, 0, 0, 0]])
array([1])
"""
def __init__(
self,
base_estimator,
*,
criterion="hamming",
max_depth=5,
min_samples_split=6,
min_samples_leaf=0.1,
max_bins=25,
categorical_features=None,
split_features=None,
linear_features=None,
n_jobs=None
):
self.base_estimator = base_estimator
self.criterion = criterion
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.max_bins = max_bins
self.categorical_features = categorical_features
self.split_features = split_features
self.linear_features = linear_features
self.n_jobs = n_jobs
def fit(self, X, y, sample_weight=None):
"""Build a Linear Tree of a linear estimator from the training
set (X, y).
Parameters
----------
X : array-like of shape (n_samples, n_features)
The training input samples.
y : array-like of shape (n_samples, )
Target values.
sample_weight : array-like of shape (n_samples, ), default=None
Sample weights. If None, then samples are equally weighted.
Note that if the base estimator does not support sample weighting,
the sample weights are still used to evaluate the splits.
Returns
-------
self : object
"""
clas_criterions = ("hamming", "crossentropy")
if self.criterion not in clas_criterions:
raise ValueError(
"Classification tasks support only criterion in {}, "
"got '{}'.".format(clas_criterions, self.criterion)
)
if (
not hasattr(self.base_estimator, "predict_proba")
and self.criterion == "crossentropy"
):
raise ValueError(
"The 'crossentropy' criterion requires a base_estimator "
"with predict_proba method."
)
# Convert data (X is required to be 2d and indexable)
X, y = self._validate_data(
X,
y,
accept_sparse=False,
dtype=None,
force_all_finite=False,
multi_output=False,
)
if sample_weight is not None:
sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
self.classes_ = np.unique(y)
self._fit(X, y, sample_weight)
return self
def predict(self, X):
"""Predict class for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, )
The predicted classes.
"""
check_is_fitted(self, attributes="_nodes")
X = check_array(X, accept_sparse=False, dtype=None, force_all_finite=False)
self._check_n_features(X, reset=False)
pred = np.empty(X.shape[0], dtype=self.classes_.dtype)
for L in self._leaves.values():
mask = _predict_branch(X, L.threshold)
if (~mask).all():
continue
pred[mask] = L.model.predict(X[np.ix_(mask, self._linear_features)])
return pred
def predict_proba(self, X):
"""Predict class probabilities for X.
If base estimators do not implement a ``predict_proba`` method,
then the one-hot encoding of the predicted class is returned.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, n_classes)
The class probabilities of the input samples. The order of the
classes corresponds to that in the attribute :term:`classes_`.
"""
check_is_fitted(self, attributes="_nodes")
X = check_array(X, accept_sparse=False, dtype=None, force_all_finite=False)
self._check_n_features(X, reset=False)
pred = np.zeros((X.shape[0], len(self.classes_)))
if hasattr(self.base_estimator, "predict_proba"):
for L in self._leaves.values():
mask = _predict_branch(X, L.threshold)
if (~mask).all():
continue
pred[
np.ix_(mask, np.isin(self.classes_, L.classes))
] = L.model.predict_proba(X[np.ix_(mask, self._linear_features)])
else:
pred_class = self.predict(X)
class_to_int = dict(map(reversed, enumerate(self.classes_)))
pred_class = np.array([class_to_int[i] for i in pred_class])
pred[np.arange(X.shape[0]), pred_class] = 1
return pred
def predict_log_proba(self, X):
"""Predict class log-probabilities for X.
If base estimators do not implement a ``predict_log_proba`` method,
then the logarithm of the one-hot encoded predicted class is returned.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, n_classes)
The class log-probabilities of the input samples. The order of the
classes corresponds to that in the attribute :term:`classes_`.
"""
return np.log(self.predict_proba(X))
class LinearBoostRegressor(_LinearBoosting, RegressorMixin):
"""A Linear Boosting Regressor.
A Linear Boosting Regressor is an iterative meta-estimator that starts
with a linear regressor, and model the residuals through decision trees.
At each iteration, the path leading to highest error (i.e. the worst leaf)
is added as a new binary variable to the base model. This kind of Linear
Boosting can be considered as an improvement over general linear models
since it enables incorporating non-linear features by residuals modeling.
Parameters
----------
base_estimator : object
The base estimator iteratively fitted.
The base estimator must be a sklearn.linear_model.
loss : {"linear", "square", "absolute", "exponential"}, default="linear"
The function used to calculate the residuals of each sample.
n_estimators : int, default=10
The number of boosting stages to perform. It corresponds to the number
of the new features generated.
max_depth : int, default=3
The maximum depth of the tree. If None, then nodes are expanded until
all leaves are pure or until all leaves contain less than
min_samples_split samples.
min_samples_split : int or float, default=2
The minimum number of samples required to split an internal node:
- If int, then consider `min_samples_split` as the minimum number.
- If float, then `min_samples_split` is a fraction and
`ceil(min_samples_split * n_samples)` are the minimum
number of samples for each split.
min_samples_leaf : int or float, default=1
The minimum number of samples required to be at a leaf node.
A split point at any depth will only be considered if it leaves at
least ``min_samples_leaf`` training samples in each of the left and
right branches. This may have the effect of smoothing the model,
especially in regression.
- If int, then consider `min_samples_leaf` as the minimum number.
- If float, then `min_samples_leaf` is a fraction and
`ceil(min_samples_leaf * n_samples)` are the minimum
number of samples for each node.
min_weight_fraction_leaf : float, default=0.0
The minimum weighted fraction of the sum total of weights (of all
the input samples) required to be at a leaf node. Samples have
equal weight when sample_weight is not provided.
max_features : int, float or {"auto", "sqrt", "log2"}, default=None
The number of features to consider when looking for the best split:
- If int, then consider `max_features` features at each split.
- If float, then `max_features` is a fraction and
`int(max_features * n_features)` features are considered at each
split.
- If "auto", then `max_features=n_features`.
- If "sqrt", then `max_features=sqrt(n_features)`.
- If "log2", then `max_features=log2(n_features)`.
- If None, then `max_features=n_features`.
Note: the search for a split does not stop until at least one
valid partition of the node samples is found, even if it requires to
effectively inspect more than ``max_features`` features.
random_state : int, RandomState instance or None, default=None
Controls the randomness of the estimator.
max_leaf_nodes : int, default=None
Grow a tree with ``max_leaf_nodes`` in best-first fashion.
Best nodes are defined as relative reduction in impurity.
If None then unlimited number of leaf nodes.
min_impurity_decrease : float, default=0.0
A node will be split if this split induces a decrease of the impurity
greater than or equal to this value.
ccp_alpha : non-negative float, default=0.0
Complexity parameter used for Minimal Cost-Complexity Pruning. The
subtree with the largest cost complexity that is smaller than
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
:ref:`minimal_cost_complexity_pruning` for details.
Attributes
----------
n_features_in_ : int
The number of features when :meth:`fit` is performed.
n_features_out_ : int
The total number of features used to fit the base estimator in the
last iteration. The number of output features is equal to the sum
of n_features_in_ and n_estimators.
coef_ : array of shape (n_features_out_, ) or (n_targets, n_features_out_)
Estimated coefficients for the linear regression problem.
If multiple targets are passed during the fit (y 2D), this is a
2D array of shape (n_targets, n_features_out_), while if only one target
is passed, this is a 1D array of length n_features_out_.
intercept_ : float or array of shape (n_targets, )
Independent term in the linear model. Set to 0 if `fit_intercept = False`
in `base_estimator`
Examples
--------
>>> from sklearn.linear_model import LinearRegression
>>> from lineartree import LinearBoostRegressor
>>> from sklearn.datasets import make_regression
>>> X, y = make_regression(n_samples=100, n_features=4,
... n_informative=2, n_targets=1,
... random_state=0, shuffle=False)
>>> regr = LinearBoostRegressor(base_estimator=LinearRegression())
>>> regr.fit(X, y)
>>> regr.predict([[0, 0, 0, 0]])
array([8.8817842e-16])
References
----------
Explainable boosted linear regression for time series forecasting.
Authors: Igor Ilic, Berk Gorgulu, Mucahit Cevik, Mustafa Gokce Baydogan.
(https://arxiv.org/abs/2009.09110)
"""
def __init__(
self,
base_estimator,
*,
loss="linear",
n_estimators=10,
max_depth=3,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features=None,
random_state=None,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
ccp_alpha=0.0
):
self.base_estimator = base_estimator
self.loss = loss
self.n_estimators = n_estimators
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.min_weight_fraction_leaf = min_weight_fraction_leaf
self.max_features = max_features
self.random_state = random_state
self.max_leaf_nodes = max_leaf_nodes
self.min_impurity_decrease = min_impurity_decrease
self.ccp_alpha = ccp_alpha
def fit(self, X, y, sample_weight=None):
"""Build a Linear Boosting from the training set (X, y).
Parameters
----------
X : array-like of shape (n_samples, n_features)
The training input samples.
y : array-like of shape (n_samples, ) or (n_samples, n_targets)
Target values.
sample_weight : array-like of shape (n_samples, ), default=None
Sample weights.
Returns
-------
self : object
"""
reg_losses = ("linear", "square", "absolute", "exponential")
if self.loss not in reg_losses:
raise ValueError(
"Regression tasks support only loss in {}, "
"got '{}'.".format(reg_losses, self.loss)
)
# Convert data (X is required to be 2d and indexable)
X, y = self._validate_data(
X, y, accept_sparse=False, dtype=np.float32, multi_output=True
)
if sample_weight is not None:
sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
y_shape = np.shape(y)
n_targets = y_shape[1] if len(y_shape) > 1 else 1
if n_targets < 2:
y = y.ravel()
self._fit(X, y, sample_weight)
return self
def predict(self, X):
"""Predict regression target for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, ) or also (n_samples, n_targets) if
multitarget regression.
The predicted values.
"""
check_is_fitted(self, attributes="base_estimator_")
return self.base_estimator_.predict(self.transform(X))
class LinearBoostClassifier(_LinearBoosting, ClassifierMixin):
"""A Linear Boosting Classifier.
A Linear Boosting Classifier is an iterative meta-estimator that starts
with a linear classifier, and model the residuals through decision trees.
At each iteration, the path leading to highest error (i.e. the worst leaf)
is added as a new binary variable to the base model. This kind of Linear
Boosting can be considered as an improvement over general linear models
since it enables incorporating non-linear features by residuals modeling.
Parameters
----------
base_estimator : object
The base estimator iteratively fitted.
The base estimator must be a sklearn.linear_model.
loss : {"hamming", "entropy"}, default="entropy"
The function used to calculate the residuals of each sample.
`"entropy"` can be used only if `base_estimator` has `predict_proba`
method.
n_estimators : int, default=10
The number of boosting stages to perform. It corresponds to the number
of the new features generated.
max_depth : int, default=3
The maximum depth of the tree. If None, then nodes are expanded until
all leaves are pure or until all leaves contain less than
min_samples_split samples.
min_samples_split : int or float, default=2
The minimum number of samples required to split an internal node:
- If int, then consider `min_samples_split` as the minimum number.
- If float, then `min_samples_split` is a fraction and
`ceil(min_samples_split * n_samples)` are the minimum
number of samples for each split.
min_samples_leaf : int or float, default=1
The minimum number of samples required to be at a leaf node.
A split point at any depth will only be considered if it leaves at
least ``min_samples_leaf`` training samples in each of the left and
right branches. This may have the effect of smoothing the model,
especially in regression.
- If int, then consider `min_samples_leaf` as the minimum number.
- If float, then `min_samples_leaf` is a fraction and
`ceil(min_samples_leaf * n_samples)` are the minimum
number of samples for each node.
min_weight_fraction_leaf : float, default=0.0
The minimum weighted fraction of the sum total of weights (of all
the input samples) required to be at a leaf node. Samples have
equal weight when sample_weight is not provided.
max_features : int, float or {"auto", "sqrt", "log2"}, default=None
The number of features to consider when looking for the best split:
- If int, then consider `max_features` features at each split.
- If float, then `max_features` is a fraction and
`int(max_features * n_features)` features are considered at each
split.
- If "auto", then `max_features=n_features`.
- If "sqrt", then `max_features=sqrt(n_features)`.
- If "log2", then `max_features=log2(n_features)`.
- If None, then `max_features=n_features`.
Note: the search for a split does not stop until at least one
valid partition of the node samples is found, even if it requires to
effectively inspect more than ``max_features`` features.
random_state : int, RandomState instance or None, default=None
Controls the randomness of the estimator.
max_leaf_nodes : int, default=None
Grow a tree with ``max_leaf_nodes`` in best-first fashion.
Best nodes are defined as relative reduction in impurity.
If None then unlimited number of leaf nodes.
min_impurity_decrease : float, default=0.0
A node will be split if this split induces a decrease of the impurity
greater than or equal to this value.
ccp_alpha : non-negative float, default=0.0
Complexity parameter used for Minimal Cost-Complexity Pruning. The
subtree with the largest cost complexity that is smaller than
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
:ref:`minimal_cost_complexity_pruning` for details.
Attributes
----------
n_features_in_ : int
The number of features when :meth:`fit` is performed.
n_features_out_ : int
The total number of features used to fit the base estimator in the
last iteration. The number of output features is equal to the sum
of n_features_in_ and n_estimators.
coef_ : ndarray of shape (1, n_features_out_) or (n_classes, n_features_out_)
Coefficient of the features in the decision function.
intercept_ : float or array of shape (n_classes, )
Independent term in the linear model. Set to 0 if `fit_intercept = False`
in `base_estimator`
classes_ : ndarray of shape (n_classes, )
A list of class labels known to the classifier.
Examples
--------
>>> from sklearn.linear_model import RidgeClassifier
>>> from lineartree import LinearBoostClassifier
>>> from sklearn.datasets import make_classification
>>> X, y = make_classification(n_samples=100, n_features=4,
... n_informative=2, n_redundant=0,
... random_state=0, shuffle=False)
>>> clf = LinearBoostClassifier(base_estimator=RidgeClassifier())
>>> clf.fit(X, y)
>>> clf.predict([[0, 0, 0, 0]])
array([1])
References
----------
Explainable boosted linear regression for time series forecasting.
Authors: Igor Ilic, Berk Gorgulu, Mucahit Cevik, Mustafa Gokce Baydogan.
(https://arxiv.org/abs/2009.09110)
"""
def __init__(
self,
base_estimator,
*,
loss="hamming",
n_estimators=10,
max_depth=3,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features=None,
random_state=None,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
ccp_alpha=0.0
):
self.base_estimator = base_estimator
self.loss = loss
self.n_estimators = n_estimators
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.min_weight_fraction_leaf = min_weight_fraction_leaf
self.max_features = max_features
self.random_state = random_state
self.max_leaf_nodes = max_leaf_nodes
self.min_impurity_decrease = min_impurity_decrease
self.ccp_alpha = ccp_alpha
def fit(self, X, y, sample_weight=None):
"""Build a Linear Boosting from the training set (X, y).
Parameters
----------
X : array-like of shape (n_samples, n_features)
The training input samples.
y : array-like of shape (n_samples, )
Target values.
sample_weight : array-like of shape (n_samples, ), default=None
Sample weights.
Returns
-------
self : object
"""
clas_losses = ("hamming", "entropy")
if self.loss not in clas_losses:
raise ValueError(
"Classification tasks support only loss in {}, "
"got '{}'.".format(clas_losses, self.loss)
)
if not hasattr(self.base_estimator, "predict_proba") and self.loss == "entropy":
raise ValueError(
"The 'entropy' loss requires a base_estimator "
"with predict_proba method."
)
# Convert data (X is required to be 2d and indexable)
X, y = self._validate_data(
X, y, accept_sparse=False, dtype=np.float32, multi_output=False
)
if sample_weight is not None:
sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
self.classes_ = np.unique(y)
self._fit(X, y, sample_weight)
return self
def predict(self, X):
"""Predict class for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, )
The predicted classes.
"""
check_is_fitted(self, attributes="base_estimator_")
return self.base_estimator_.predict(self.transform(X))
def predict_proba(self, X):
"""Predict class probabilities for X.
If base estimators do not implement a ``predict_proba`` method,
then the one-hot encoding of the predicted class is returned.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, n_classes)
The class probabilities of the input samples. The order of the
classes corresponds to that in the attribute :term:`classes_`.
"""
if hasattr(self.base_estimator, "predict_proba"):
check_is_fitted(self, attributes="base_estimator_")
pred = self.base_estimator_.predict_proba(self.transform(X))
else:
pred_class = self.predict(X)
pred = np.zeros((pred_class.shape[0], len(self.classes_)))
class_to_int = dict(map(reversed, enumerate(self.classes_)))
pred_class = np.array([class_to_int[v] for v in pred_class])
pred[np.arange(pred_class.shape[0]), pred_class] = 1
return pred
def predict_log_proba(self, X):
"""Predict class log-probabilities for X.
If base estimators do not implement a ``predict_log_proba`` method,
then the logarithm of the one-hot encoded predicted class is returned.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, n_classes)
The class log-probabilities of the input samples. The order of the
classes corresponds to that in the attribute :term:`classes_`.
"""
return np.log(self.predict_proba(X))
class LinearForestClassifier(_LinearForest, ClassifierMixin):
""" "A Linear Forest Classifier.
Linear forests generalizes the well known random forests by combining
linear models with the same random forests.
The key idea of linear forests is to use the strength of linear models
to improve the nonparametric learning ability of tree-based algorithms.
Firstly, a linear model is fitted on the whole dataset, then a random
forest is trained on the same dataset but using the residuals of the
previous steps as target. The final predictions are the sum of the raw
linear predictions and the residuals modeled by the random forest.
For classification tasks the same approach used in regression context
is adopted. The binary targets are transformed into logits using the
inverse sigmoid function. A linear regression is fitted. A random forest
regressor is trained to approximate the residulas from logits and linear
predictions. Finally the sigmoid of the combinded predictions are taken
to obtain probabilities.
The multi-label scenario is carried out using OneVsRestClassifier.
Parameters
----------
base_estimator : object
The linear estimator fitted on the raw target.
The linear estimator must be a regressor from sklearn.linear_model.
n_estimators : int, default=100
The number of trees in the forest.
max_depth : int, default=None
The maximum depth of the tree. If None, then nodes are expanded until
all leaves are pure or until all leaves contain less than
min_samples_split samples.
min_samples_split : int or float, default=2
The minimum number of samples required to split an internal node:
- If int, then consider `min_samples_split` as the minimum number.
- If float, then `min_samples_split` is a fraction and
`ceil(min_samples_split * n_samples)` are the minimum
number of samples for each split.
min_samples_leaf : int or float, default=1
The minimum number of samples required to be at a leaf node.
A split point at any depth will only be considered if it leaves at
least ``min_samples_leaf`` training samples in each of the left and
right branches. This may have the effect of smoothing the model,
especially in regression.
- If int, then consider `min_samples_leaf` as the minimum number.
- If float, then `min_samples_leaf` is a fraction and
`ceil(min_samples_leaf * n_samples)` are the minimum
number of samples for each node.
min_weight_fraction_leaf : float, default=0.0
The minimum weighted fraction of the sum total of weights (of all
the input samples) required to be at a leaf node. Samples have
equal weight when sample_weight is not provided.
max_features : {"auto", "sqrt", "log2"}, int or float, default="auto"
The number of features to consider when looking for the best split:
- If int, then consider `max_features` features at each split.
- If float, then `max_features` is a fraction and
`round(max_features * n_features)` features are considered at each
split.
- If "auto", then `max_features=n_features`.
- If "sqrt", then `max_features=sqrt(n_features)`.
- If "log2", then `max_features=log2(n_features)`.
- If None, then `max_features=n_features`.
Note: the search for a split does not stop until at least one
valid partition of the node samples is found, even if it requires to
effectively inspect more than ``max_features`` features.
max_leaf_nodes : int, default=None
Grow trees with ``max_leaf_nodes`` in best-first fashion.
Best nodes are defined as relative reduction in impurity.
If None then unlimited number of leaf nodes.
min_impurity_decrease : float, default=0.0
A node will be split if this split induces a decrease of the impurity
greater than or equal to this value.
bootstrap : bool, default=True
Whether bootstrap samples are used when building trees. If False, the
whole dataset is used to build each tree.
oob_score : bool, default=False
Whether to use out-of-bag samples to estimate the generalization score.
Only available if bootstrap=True.
n_jobs : int, default=None
The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`,
:meth:`decision_path` and :meth:`apply` are all parallelized over the
trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend`
context. ``-1`` means using all processors.
random_state : int, RandomState instance or None, default=None
Controls both the randomness of the bootstrapping of the samples used
when building trees (if ``bootstrap=True``) and the sampling of the
features to consider when looking for the best split at each node
(if ``max_features < n_features``).
ccp_alpha : non-negative float, default=0.0
Complexity parameter used for Minimal Cost-Complexity Pruning. The
subtree with the largest cost complexity that is smaller than
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
:ref:`minimal_cost_complexity_pruning` for details.
max_samples : int or float, default=None
If bootstrap is True, the number of samples to draw from X
to train each base estimator.
- If None (default), then draw `X.shape[0]` samples.
- If int, then draw `max_samples` samples.
- If float, then draw `max_samples * X.shape[0]` samples. Thus,
`max_samples` should be in the interval `(0, 1]`.
Attributes
----------
n_features_in_ : int
The number of features when :meth:`fit` is performed.
feature_importances_ : ndarray of shape (n_features, )
The impurity-based feature importances.
The higher, the more important the feature.
The importance of a feature is computed as the (normalized)
total reduction of the criterion brought by that feature. It is also
known as the Gini importance.
coef_ : ndarray of shape (1, n_features_out_)
Coefficient of the features in the decision function.
intercept_ : float
Independent term in the linear model. Set to 0 if `fit_intercept = False`
in `base_estimator`.
classes_ : ndarray of shape (n_classes, )
A list of class labels known to the classifier.
base_estimator_ : object
A fitted linear model instance.
forest_estimator_ : object
A fitted random forest instance.
Examples
--------
>>> from sklearn.linear_model import LinearRegression
>>> from lineartree import LinearForestClassifier
>>> from sklearn.datasets import make_classification
>>> X, y = make_classification(n_samples=100, n_classes=2, n_features=4,
... n_informative=2, n_redundant=0,
... random_state=0, shuffle=False)
>>> clf = LinearForestClassifier(base_estimator=LinearRegression())
>>> clf.fit(X, y)
>>> clf.predict([[0, 0, 0, 0]])
array([1])
References
----------
Regression-Enhanced Random Forests.
Authors: Haozhe Zhang, Dan Nettleton, Zhengyuan Zhu.
(https://arxiv.org/abs/1904.10416)
"""
def __init__(
self,
base_estimator,
*,
n_estimators=100,
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features="auto",
max_leaf_nodes=None,
min_impurity_decrease=0.0,
bootstrap=True,
oob_score=False,
n_jobs=None,
random_state=None,
ccp_alpha=0.0,
max_samples=None
):
self.base_estimator = base_estimator
self.n_estimators = n_estimators
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.min_weight_fraction_leaf = min_weight_fraction_leaf
self.max_features = max_features
self.max_leaf_nodes = max_leaf_nodes
self.min_impurity_decrease = min_impurity_decrease
self.bootstrap = bootstrap
self.oob_score = oob_score
self.n_jobs = n_jobs
self.random_state = random_state
self.ccp_alpha = ccp_alpha
self.max_samples = max_samples
def fit(self, X, y, sample_weight=None):
"""Build a Linear Forest from the training set (X, y).
Parameters
----------
X : array-like of shape (n_samples, n_features)
The training input samples.
y : array-like of shape (n_samples, ) or (n_samples, n_targets)
Target values.
sample_weight : array-like of shape (n_samples, ), default=None
Sample weights.
Returns
-------
self : object
"""
# Convert data (X is required to be 2d and indexable)
X, y = self._validate_data(
X, y, accept_sparse=True, dtype=None, multi_output=False
)
if sample_weight is not None:
sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
self.classes_ = np.unique(y)
if len(self.classes_) > 2:
raise ValueError(
"LinearForestClassifier supports only binary classification task. "
"To solve a multi-lable classification task use "
"LinearForestClassifier with OneVsRestClassifier from sklearn."
)
self._fit(X, y, sample_weight)
return self
def decision_function(self, X):
"""Predict confidence scores for samples.
The confidence score for a sample is proportional to the signed
distance of that sample to the hyperplane.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, )
Confidence scores.
Confidence score for self.classes_[1] where >0 means this
class would be predicted.
"""
check_is_fitted(self, attributes="base_estimator_")
X = check_array(X, dtype=None, accept_sparse=False)
self._check_n_features(X, reset=False)
linear_pred = self.base_estimator_.predict(X)
forest_pred = self.forest_estimator_.predict(X)
return linear_pred + forest_pred
def predict(self, X):
"""Predict class for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, )
The predicted classes.
"""
pred = self.decision_function(X)
pred_class = (self._sigmoid(pred) > 0.5).astype(int)
int_to_class = dict(enumerate(self.classes_))
pred_class = np.array([int_to_class[i] for i in pred_class])
return pred_class
def predict_proba(self, X):
"""Predict class probabilities for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
proba : ndarray of shape (n_samples, n_classes)
The class probabilities of the input samples. The order of the
classes corresponds to that in the attribute :term:`classes_`.
"""
check_is_fitted(self, attributes="base_estimator_")
X = check_array(X, dtype=None, accept_sparse=False)
self._check_n_features(X, reset=False)
pred = self._sigmoid(self.base_estimator_.predict(X))
proba = np.zeros((X.shape[0], 2))
proba[:, 0] = 1 - pred
proba[:, 1] = pred
return proba
def predict_log_proba(self, X):
"""Predict class log-probabilities for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, n_classes)
The class log-probabilities of the input samples. The order of the
classes corresponds to that in the attribute :term:`classes_`.
"""
return np.log(self.predict_proba(X))
class LinearForestRegressor(_LinearForest, RegressorMixin):
""" "A Linear Forest Regressor.
Linear forests generalizes the well known random forests by combining
linear models with the same random forests.
The key idea of linear forests is to use the strength of linear models
to improve the nonparametric learning ability of tree-based algorithms.
Firstly, a linear model is fitted on the whole dataset, then a random
forest is trained on the same dataset but using the residuals of the
previous steps as target. The final predictions are the sum of the raw
linear predictions and the residuals modeled by the random forest.
Parameters
----------
base_estimator : object
The linear estimator fitted on the raw target.
The linear estimator must be a regressor from sklearn.linear_model.
n_estimators : int, default=100
The number of trees in the forest.
max_depth : int, default=None
The maximum depth of the tree. If None, then nodes are expanded until
all leaves are pure or until all leaves contain less than
min_samples_split samples.
min_samples_split : int or float, default=2
The minimum number of samples required to split an internal node:
- If int, then consider `min_samples_split` as the minimum number.
- If float, then `min_samples_split` is a fraction and
`ceil(min_samples_split * n_samples)` are the minimum
number of samples for each split.
min_samples_leaf : int or float, default=1
The minimum number of samples required to be at a leaf node.
A split point at any depth will only be considered if it leaves at
least ``min_samples_leaf`` training samples in each of the left and
right branches. This may have the effect of smoothing the model,
especially in regression.
- If int, then consider `min_samples_leaf` as the minimum number.
- If float, then `min_samples_leaf` is a fraction and
`ceil(min_samples_leaf * n_samples)` are the minimum
number of samples for each node.
min_weight_fraction_leaf : float, default=0.0
The minimum weighted fraction of the sum total of weights (of all
the input samples) required to be at a leaf node. Samples have
equal weight when sample_weight is not provided.
max_features : {"auto", "sqrt", "log2"}, int or float, default="auto"
The number of features to consider when looking for the best split:
- If int, then consider `max_features` features at each split.
- If float, then `max_features` is a fraction and
`round(max_features * n_features)` features are considered at each
split.
- If "auto", then `max_features=n_features`.
- If "sqrt", then `max_features=sqrt(n_features)`.
- If "log2", then `max_features=log2(n_features)`.
- If None, then `max_features=n_features`.
Note: the search for a split does not stop until at least one
valid partition of the node samples is found, even if it requires to
effectively inspect more than ``max_features`` features.
max_leaf_nodes : int, default=None
Grow trees with ``max_leaf_nodes`` in best-first fashion.
Best nodes are defined as relative reduction in impurity.
If None then unlimited number of leaf nodes.
min_impurity_decrease : float, default=0.0
A node will be split if this split induces a decrease of the impurity
greater than or equal to this value.
bootstrap : bool, default=True
Whether bootstrap samples are used when building trees. If False, the
whole dataset is used to build each tree.
oob_score : bool, default=False
Whether to use out-of-bag samples to estimate the generalization score.
Only available if bootstrap=True.
n_jobs : int, default=None
The number of jobs to run in parallel. :meth:`fit`, :meth:`predict`,
:meth:`decision_path` and :meth:`apply` are all parallelized over the
trees. ``None`` means 1 unless in a :obj:`joblib.parallel_backend`
context. ``-1`` means using all processors.
random_state : int, RandomState instance or None, default=None
Controls both the randomness of the bootstrapping of the samples used
when building trees (if ``bootstrap=True``) and the sampling of the
features to consider when looking for the best split at each node
(if ``max_features < n_features``).
ccp_alpha : non-negative float, default=0.0
Complexity parameter used for Minimal Cost-Complexity Pruning. The
subtree with the largest cost complexity that is smaller than
``ccp_alpha`` will be chosen. By default, no pruning is performed. See
:ref:`minimal_cost_complexity_pruning` for details.
max_samples : int or float, default=None
If bootstrap is True, the number of samples to draw from X
to train each base estimator.
- If None (default), then draw `X.shape[0]` samples.
- If int, then draw `max_samples` samples.
- If float, then draw `max_samples * X.shape[0]` samples. Thus,
`max_samples` should be in the interval `(0, 1]`.
Attributes
----------
n_features_in_ : int
The number of features when :meth:`fit` is performed.
feature_importances_ : ndarray of shape (n_features, )
The impurity-based feature importances.
The higher, the more important the feature.
The importance of a feature is computed as the (normalized)
total reduction of the criterion brought by that feature. It is also
known as the Gini importance.
coef_ : array of shape (n_features, ) or (n_targets, n_features)
Estimated coefficients for the linear regression problem.
If multiple targets are passed during the fit (y 2D), this is a
2D array of shape (n_targets, n_features), while if only one target
is passed, this is a 1D array of length n_features.
intercept_ : float or array of shape (n_targets,)
Independent term in the linear model. Set to 0 if `fit_intercept = False`
in `base_estimator`.
base_estimator_ : object
A fitted linear model instance.
forest_estimator_ : object
A fitted random forest instance.
Examples
--------
>>> from sklearn.linear_model import LinearRegression
>>> from lineartree import LinearForestRegressor
>>> from sklearn.datasets import make_regression
>>> X, y = make_regression(n_samples=100, n_features=4,
... n_informative=2, n_targets=1,
... random_state=0, shuffle=False)
>>> regr = LinearForestRegressor(base_estimator=LinearRegression())
>>> regr.fit(X, y)
>>> regr.predict([[0, 0, 0, 0]])
array([8.8817842e-16])
References
----------
Regression-Enhanced Random Forests.
Authors: Haozhe Zhang, Dan Nettleton, Zhengyuan Zhu.
(https://arxiv.org/abs/1904.10416)
"""
def __init__(
self,
base_estimator,
*,
n_estimators=100,
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features="auto",
max_leaf_nodes=None,
min_impurity_decrease=0.0,
bootstrap=True,
oob_score=False,
n_jobs=None,
random_state=None,
ccp_alpha=0.0,
max_samples=None
):
self.base_estimator = base_estimator
self.n_estimators = n_estimators
self.max_depth = max_depth
self.min_samples_split = min_samples_split
self.min_samples_leaf = min_samples_leaf
self.min_weight_fraction_leaf = min_weight_fraction_leaf
self.max_features = max_features
self.max_leaf_nodes = max_leaf_nodes
self.min_impurity_decrease = min_impurity_decrease
self.bootstrap = bootstrap
self.oob_score = oob_score
self.n_jobs = n_jobs
self.random_state = random_state
self.ccp_alpha = ccp_alpha
self.max_samples = max_samples
def fit(self, X, y, sample_weight=None):
"""Build a Linear Forest from the training set (X, y).
Parameters
----------
X : array-like of shape (n_samples, n_features)
The training input samples.
y : array-like of shape (n_samples, ) or (n_samples, n_targets)
Target values.
sample_weight : array-like of shape (n_samples, ), default=None
Sample weights.
Returns
-------
self : object
"""
# Convert data (X is required to be 2d and indexable)
X, y = self._validate_data(
X, y, accept_sparse=True, dtype=None, multi_output=True
)
if sample_weight is not None:
sample_weight = _check_sample_weight(sample_weight, X, dtype=None)
y_shape = np.shape(y)
n_targets = y_shape[1] if len(y_shape) > 1 else 1
if n_targets < 2:
y = y.ravel()
self._fit(X, y, sample_weight)
return self
def predict(self, X):
"""Predict regression target for X.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Samples.
Returns
-------
pred : ndarray of shape (n_samples, ) or also (n_samples, n_targets) if
multitarget regression.
The predicted values.
"""
check_is_fitted(self, attributes="base_estimator_")
X = check_array(X, dtype=None, accept_sparse=False)
self._check_n_features(X, reset=False)
linear_pred = self.base_estimator_.predict(X)
forest_pred = self.forest_estimator_.predict(X)
return linear_pred + forest_pred