import numpy as np
from copy import deepcopy
from scipy.stats import norm, t
from sklearn.metrics.pairwise import pairwise_kernels, KERNEL_PARAMS
from sklearn.utils import check_array
from sklearn.utils.validation import check_is_fitted
from skactiveml.base import ProbabilisticRegressor
from skactiveml.utils import (
MISSING_LABEL,
check_scalar,
check_type,
check_n_features,
)
from skactiveml.utils._label import _observed_numerical_labels
[docs]
class NICKernelRegressor(ProbabilisticRegressor):
"""NIC Kernel Regressor
The NICKernelRegressor (Normal Inverse Chi-square Kernel Regressor)
locally fits a t-distribution using the training data, weighting the
samples by a kernel.
Parameters
----------
metric : str or callable, default='rbf'
The metric must a be a valid kernel defined by the function
`sklearn.metrics.pairwise.pairwise_kernels`, or `'precomputed'`.
Its values weight the training targets, so it must not be negative
for the given data.
`'rbf'`, `'laplacian'` and `'chi2'` always satisfy this.
`'linear'`, `'poly'`, `'polynomial'`, `'sigmoid'` and `'cosine'` do
so only for some data, and `'additive_chi2'` never does because it
is non-positive by construction. Prediction raises a `ValueError`
when the resulting kernel evidence is negative.
metric_dict : dict, default=None
Any further parameters are passed directly to the kernel function.
mu_0 : int or float, default=0
The prior mean.
kappa_0 : int or float >= 0, default=0.1
The weight of the prior mean.
sigma_sq_0 : int or float >= 0, default=1.0
The prior variance.
nu_0 : int or float >= 0, default=2.5
The weight of the prior variance.
missing_label : scalar or string or np.nan or None, default=np.nan
Value to represent a missing label.
random_state : int, RandomState instance or None, default=None
Determines random number for `predict` method. Pass an int for
reproducible results across multiple method calls.
"""
METRICS = list(KERNEL_PARAMS.keys()) + ["precomputed"]
def __init__(
self,
metric="rbf",
metric_dict=None,
mu_0=0,
kappa_0=0.1,
sigma_sq_0=1.0,
nu_0=2.5,
missing_label=MISSING_LABEL,
random_state=None,
):
super().__init__(
random_state=random_state,
missing_label=missing_label,
)
self.kappa_0 = kappa_0
self.nu_0 = nu_0
self.mu_0 = mu_0
self.sigma_sq_0 = sigma_sq_0
self.metric = metric
self.metric_dict = metric_dict
[docs]
def fit(self, X, y, sample_weight=None):
"""Fit the model using `X` as training data and `y` as labels.
Parameters
----------
X : matrix-like of shape (n_samples, n_features) or \
(n_samples, n_samples) if metric='precomputed'
Training data set, usually complete, i.e., including the labeled
and unlabeled samples. If `metric='precomputed'`, `X` contains the
pairwise kernels between all training samples.
y : array-like of shape (n_samples,) or (n_samples, 1)
Labels of the training data set (possibly including unlabeled ones
indicated by `self.missing_label`).
sample_weight : array-like of shape (n_samples,)
It contains the weights of the training samples' values.
Returns
-------
self: SkactivemlRegressor,
The SkactivemlRegressor is fitted on the training data.
"""
X, y, sample_weight = self._validate_data(X, y, sample_weight)
if self.metric == "precomputed" and X.shape[0] != X.shape[1]:
raise ValueError(
"For metric='precomputed', the training kernel matrix `X` "
"must have shape (n_train_samples, n_train_samples)."
)
is_lbld, y_observed = _observed_numerical_labels(
y, self.missing_label_
)
for value, name, min_val in [
(self.kappa_0, "self.kappa_0", 0),
(self.nu_0, "self.nu_0", 0),
(self.sigma_sq_0, "self.sigma_sq_0", 0),
(self.mu_0, "self.mu_0", None),
]:
check_scalar(value, name, (int, float), min_val=min_val)
if not np.isfinite(value):
raise ValueError(f"`{name}` must be finite, got {value}.")
self._is_lbld = is_lbld
self.X_ = X[is_lbld]
self.y_ = y_observed
self.prior_params_ = (
self.kappa_0,
self.nu_0,
self.mu_0,
self.sigma_sq_0,
)
if sample_weight is not None:
self.weights_ = sample_weight[is_lbld]
if len(self.weights_) > 0 and np.sum(self.weights_) == 0:
raise ValueError(
"The sample weights of the labeled samples "
"must not be all zero."
)
else:
self.weights_ = None
check_type(
self.metric,
"self.metric",
target_vals=self.METRICS,
indicator_funcs=[callable],
)
check_type(
self.metric_dict, "self.metric_dict", dict, target_vals=[None]
)
self.metric_dict_ = (
deepcopy(self.metric_dict) if self.metric_dict is not None else {}
)
return self
def _validate_prediction_data(self, X):
"""Validate feature data or a precomputed test kernel.
Parameters
----------
X : array-like of shape (n_samples, n_features) or \
(n_samples, n_train_samples) if metric='precomputed'
Input samples or kernels against all samples from the latest fit.
Returns
-------
X : numpy.ndarray
Validated input data.
"""
X = check_array(X)
if self.metric == "precomputed":
if X.shape[1] != self.n_features_in_:
raise ValueError(
"For metric='precomputed', the kernel matrix `X` must "
"have shape (n_test_samples, n_train_samples)."
)
else:
check_n_features(self, X, reset=False)
return X
def _estimate_ml_params(self, X):
if self.metric == "precomputed":
K = X[:, self._is_lbld]
else:
K = pairwise_kernels(
X, self.X_, metric=self.metric, **self.metric_dict_
)
K = np.asarray(K, dtype=float)
if self.weights_ is not None:
K = self.weights_.reshape(1, -1) * K
N = np.sum(K, axis=1)
mu_ml = np.divide(K @ self.y_, N, out=np.zeros_like(N), where=N != 0)
scatter = np.sum(
K * (self.y_[np.newaxis, :] - mu_ml[:, np.newaxis]) ** 2, axis=1
)
self._check_kernel_evidence(N, "kernel mass")
self._check_kernel_evidence(
scatter, "weighted sum of squared deviations (scatter)"
)
var_ml = np.divide(scatter, N, out=np.zeros_like(N), where=N != 0)
return N, mu_ml, var_ml
def _check_kernel_evidence(self, values, name):
"""Reject kernel evidence that cannot be a weighted count.
Parameters
----------
values : numpy.ndarray of shape (n_samples,)
Kernel-weighted quantity to validate.
name : str
Name of the quantity reported in the error message.
Raises
------
ValueError
If any value is negative.
"""
if np.any(values < 0):
raise ValueError(
f"The {name} weighting the training targets is negative, the "
f"smallest being {np.min(values)}, so it cannot serve as a "
"pseudo-count. This happens when `sample_weight` contains "
f"negative values, or when `metric={self.metric!r}` produces "
"negative similarities on the given data. Use non-negative "
"sample weights and a non-negative kernel, e.g. 'rbf' or "
"'laplacian', or supply a non-negative 'precomputed' matrix."
)
def _estimate_update_params(self, X):
if len(self.X_) != 0:
N, mu_ml, var_ml = self._estimate_ml_params(X)
return N, N, mu_ml, var_ml
else:
neutral_params = (np.zeros(len(X)),) * 4
return neutral_params
[docs]
def predict_target_distribution(self, X):
"""Returns the estimated target distribution conditioned on the test
samples `X`.
Parameters
----------
X : array-like of shape (n_samples, n_features) or \
(n_samples, n_train_samples) if metric='precomputed'
Input samples. If `metric='precomputed'`, `X` contains kernels
between the test samples and every sample passed to the latest
`fit` call.
Returns
-------
dist : scipy.stats._distn_infrastructure.rv_frozen
The distribution of the targets at the test samples. A posterior
scale of zero yields a point mass at the posterior mean.
"""
check_is_fitted(self)
X = self._validate_prediction_data(X)
prior_params = self.prior_params_
update_params = self._estimate_update_params(X)
self._check_posterior_evidence(
prior_params[0] + update_params[0],
prior_params[1] + update_params[1],
)
post_params = _combine_params(prior_params, update_params)
kappa_post, nu_post, mu_post, sigma_sq_post = post_params
loc = mu_post
scale = np.sqrt((1 + kappa_post) / kappa_post * sigma_sq_post)
# Approximate a point mass with a tiny positive scale and df > 2
# so the Student's t distribution has finite mean and variance.
# The zero-scale limit is independent of df.
is_point_mass = scale < np.finfo(float).tiny
df = np.where(is_point_mass, np.maximum(nu_post, 3), nu_post)
scale = np.maximum(scale, np.finfo(float).tiny)
return t(df=df, loc=loc, scale=scale)
def _check_posterior_evidence(self, kappa_post, nu_post):
"""Reject test samples the posterior says nothing about.
Parameters
----------
kappa_post : numpy.ndarray of shape (n_samples,)
Posterior weight on the target mean, per test sample.
nu_post : numpy.ndarray of shape (n_samples,)
Posterior weight on the target variance, per test sample.
Raises
------
ValueError
If any test sample carries no posterior weight on the target
mean or variance.
"""
for weight_post, name, prior_weight, quantity in [
(kappa_post, "kappa_0", self.kappa_0, "mean"),
(nu_post, "nu_0", self.nu_0, "variance"),
]:
without_evidence = weight_post <= 0
if np.any(without_evidence):
raise ValueError(
f"{np.sum(without_evidence)} of {len(weight_post)} test "
f"samples carry no evidence for the target {quantity}, "
"so it is undefined for them. The kernel gives them zero "
f"mass and `{name}={prior_weight}` puts no weight on the "
f"prior {quantity} either. Widen the kernel, e.g. with a "
"smaller 'gamma' in `metric_dict`, or use a positive "
f"`{name}` so that such samples fall back to the prior "
f"{quantity}."
)
def _combine_params(prior_params, update_params):
kappa_1, nu_1, mu_1, sigma_sq_1 = prior_params
kappa_2, nu_2, mu_2, sigma_sq_2 = update_params
kappa_com = kappa_1 + kappa_2
nu_com = nu_1 + nu_2
mu_com = (kappa_1 * mu_1 + kappa_2 * mu_2) / kappa_com
scatter_com = (
nu_1 * sigma_sq_1
+ nu_2 * sigma_sq_2
+ kappa_1 * kappa_2 * (mu_1 - mu_2) ** 2 / kappa_com
)
sigma_sq_com = scatter_com / nu_com
return kappa_com, nu_com, mu_com, sigma_sq_com
[docs]
class NadarayaWatsonRegressor(NICKernelRegressor):
"""Nadaraya Watson Regressor
The Nadaraya Watson Regressor predicts the target value by taking a
weighted average based on a kernel. It is implemented as a
`NICKernelRegressor` with different prior values.
Parameters
----------
metric : str or callable, default='rbf'
The metric must a be a valid kernel defined by the function
`sklearn.metrics.pairwise.pairwise_kernels`, or `'precomputed'`.
Its values weight the training targets, so it must not be negative
for the given data.
`'rbf'`, `'laplacian'` and `'chi2'` always satisfy this.
`'linear'`, `'poly'`, `'polynomial'`, `'sigmoid'` and `'cosine'` do
so only for some data, and `'additive_chi2'` never does because it
is non-positive by construction. Prediction raises a `ValueError`
when the resulting kernel evidence is negative.
metric_dict : dict, default=None
Any further parameters are passed directly to the kernel function.
missing_label : scalar or string or np.nan or or None, default=np.nan
Value to represent a missing label.
random_state : int or RandomState instance or None, default=None
Determines random number for `predict` method. Pass an int for
reproducible results across multiple method calls.
Notes
-----
Without observed targets, the Nadaraya-Watson estimate is undefined. This
estimator then returns a standard normal fallback distribution, with mean
zero and standard deviation one, until labeled data are fitted.
"""
def __init__(
self,
metric="rbf",
metric_dict=None,
missing_label=MISSING_LABEL,
random_state=None,
):
super().__init__(
random_state=random_state,
missing_label=missing_label,
metric=metric,
metric_dict=metric_dict,
kappa_0=0,
nu_0=3,
sigma_sq_0=1,
)
[docs]
def predict_target_distribution(self, X):
"""Return the estimated or fallback target distribution."""
check_is_fitted(self)
if len(self.X_) != 0:
return super().predict_target_distribution(X)
X = self._validate_prediction_data(X)
return norm(loc=np.zeros(len(X)), scale=np.ones(len(X)))