Source code for skactiveml.regressor._nic_kernel_regressor

import numpy as np

from copy import deepcopy
from scipy.stats import norm, t
from sklearn.metrics.pairwise import pairwise_kernels, KERNEL_PARAMS
from sklearn.utils import check_array
from sklearn.utils.validation import check_is_fitted

from skactiveml.base import ProbabilisticRegressor
from skactiveml.utils import (
    MISSING_LABEL,
    check_scalar,
    check_type,
    check_n_features,
)
from skactiveml.utils._label import _observed_numerical_labels


[docs] class NICKernelRegressor(ProbabilisticRegressor): """NIC Kernel Regressor The NICKernelRegressor (Normal Inverse Chi-square Kernel Regressor) locally fits a t-distribution using the training data, weighting the samples by a kernel. Parameters ---------- metric : str or callable, default='rbf' The metric must a be a valid kernel defined by the function `sklearn.metrics.pairwise.pairwise_kernels`, or `'precomputed'`. Its values weight the training targets, so it must not be negative for the given data. `'rbf'`, `'laplacian'` and `'chi2'` always satisfy this. `'linear'`, `'poly'`, `'polynomial'`, `'sigmoid'` and `'cosine'` do so only for some data, and `'additive_chi2'` never does because it is non-positive by construction. Prediction raises a `ValueError` when the resulting kernel evidence is negative. metric_dict : dict, default=None Any further parameters are passed directly to the kernel function. mu_0 : int or float, default=0 The prior mean. kappa_0 : int or float >= 0, default=0.1 The weight of the prior mean. sigma_sq_0 : int or float >= 0, default=1.0 The prior variance. nu_0 : int or float >= 0, default=2.5 The weight of the prior variance. missing_label : scalar or string or np.nan or None, default=np.nan Value to represent a missing label. random_state : int, RandomState instance or None, default=None Determines random number for `predict` method. Pass an int for reproducible results across multiple method calls. """ METRICS = list(KERNEL_PARAMS.keys()) + ["precomputed"] def __init__( self, metric="rbf", metric_dict=None, mu_0=0, kappa_0=0.1, sigma_sq_0=1.0, nu_0=2.5, missing_label=MISSING_LABEL, random_state=None, ): super().__init__( random_state=random_state, missing_label=missing_label, ) self.kappa_0 = kappa_0 self.nu_0 = nu_0 self.mu_0 = mu_0 self.sigma_sq_0 = sigma_sq_0 self.metric = metric self.metric_dict = metric_dict
[docs] def fit(self, X, y, sample_weight=None): """Fit the model using `X` as training data and `y` as labels. Parameters ---------- X : matrix-like of shape (n_samples, n_features) or \ (n_samples, n_samples) if metric='precomputed' Training data set, usually complete, i.e., including the labeled and unlabeled samples. If `metric='precomputed'`, `X` contains the pairwise kernels between all training samples. y : array-like of shape (n_samples,) or (n_samples, 1) Labels of the training data set (possibly including unlabeled ones indicated by `self.missing_label`). sample_weight : array-like of shape (n_samples,) It contains the weights of the training samples' values. Returns ------- self: SkactivemlRegressor, The SkactivemlRegressor is fitted on the training data. """ X, y, sample_weight = self._validate_data(X, y, sample_weight) if self.metric == "precomputed" and X.shape[0] != X.shape[1]: raise ValueError( "For metric='precomputed', the training kernel matrix `X` " "must have shape (n_train_samples, n_train_samples)." ) is_lbld, y_observed = _observed_numerical_labels( y, self.missing_label_ ) for value, name, min_val in [ (self.kappa_0, "self.kappa_0", 0), (self.nu_0, "self.nu_0", 0), (self.sigma_sq_0, "self.sigma_sq_0", 0), (self.mu_0, "self.mu_0", None), ]: check_scalar(value, name, (int, float), min_val=min_val) if not np.isfinite(value): raise ValueError(f"`{name}` must be finite, got {value}.") self._is_lbld = is_lbld self.X_ = X[is_lbld] self.y_ = y_observed self.prior_params_ = ( self.kappa_0, self.nu_0, self.mu_0, self.sigma_sq_0, ) if sample_weight is not None: self.weights_ = sample_weight[is_lbld] if len(self.weights_) > 0 and np.sum(self.weights_) == 0: raise ValueError( "The sample weights of the labeled samples " "must not be all zero." ) else: self.weights_ = None check_type( self.metric, "self.metric", target_vals=self.METRICS, indicator_funcs=[callable], ) check_type( self.metric_dict, "self.metric_dict", dict, target_vals=[None] ) self.metric_dict_ = ( deepcopy(self.metric_dict) if self.metric_dict is not None else {} ) return self
def _validate_prediction_data(self, X): """Validate feature data or a precomputed test kernel. Parameters ---------- X : array-like of shape (n_samples, n_features) or \ (n_samples, n_train_samples) if metric='precomputed' Input samples or kernels against all samples from the latest fit. Returns ------- X : numpy.ndarray Validated input data. """ X = check_array(X) if self.metric == "precomputed": if X.shape[1] != self.n_features_in_: raise ValueError( "For metric='precomputed', the kernel matrix `X` must " "have shape (n_test_samples, n_train_samples)." ) else: check_n_features(self, X, reset=False) return X def _estimate_ml_params(self, X): if self.metric == "precomputed": K = X[:, self._is_lbld] else: K = pairwise_kernels( X, self.X_, metric=self.metric, **self.metric_dict_ ) K = np.asarray(K, dtype=float) if self.weights_ is not None: K = self.weights_.reshape(1, -1) * K N = np.sum(K, axis=1) mu_ml = np.divide(K @ self.y_, N, out=np.zeros_like(N), where=N != 0) scatter = np.sum( K * (self.y_[np.newaxis, :] - mu_ml[:, np.newaxis]) ** 2, axis=1 ) self._check_kernel_evidence(N, "kernel mass") self._check_kernel_evidence( scatter, "weighted sum of squared deviations (scatter)" ) var_ml = np.divide(scatter, N, out=np.zeros_like(N), where=N != 0) return N, mu_ml, var_ml def _check_kernel_evidence(self, values, name): """Reject kernel evidence that cannot be a weighted count. Parameters ---------- values : numpy.ndarray of shape (n_samples,) Kernel-weighted quantity to validate. name : str Name of the quantity reported in the error message. Raises ------ ValueError If any value is negative. """ if np.any(values < 0): raise ValueError( f"The {name} weighting the training targets is negative, the " f"smallest being {np.min(values)}, so it cannot serve as a " "pseudo-count. This happens when `sample_weight` contains " f"negative values, or when `metric={self.metric!r}` produces " "negative similarities on the given data. Use non-negative " "sample weights and a non-negative kernel, e.g. 'rbf' or " "'laplacian', or supply a non-negative 'precomputed' matrix." ) def _estimate_update_params(self, X): if len(self.X_) != 0: N, mu_ml, var_ml = self._estimate_ml_params(X) return N, N, mu_ml, var_ml else: neutral_params = (np.zeros(len(X)),) * 4 return neutral_params
[docs] def predict_target_distribution(self, X): """Returns the estimated target distribution conditioned on the test samples `X`. Parameters ---------- X : array-like of shape (n_samples, n_features) or \ (n_samples, n_train_samples) if metric='precomputed' Input samples. If `metric='precomputed'`, `X` contains kernels between the test samples and every sample passed to the latest `fit` call. Returns ------- dist : scipy.stats._distn_infrastructure.rv_frozen The distribution of the targets at the test samples. A posterior scale of zero yields a point mass at the posterior mean. """ check_is_fitted(self) X = self._validate_prediction_data(X) prior_params = self.prior_params_ update_params = self._estimate_update_params(X) self._check_posterior_evidence( prior_params[0] + update_params[0], prior_params[1] + update_params[1], ) post_params = _combine_params(prior_params, update_params) kappa_post, nu_post, mu_post, sigma_sq_post = post_params loc = mu_post scale = np.sqrt((1 + kappa_post) / kappa_post * sigma_sq_post) # Approximate a point mass with a tiny positive scale and df > 2 # so the Student's t distribution has finite mean and variance. # The zero-scale limit is independent of df. is_point_mass = scale < np.finfo(float).tiny df = np.where(is_point_mass, np.maximum(nu_post, 3), nu_post) scale = np.maximum(scale, np.finfo(float).tiny) return t(df=df, loc=loc, scale=scale)
def _check_posterior_evidence(self, kappa_post, nu_post): """Reject test samples the posterior says nothing about. Parameters ---------- kappa_post : numpy.ndarray of shape (n_samples,) Posterior weight on the target mean, per test sample. nu_post : numpy.ndarray of shape (n_samples,) Posterior weight on the target variance, per test sample. Raises ------ ValueError If any test sample carries no posterior weight on the target mean or variance. """ for weight_post, name, prior_weight, quantity in [ (kappa_post, "kappa_0", self.kappa_0, "mean"), (nu_post, "nu_0", self.nu_0, "variance"), ]: without_evidence = weight_post <= 0 if np.any(without_evidence): raise ValueError( f"{np.sum(without_evidence)} of {len(weight_post)} test " f"samples carry no evidence for the target {quantity}, " "so it is undefined for them. The kernel gives them zero " f"mass and `{name}={prior_weight}` puts no weight on the " f"prior {quantity} either. Widen the kernel, e.g. with a " "smaller 'gamma' in `metric_dict`, or use a positive " f"`{name}` so that such samples fall back to the prior " f"{quantity}." )
def _combine_params(prior_params, update_params): kappa_1, nu_1, mu_1, sigma_sq_1 = prior_params kappa_2, nu_2, mu_2, sigma_sq_2 = update_params kappa_com = kappa_1 + kappa_2 nu_com = nu_1 + nu_2 mu_com = (kappa_1 * mu_1 + kappa_2 * mu_2) / kappa_com scatter_com = ( nu_1 * sigma_sq_1 + nu_2 * sigma_sq_2 + kappa_1 * kappa_2 * (mu_1 - mu_2) ** 2 / kappa_com ) sigma_sq_com = scatter_com / nu_com return kappa_com, nu_com, mu_com, sigma_sq_com
[docs] class NadarayaWatsonRegressor(NICKernelRegressor): """Nadaraya Watson Regressor The Nadaraya Watson Regressor predicts the target value by taking a weighted average based on a kernel. It is implemented as a `NICKernelRegressor` with different prior values. Parameters ---------- metric : str or callable, default='rbf' The metric must a be a valid kernel defined by the function `sklearn.metrics.pairwise.pairwise_kernels`, or `'precomputed'`. Its values weight the training targets, so it must not be negative for the given data. `'rbf'`, `'laplacian'` and `'chi2'` always satisfy this. `'linear'`, `'poly'`, `'polynomial'`, `'sigmoid'` and `'cosine'` do so only for some data, and `'additive_chi2'` never does because it is non-positive by construction. Prediction raises a `ValueError` when the resulting kernel evidence is negative. metric_dict : dict, default=None Any further parameters are passed directly to the kernel function. missing_label : scalar or string or np.nan or or None, default=np.nan Value to represent a missing label. random_state : int or RandomState instance or None, default=None Determines random number for `predict` method. Pass an int for reproducible results across multiple method calls. Notes ----- Without observed targets, the Nadaraya-Watson estimate is undefined. This estimator then returns a standard normal fallback distribution, with mean zero and standard deviation one, until labeled data are fitted. """ def __init__( self, metric="rbf", metric_dict=None, missing_label=MISSING_LABEL, random_state=None, ): super().__init__( random_state=random_state, missing_label=missing_label, metric=metric, metric_dict=metric_dict, kappa_0=0, nu_0=3, sigma_sq_0=1, )
[docs] def predict_target_distribution(self, X): """Return the estimated or fallback target distribution.""" check_is_fitted(self) if len(self.X_) != 0: return super().predict_target_distribution(X) X = self._validate_prediction_data(X) return norm(loc=np.zeros(len(X)), scale=np.ones(len(X)))