import numpy as np
from sklearn.utils import check_array
from ._target import _fit_and_resolve_clf_target_spec
from ._validation import (
_validate_X_y_sample_weight,
_validate_budget_manager,
)
from .budgetmanager import (
FixedUncertaintyBudgetManager,
VariableUncertaintyBudgetManager,
SplitBudgetManager,
RandomVariableUncertaintyBudgetManager,
)
from ..base import SingleAnnotatorStreamQueryStrategy
from ..utils import call_func
from ..utils._validation import _validate_budget_update
[docs]
class UncertaintyZliobaite(SingleAnnotatorStreamQueryStrategy):
"""Base class for the uncertainty sampling strategies proposed by Žliobaitė
et al. in [1]_.
The UncertaintyZliobaite class provides the base for query strategies
proposed by Žliobaitė et al. in [1]_. The strategies evaluate the
classifier's uncertainty based on its predictions and samples' labels are
queried when the uncertainty exceeds a specific threshold. Žliobaitė et al.
propose various techniques to calculate such a threshold.
Parameters
----------
budget_manager : BudgetManager, default=None
The BudgetManager which models the budgeting constraint used in the
stream-based active learning setting. if set to `None`, a default
budger manager will be used. The budget manager will be initialized
based on the following conditions:
- If only a `budget` is given, the default budget manager is
initialized with the given budget.
- If only a budget manager is given, use the budget manager.
- If both are not given, the default budget manager with the default
budget.
- If both are given, and the budget differs from
`budgetmanager.budget`, throw a warning and the budget manager is
used as is.
budget : float, default=None
Specifies the ratio of samples which are allowed to be sampled, with
`0 < budget <= 1`. If `budget` is `None`, it is replaced with the
default budget 0.1.
random_state : int or RandomState instance, default=None
Controls the randomness of the estimator.
References
----------
.. [1] I. Žliobaitė, A. Bifet, B. Pfahringer, and G. Holmes. Active
Learning With Drifting Streaming Data. IEEE Trans. Neural Netw. Learn.
Syst., 25(1):27–39, 2014.
"""
def __init__(
self,
budget_manager=None,
budget=None,
random_state=None,
):
super().__init__(budget=budget, random_state=random_state)
self.budget_manager = budget_manager
[docs]
def query(
self,
candidates,
clf,
X=None,
y=None,
sample_weight=None,
fit_clf=False,
return_utilities=False,
):
"""Determines for which candidate samples labels are to be queried.
The query startegy determines the most useful samples in candidates,
which can be acquired within the budgeting constraint specified by
`budget`. Please note that, this method does not change the internal
state of the query strategy. To adapt the query strategy to the
selected candidates, use `update(...)`.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
clf : skactiveml.base.SkactivemlClassifier
Model implementing the methods `fit` and `predict_proba`.
X : array-like of shape (n_samples, n_features), default=None
Training data set used to fit the classifier.
y : array-like of shape (n_samples,)
Labels of the training data set (possibly including unlabeled ones
indicated by `clf.missing_label`).
sample_weight : array-like of shape (n_samples,), default=None
Weights of training samples in `X`.
fit_clf : bool, default=False
Defines whether the classifier should be fitted on `X`, `y`, and
`sample_weight`.
return_utilities : bool, default=False
If `True`, also return the `utilities` based on the query strategy.
Returns
-------
queried_indices : list of int or ndarray of shape (n_queried_indices,)
The indices of samples in candidates whose labels are queried,
with `0 <= queried_indices < n_candidates`.
The default budget manager returns a list.
utilities : np.ndarray of shape (n_candidates,)
The utilities based on the query strategy. Only provided if
`return_utilities` is `True`.
"""
(
candidates,
clf,
X,
y,
sample_weight,
fit_clf,
return_utilities,
) = self._validate_data(
candidates,
clf=clf,
X=X,
y=y,
sample_weight=sample_weight,
fit_clf=fit_clf,
return_utilities=return_utilities,
)
predict_proba = clf.predict_proba(candidates)
confidence = np.max(predict_proba, axis=1)
utilities = 1 - confidence
queried_indices = self.budget_manager_.query_by_utility(utilities)
if return_utilities:
return queried_indices, utilities
else:
return queried_indices
[docs]
def update(
self, candidates, queried_indices, budget_manager_param_dict=None
):
"""Updates the budget manager and the count for seen and queried
labels. This function should be used in conjunction with the `query`
function.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
queried_indices : array-like of shape (n_queried_indices,)
The indices of samples in candidates whose labels are queried,
with `0 <= queried_indices < n_candidates`.
budget_manager_param_dict : dict, default=None
Optional kwargs for `budget_manager`. If supplied, `utilities`
must be a one-dimensional array with one value per candidate.
Returns
-------
self : SingleAnnotatorStreamQueryStrategy
The query strategy returns itself, after it is updated.
"""
candidates = check_array(candidates)
_validate_budget_update(
candidates, queried_indices, budget_manager_param_dict
)
_validate_budget_manager(
self,
self._get_default_budget_manager(),
self._get_default_budget_manager_kwargs(),
)
budget_manager_param_dict = (
{}
if budget_manager_param_dict is None
else budget_manager_param_dict
)
call_func(
self.budget_manager_.update,
candidates=candidates,
queried_indices=queried_indices,
**budget_manager_param_dict,
)
return self
def _validate_data(
self,
candidates,
clf,
X,
y,
sample_weight,
fit_clf,
return_utilities,
reset=True,
**check_candidates_params,
):
"""Validate input data and set or check the `n_features_in_` attribute.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
clf : skactiveml.base.SkactivemlClassifier
Model implementing the methods `fit` and `predict_proba`.
X : array-like of shape (n_samples, n_features), default=None
Training data set used to fit the classifier.
y : array-like of shape (n_samples,)
Labels of the training data set (possibly including unlabeled ones
indicated by `clf.missing_label`).
sample_weight : array-like of shape (n_samples,), default=None
Weights of training samples in `X`.
fit_clf : bool, default=False
Defines whether the classifier should be fitted on `X`, `y`, and
`sample_weight`.
return_utilities : bool, default=False
If `True`, also return the utilities based on the query strategy.
reset : bool, default=True
Whether to reset the `n_features_in_` attribute. If False, the
input will be checked for consistency with data provided when reset
was last True.
**check_candidates_params : kwargs
Parameters passed to :func:`sklearn.utils.check_array`.
Returns
-------
candidates: np.ndarray, shape (n_candidates, n_features)
Checked candidate samples.
clf : SkactivemlClassifier
Checked model implementing the methods `fit` and `predict_freq`.
X: np.ndarray, shape (n_samples, n_features)
Checked training data set.
y: np.ndarray, shape (n_samples)
Checked training labels.
sampling_weight: np.ndarray, shape (n_candidates)
Checked training sample weight.
fit_clf : bool,
Checked boolean value of `fit_clf`.
return_utilities : bool,
Checked boolean value of `return_utilities`.
"""
X, y, sample_weight = _validate_X_y_sample_weight(
X=X, y=y, sample_weight=sample_weight
)
clf, _ = _fit_and_resolve_clf_target_spec(
self, clf, X, y, fit_clf=fit_clf, sample_weight=sample_weight
)
candidates, return_utilities = super()._validate_data(
candidates,
return_utilities,
reset=reset,
**check_candidates_params,
)
_validate_budget_manager(
self,
self._get_default_budget_manager(),
self._get_default_budget_manager_kwargs(),
)
return candidates, clf, X, y, sample_weight, fit_clf, return_utilities
def _get_default_budget_manager_kwargs(self):
"""Provide the kwargs for the budget manager that will be used as
default.
Returns
-------
default_budget_manager_kwargs : dict
The arguments necessary to initialize the budget manager.
"""
return {}
[docs]
class FixedUncertainty(UncertaintyZliobaite):
"""Fixed Uncertainty Strategy
The FixedUncertainty (Fixed Uncertainty Strategy in [1]_) query strategy
queries samples based on the classifiers uncertainty that is assessed
based on the classifier's predictions. The sample is queried when the
probability of the most likely class does not exceed a threshold
calculated based on the budget and the number of classes. See also
:class:`.budgetmanager.FixedUncertaintyBudgetManager`
Parameters
----------
classes : array-like of shape (n_classes,)
Holds the label for each class.
budget_manager : BudgetManager, default=None
The BudgetManager which models the budgeting constraint used in the
stream-based active learning setting. if set to `None`,
`FixedUncertaintyBudgetManager` will be used by default. The budget
manager will be initialized based on the following conditions:
- If only a `budget` is given, the default budget manager is
initialized with the given budget.
- If only a budget manager is given, use the budget manager.
- If both are not given, the default budget manager with the default
budget.
- If both are given, and the budget differs from
`budgetmanager.budget`, throw a warning and the budget manager is
used as is.
budget : float, default=None
Specifies the ratio of samples which are allowed to be sampled, with
`0 < budget <= 1`. If `budget` is `None`, it is replaced with the
default budget 0.1.
random_state : int or RandomState instance, default=None
Controls the randomness of the estimator.
References
----------
.. [1] I. Žliobaitė, A. Bifet, B. Pfahringer, and G. Holmes. Active
Learning With Drifting Streaming Data. IEEE Trans. Neural Netw. Learn.
Syst., 25(1):27–39, 2014.
"""
def __init__(
self,
classes,
budget_manager=None,
budget=None,
random_state=None,
):
super().__init__(
budget_manager=budget_manager,
budget=budget,
random_state=random_state,
)
self.classes = classes
def _get_default_budget_manager(self):
"""Provide the budget manager that will be used as default.
Returns
-------
budget_manager : BudgetManager
The BudgetManager that should be used by default.
"""
return FixedUncertaintyBudgetManager
def _get_default_budget_manager_kwargs(self):
"""Provide the kwargs for the budget manager that will be used as
default.
Returns
-------
default_budget_manager_kwargs : dict
The arguments necessary to initialize the budget manager.
"""
return {"classes": self.classes}
[docs]
class VariableUncertainty(UncertaintyZliobaite):
"""Variable Uncertainty Strategy
The VariableUncertainty query strategy (Variable Uncertainty Strategy in
[1]_) queries labels based on the classifiers uncertainty assessed based on
the classifier's predictions. The sample is queried when the probability
of the most likely class is lower than a time-dependent threshold
calculated based on the budget, number of observed and acquired samples.
See also
:class:`.budgetmanager.VariableUncertaintyBudgetManager`
Parameters
----------
budget_manager : BudgetManager, default=None
The BudgetManager which models the budgeting constraint used in the
stream-based active learning setting. if set to `None`,
`VariableUncertaintyBudgetManager` will be used by default. The budget
manager will be initialized based on the following conditions:
- If only a `budget` is given, the default budget manager is
initialized with the given budget.
- If only a budget manager is given, use the budget manager.
- If both are not given, the default budget manager with the default
budget.
- If both are given, and the budget differs from
`budgetmanager.budget`, throw a warning and the budget manager is
used as is.
budget : float, default=None
Specifies the ratio of samples which are allowed to be sampled, with
`0 < budget <= 1`. If `budget` is `None`, it is replaced with the
default budget 0.1.
random_state : int or RandomState instance, default=None
Controls the randomness of the estimator.
References
----------
.. [1] I. Žliobaitė, A. Bifet, B. Pfahringer, and G. Holmes. Active
Learning With Drifting Streaming Data. IEEE Trans. Neural Netw. Learn.
Syst., 25(1):27–39, 2014.
"""
def _get_default_budget_manager(self):
"""Provide the budget manager that will be used as default.
Returns
-------
budget_manager : BudgetManager
The BudgetManager that should be used by default.
"""
return VariableUncertaintyBudgetManager
[docs]
class RandomVariableUncertainty(UncertaintyZliobaite):
"""RandomVariableUncertainty
The RandomVariableUncertainty (Uncertainty Strategy With Randomization in
[1]_) query strategy samples samples based on the classifier's
uncertainty assessed based on the classifier's predictions. The sample is
queried when the probability of the most likely class is lower than a
time-dependent threshold calculated based on the budget, and the number of
observed and acquired samples. The threshold is randomized by being
multiplied with a random number sampled from N(1,delta). See also
:class:`.budgetmanager.RandomVariableUncertaintyBudgetManager`
Parameters
----------
budget_manager : BudgetManager, default=None
The BudgetManager which models the budgeting constraint used in the
stream-based active learning setting. if set to `None`,
`RandomVariableUncertaintyBudgetManager` will be used by default. The
budget manager will be initialized based on the following conditions:
- If only a `budget` is given, the default budget manager is
initialized with the given budget.
- If only a budget manager is given, use the budget manager.
- If both are not given, the default budget manager with the default
budget.
- If both are given, and the budget differs from
`budgetmanager.budget`, throw a warning and the budget manager is
used as is.
budget : float, default=None
Specifies the ratio of samples which are allowed to be sampled, with
`0 < budget <= 1`. If `budget` is `None`, it is replaced with the
default budget 0.1.
random_state : int or RandomState instance, default=None
Controls the randomness of the estimator.
References
----------
.. [1] I. Žliobaitė, A. Bifet, B. Pfahringer, and G. Holmes. Active
Learning With Drifting Streaming Data. IEEE Trans. Neural Netw. Learn.
Syst., 25(1):27–39, 2014.
"""
def _get_default_budget_manager(self):
"""Provide the budget manager that will be used as default.
Returns
-------
budget_manager : BudgetManager
The BudgetManager that should be used by default.
"""
return RandomVariableUncertaintyBudgetManager
[docs]
class Split(UncertaintyZliobaite):
"""Split
The Split query strategy (Split Strategy in [1]_) queries labels based on
the classifiers uncertainty assessed based on the classifier's predictions.
The sample is queried when the probability of the most likely class is
lower than a time-dependent threshold calculated based on the budget,
number of observed and acquired samples. It is a hybrid strategy that
combines `VariableUncertainty` with randomly sampling samples with a given
probability. See also
:class:`.budgetmanager.SplitBudgetManager`
Parameters
----------
budget_manager : BudgetManager, default=None
The BudgetManager which models the budgeting constraint used in the
stream-based active learning setting. if set to `None`,
`SplitBudgetManager` will be used by default. The budget manager will
be initialized based on the following conditions:
- If only a `budget` is given, the default budget manager is
initialized with the given budget.
- If only a budget manager is given, use the budget manager.
- If both are not given, the default budget manager with the default
budget.
- If both are given, and the budget differs from
`budgetmanager.budget`, throw a warning and the budget manager is
used as is.
budget : float, default=None
Specifies the ratio of samples which are allowed to be sampled, with
`0 < budget <= 1`. If `budget` is `None`, it is replaced with the
default budget 0.1.
random_state : int or RandomState instance, default=None
Controls the randomness of the estimator.
References
----------
.. [1] I. Žliobaitė, A. Bifet, B. Pfahringer, and G. Holmes. Active
Learning With Drifting Streaming Data. IEEE Trans. Neural Netw. Learn.
Syst., 25(1):27–39, 2014
"""
def _get_default_budget_manager(self):
"""Provide the budget manager that will be used as default.
Returns
-------
budget_manager : BudgetManager
The BudgetManager that should be used by default.
"""
return SplitBudgetManager