import numpy as np
from sklearn.utils import check_array
from ..base import SingleAnnotatorStreamQueryStrategy
from ..utils import check_scalar
from ..utils._validation import _validate_budget_update
# Label-free baselines consume neither labels nor a model. They declare the
# same task-agnostic capabilities as their pool-based counterparts.
_TASK_AGNOSTIC_CAPABILITIES = frozenset(
{
("classification", "single-output", "single-annotator"),
("classification", "multi-label", "single-annotator"),
("regression", "single-output", "single-annotator"),
}
)
def _validate_update_state(strategy):
"""Validate and commit the state an `update` needs.
Parameters
----------
strategy : SingleAnnotatorStreamQueryStrategy
The query strategy being updated.
"""
strategy._validate_random_state()
strategy._validate_budget()
if not hasattr(strategy, "observed_samples_"):
strategy.observed_samples_ = 0
if not hasattr(strategy, "queried_samples_"):
strategy.queried_samples_ = 0
[docs]
class StreamRandomSampling(SingleAnnotatorStreamQueryStrategy):
"""Random Sampling for Data Streams.
The RandomSampling strategy queries labels completely randomly. The
probability to query a sample is dependent on `budget`. Given a budget
of 10%, the utility exceeds 0.9 (1-0.1) with a probability of 10%.
Samples are queried regardless of their position in the feature space
and without using information about the sample. Thus, it should only be
used as a baseline strategy. The `allow_exceeding_budget` parameter
controls whether the strategy strictly adheres to a given budget.
Neither labels nor a model are consumed, so the strategy is task-agnostic
and supports single-output and multi-label classification as well as
regression.
Parameters
----------
allow_exceeding_budget : bool, default=True
If `True`, the query strategy is allowed to exceed it's budget as long
as the average number of queries will be within the budget. If `False`,
queries are not allowed if the budget is exhausted.
budget : float, default=None
Specifies the ratio of samples which may be queried, with
`0 < budget <= 1`. If `budget` is `None`, the default
budget 0.1 is used.
random_state : int or RandomState instance or None, default=None
Controls the randomness of the estimator.
"""
def __init__(
self, allow_exceeding_budget=True, budget=None, random_state=None
):
super().__init__(budget=budget, random_state=random_state)
self.allow_exceeding_budget = allow_exceeding_budget
@property
def _target_capabilities(self):
return _TASK_AGNOSTIC_CAPABILITIES
[docs]
def query(self, candidates, return_utilities=False):
"""Determines for which candidate samples labels are to be queried.
The query startegy determines the most useful samples in candidates,
which can be acquired within the budgeting constraint specified by
`budget`. Please note that, this method does not change the internal
state of the query strategy. To adapt the query strategy to the
selected candidates, use `update(...)`.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
return_utilities : bool, default=False
If `True`, also return the `utilities` based on the query strategy.
Returns
-------
queried_indices : np.ndarray of shape (n_queried_indices,)
The indices of samples in candidates whose labels are queried,
with `0 <= queried_indices < n_candidates`.
utilities : np.ndarray of shape (n_candidates,)
The utilities based on the query strategy. Only provided if
`return_utilities` is `True`.
"""
candidates, return_utilities = self._validate_data(
candidates, return_utilities
)
# copy random state in case of simulating the query
prior_random_state = self.random_state_.get_state()
utilities = self.random_state_.random_sample(len(candidates))
self.random_state_.set_state(prior_random_state)
queried = np.full(len(utilities), False)
# Track counts locally without advancing acquisition state.
tmp_observed_samples = self.observed_samples_
tmp_queried_samples = self.queried_samples_
for i, utility in enumerate(utilities):
tmp_observed_samples += 1
# Include exact boundaries that round one ULP below the limit.
budget_limit = np.nextafter(
tmp_observed_samples * self.budget_, np.inf
)
queried[i] = (
self.allow_exceeding_budget
or tmp_queried_samples + 1 <= budget_limit
) and (utility >= 1 - self.budget_)
tmp_queried_samples += queried[i]
queried_indices = np.where(queried)[0]
if return_utilities:
return queried_indices, utilities
else:
return queried_indices
[docs]
def update(
self, candidates, queried_indices, budget_manager_param_dict=None
):
"""Updates the count for seen and queried labels. This function should
be used in conjunction with the `query` function.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
queried_indices : array-like of shape (n_queried_indices,)
The indices of samples in candidates whose labels are queried,
with `0 <= queried_indices < n_candidates`.
budget_manager_param_dict : Ignored
Not used, present here for API consistency by convention.
Returns
-------
self : SingleAnnotatorStreamQueryStrategy
The query strategy returns itself, after it is updated.
"""
candidates = check_array(candidates)
queried = _validate_budget_update(candidates, queried_indices)
check_scalar(
self.allow_exceeding_budget, "allow_exceeding_budget", bool
)
_validate_update_state(self)
self.observed_samples_ += len(queried)
self.queried_samples_ += np.sum(queried)
# Advance the draws consumed by query.
self.random_state_.random_sample(len(candidates))
return self
def _validate_data(
self,
candidates,
return_utilities,
reset=True,
**check_candidates_params,
):
"""Validate input data and set or check the `n_features_in_` attribute.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
return_utilities : bool, default=False
If `True`, also return the utilities based on the query strategy.
reset : bool, default=True
Whether to reset the `n_features_in_` attribute. If False, the
input will be checked for consistency with data provided when reset
was last True.
**check_candidates_params : kwargs
Parameters passed to :func:`sklearn.utils.check_array`.
Returns
-------
candidates: np.ndarray, shape (n_candidates, n_features)
Checked candidate samples.
return_utilities : bool,
Checked boolean value of `return_utilities`.
"""
if not hasattr(self, "observed_samples_"):
self.observed_samples_ = 0
if not hasattr(self, "queried_samples_"):
self.queried_samples_ = 0
check_scalar(
self.allow_exceeding_budget, "allow_exceeding_budget", bool
)
candidates, return_utilities = super()._validate_data(
candidates,
return_utilities,
reset=reset,
**check_candidates_params,
)
self._validate_random_state()
return candidates, return_utilities
[docs]
class PeriodicSampling(SingleAnnotatorStreamQueryStrategy):
"""Periodic Sampling for Data Streams
The PeriodicSampling strategy samples labels periodically. The length of
that period is determined by the `budget`. For instance, a `budget` of 0.25
would result in querying every fourth sample. The main idea behind this
query strategy is to exhaust a given budget as soon as it is available.
Samples are queried regardless of their position in the feature space and
without using information about the sample. Thus, it should only be used
as a baseline strategy. Neither labels nor a model are consumed, so the
strategy is task-agnostic and supports single-output and multi-label
classification as well as regression.
Parameters
----------
budget : float, default=None
Specifies the ratio of samples which may be queried, with
`0 < budget <= 1`. If `budget` is `None`, the default
budget 0.1 is used.
random_state : int or RandomState instance or None, default=None
Controls the randomness of the estimator.
"""
def __init__(self, budget=None, random_state=None):
super().__init__(budget=budget, random_state=random_state)
@property
def _target_capabilities(self):
return _TASK_AGNOSTIC_CAPABILITIES
[docs]
def query(self, candidates, return_utilities=False):
"""Determines for which candidate samples labels are to be queried.
The query startegy determines the most useful samples in candidates,
which can be acquired within the budgeting constraint specified by
`budget`. Please note that, this method does not change the internal
state of the query strategy. To adapt the query strategy to the
selected candidates, use `update(...)`.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
return_utilities : bool, default=False
If `True`, also return the `utilities` based on the query strategy.
Returns
-------
queried_indices : np.ndarray of shape (n_queried_indices,)
The indices of samples in candidates whose labels are queried,
with `0 <= queried_indices < n_candidates`.
utilities : np.ndarray of shape (n_candidates,)
The utilities based on the query strategy. Only provided if
`return_utilities` is `True`.
"""
candidates, return_utilities = self._validate_data(
candidates, return_utilities
)
utilities = np.zeros(candidates.shape[0])
queried = np.full(len(candidates), False)
tmp_observed_samples = self.observed_samples_
tmp_queried_samples = self.queried_samples_
for i, x in enumerate(candidates):
tmp_observed_samples += 1
# Include exact boundaries that round one ULP below the limit.
budget_limit = np.nextafter(
tmp_observed_samples * self.budget_, np.inf
)
queried[i] = tmp_queried_samples + 1 <= budget_limit
if queried[i]:
utilities[i] = 1
tmp_queried_samples += queried[i]
queried_indices = np.where(queried)[0]
if return_utilities:
return queried_indices, utilities
else:
return queried_indices
[docs]
def update(
self, candidates, queried_indices, budget_manager_param_dict=None
):
"""Updates the count for seen and queried labels. This function should
be used in conjunction with the `query` function.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted only
if they are supported by the base query strategy.
queried_indices : array-like of shape (n_queried_indices,)
The indices of samples in candidates whose labels are queried, with
`0 <= queried_indices < n_candidates`.
budget_manager_param_dict : Ignored
Not used, present here for API consistency by convention.
Returns
-------
self : SingleAnnotatorStreamQueryStrategy
The query strategy returns itself, after it is updated.
"""
candidates = check_array(candidates)
queried = _validate_budget_update(candidates, queried_indices)
_validate_update_state(self)
self.observed_samples_ += len(queried)
self.queried_samples_ += np.sum(queried)
return self
def _validate_data(
self,
candidates,
return_utilities,
reset=True,
**check_candidates_params,
):
"""Validate input data and set or check the `n_features_in_` attribute.
Parameters
----------
candidates : {array-like, sparse matrix} of shape\
(n_candidates, n_features)
The samples which may be queried. Sparse matrices are accepted
only if they are supported by the base query strategy.
return_utilities : bool, default=False
If `True`, also return the utilities based on the query strategy.
reset : bool, default=True
Whether to reset the `n_features_in_` attribute. If False, the
input will be checked for consistency with data provided when reset
was last True.
**check_candidates_params : kwargs
Parameters passed to :func:`sklearn.utils.check_array`.
Returns
-------
candidates: np.ndarray, shape (n_candidates, n_features)
Checked candidate samples.
return_utilities : bool,
Checked boolean value of `return_utilities`.
"""
candidates, return_utilities = super()._validate_data(
candidates,
return_utilities,
reset=reset,
**check_candidates_params,
)
self._validate_random_state()
if not hasattr(self, "observed_samples_"):
self.observed_samples_ = 0
if not hasattr(self, "queried_samples_"):
self.queried_samples_ = 0
return candidates, return_utilities