import numpy as np
from scipy.stats import t, rankdata
from sklearn.base import BaseEstimator, clone
from sklearn.utils.validation import check_array, check_is_fitted
from ...base import (
MultiAnnotatorPoolQueryStrategy,
SkactivemlClassifier,
)
from ...utils._label_dtype import _as_label_array
from ...utils._target import _check_target_spec_capability
from ...pool._uncertainty_sampling import uncertainty_scores
from ...utils import (
check_scalar,
MISSING_LABEL,
is_labeled,
check_type,
simple_batch,
majority_vote,
resolve_target_spec,
check_missing_label,
)
def _check_classifier_prediction_contract(
classifier_target_spec, query_target_spec
):
has_conflicting_prediction_contract = (
classifier_target_spec.task != query_target_spec.task
or classifier_target_spec.target_type != query_target_spec.target_type
or classifier_target_spec.classes != query_target_spec.classes
)
if has_conflicting_prediction_contract:
raise ValueError(
"The classifier's task, target type, or class vocabulary "
"conflicts with IntervalEstimationThreshold's resolved outer "
"target specification."
)
[docs]
class IntervalEstimationAnnotModel(BaseEstimator):
"""IELearning
This annotator model relies on 'Interval Estimation Learning' (IELearning)
for estimating the annotation performances, i.e., labeling accuracies,
of multiple annotators [1]_. Therefore, it computes the mean accuracy and
the lower as well as the upper bound of the labeling accuracy per
annotator. (Weighted) majority vote is used as estimated ground truth.
Parameters
----------
classes : array-like of shape (n_classes,), default=None
Holds the label for each class.
missing_label : scalar or string or np.nan or None, default=np.nan
Value to represent a missing label.
alpha : float, interval=(0, 1), default=0.05
Significance level of the confidence interval based on Student's
t-distribution, i.e., the confidence level is `1 - alpha`.
mode : 'lower' or 'mean' or 'upper', default='upper'
Mode of the estimated annotation performance.
random_state : None or int or numpy.random.RandomState, default=None
The random state used for deciding on majority vote labels in case of
ties.
Attributes
----------
n_annotators_ : int
Number of annotators.
A_perf_ : ndarray, shape (n_annotators, 3)
Estimated annotation performances (i.e., labeling accuracies), where
`A_perf_[i, 0]` indicates the lower bound, `A_perf_[i, 1]` indicates
the mean, and `A_perf_[i, 2]` indicates the upper bound of the
estimated labeling accuracy.
target_spec_ : skactiveml.utils.TargetSpec
Resolved target semantics established during fitting.
References
----------
.. [1] P. Donmez, J. G. Carbonell, and J. Schneider. Efficiently Learning
the Accuracy of Labeling Sources for Selective Sampling. In ACM SIGKDD
Int. Conf. Knowl. Discov. Data Min., pages 259–268, 2009.
"""
def __init__(
self,
classes=None,
missing_label=MISSING_LABEL,
alpha=0.05,
mode="upper",
random_state=None,
):
self.classes = classes
self.missing_label = missing_label
self.alpha = alpha
self.mode = mode
self.random_state = random_state
@property
def _target_capabilities(self):
return frozenset(
{("classification", "single-output", "multi-annotator")}
)
[docs]
def fit(self, X, y, sample_weight=None):
"""Fit annotator model for given samples.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Test samples.
y : array-like of shape (n_samples, n_annotators)
Class labels of annotators.
sample_weight : array-like of shape (n_samples, n_annotators),\
default=None
Sample weight for each label and annotator.
Returns
-------
self : IntervalEstimationAnnotModel object
The fitted annotator model.
"""
y_array = _as_label_array(y)
check_missing_label(
self.missing_label, target_type=y_array.dtype, name="y"
)
target_spec = resolve_target_spec(
y,
task="classification",
annotation_type="multi-annotator",
classes=self.classes,
missing_label=self.missing_label,
)
_check_target_spec_capability(
type(self).__name__, target_spec, self._target_capabilities
)
if (
sample_weight is not None
and np.shape(sample_weight) != y_array.shape
):
raise ValueError(
"`sample_weight` must have the same shape as `y`."
)
# Check whether alpha is float in (0, 1).
check_scalar(
x=self.alpha,
target_type=float,
name="alpha",
min_val=0,
max_val=1,
min_inclusive=False,
max_inclusive=False,
)
# Check mode.
if self.mode not in ["lower", "mean", "upper"]:
raise ValueError("`mode` must be in `['lower', 'mean', `upper`].`")
# Compute majority vote labels.
y_mv = majority_vote(
y=y,
w=sample_weight,
classes=self.classes,
random_state=self.random_state,
missing_label=self.missing_label,
)
# Number of annotators.
self.n_annotators_ = y_array.shape[1]
is_lbld = is_labeled(y, missing_label=self.missing_label)
self.A_perf_ = np.zeros((self.n_annotators_, 3))
for a_idx in range(self.n_annotators_):
is_correct = np.equal(
y_mv[is_lbld[:, a_idx]], y_array[is_lbld[:, a_idx], a_idx]
)
is_correct = np.concatenate((is_correct, [0, 1]))
mean = np.mean(is_correct)
std = np.std(is_correct)
t_value = t.isf([self.alpha / 2], len(is_correct) - 1)[0]
t_value *= std / np.sqrt(len(is_correct))
self.A_perf_[a_idx, 0] = mean - t_value
self.A_perf_[a_idx, 1] = mean
self.A_perf_[a_idx, 2] = mean + t_value
self.target_spec_ = target_spec
return self
[docs]
def predict_annotator_perf(self, X):
"""Calculates the probability that an annotator provides the true label
for a given sample.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Test samples.
Returns
-------
P_annot : numpy.ndarray of shape (n_samples, n_annotators)
`P_annot[i,l]` is the probability, that annotator `l` provides the
correct class label for sample `X[i]`.
"""
check_is_fitted(self)
X = check_array(X)
if self.mode == "lower":
mode = 0
elif self.mode == "mean":
mode = 1
else:
mode = 2
return np.tile(self.A_perf_[:, mode], (len(X), 1))
[docs]
class IntervalEstimationThreshold(MultiAnnotatorPoolQueryStrategy):
"""Interval Estimation Threshold (IEThresh)
The strategy 'Interval Estimation Threshold' (IEThresh) [1]_ is useful for
addressing the exploration vs. exploitation trade-off when dealing with
multiple error-prone annotators in active learning. This class relies on
`IntervalEstimationAnnotModel` for estimating the annotation
performances, i.e., label accuracies, of multiple annotators. Samples are
selected based on 'Uncertainty Sampling' (US). The selected samples are
labeled by the annotators whose estimated annotation performances are equal
or greater than an adaptive threshold.
The features to be labeled are chosen by using uncertainty sampling with
a least confidence score.
The strategy assumes all annotators are available for each candidate
sample. Samples with any unavailable annotator are excluded from selection.
By default, only fully unlabeled samples are candidates. Samples with
partial annotations can be used for training, but are not selected again
by default.
Explicit candidates and availability masks define the query pool directly.
Callers must ensure that it represents previously unqueried samples to
follow the workflow in [1]_.
Parameters
----------
epsilon : float, interval=[0, 1], default=0.9
Parameter for specifying the adaptive threshold used for annotator
selection.
alpha : float, interval=(0, 1), default=0.05
Significance level of the confidence interval based on Student's
t-distribution, i.e., the confidence level is `1 - alpha`.
random_state : int or np.random.RandomState or None, default=None
The random state to use.
missing_label : scalar or string or np.nan or None, default=np.nan
Value to represent a missing label.
References
----------
.. [1] P. Donmez, J. G. Carbonell, and J. Schneider. Efficiently Learning
the Accuracy of Labeling Sources for Selective Sampling. In ACM SIGKDD
Int. Conf. Knowl. Discov. Data Min., pages 259–268, 2009.
"""
def __init__(
self,
epsilon=0.9,
alpha=0.05,
random_state=None,
missing_label=MISSING_LABEL,
):
super().__init__(
random_state=random_state,
missing_label=missing_label,
)
self.epsilon = epsilon
self.alpha = alpha
[docs]
def query(
self,
X,
y,
clf,
fit_clf=True,
candidates=None,
annotators=None,
sample_weight=None,
batch_size="adaptive",
return_utilities=False,
):
"""Determines which candidate sample is to be annotated by which
annotator.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Training data set, usually complete, i.e., including the labeled
and unlabeled samples.
y : array-like of shape (n_samples, n_annotators)
Labels of the training data set for each annotator (possibly
including missing annotations indicated by `self.missing_label`),
meaning that `y[i, j]` contains the label from annotator `j` for
sample `i`.
Partially annotated samples are supported as training data.
clf : skactiveml.base.SkactivemlClassifier
Model implementing the methods `fit` and `predict_proba`.
fit_clf : bool, default=True
Defines whether the classifier should be fitted on `X`, `y`, and
`sample_weight`.
candidates : None or array-like of shape (n_candidates), dtype=int or\
array-like of shape (n_candidates, n_features), default=None
Candidate samples to query. Explicit candidates define the query
pool independently of missing labels in `y`. See `annotators` for
the supported representations and availability restrictions.
annotators : None or array-like of shape (n_avl_annotators), dtype=int\
or array-like of shape (n_candidates, n_annotators),\
default=None
A candidate is eligible only if all annotators are available for
it. Availability determines eligibility before the performance
threshold selects which annotators to query.
- If `candidates=None` and `annotators=None`, only fully unlabeled
rows of `y` are eligible.
- If `candidates=None` and `annotators` is specified, all samples
of `X` are candidates and `n_candidates` equals `len(X)`.
The supplied availability replaces the missing-label mask.
- If `candidates` is of shape `(n_candidates,)` and of type int,
`candidates` is considered as the indices of the sample
candidates in `(X, y)`.
- If `candidates` is of shape (n_candidates, n_features), the
sample candidates are directly given in `candidates` (not
necessarily contained in `X`).
- If `annotators=None` and candidates are explicitly specified,
all annotators are considered available.
- If `annotators` is of shape (n_avl_annotators), and of type int,
`annotators` is considered as the indices of the available
annotators. A strict subset leaves no eligible samples.
- If `annotators` is a boolean array of shape `(n_candidates,
n_annotators)`, a sample is eligible only if all entries in its
row are `True`.
sample_weight : array-like, (n_samples, n_annotators), default=None
It contains the weights of the training samples' class labels.
It must have the same shape as `y`.
batch_size : 'adaptive' or int, default='adaptive'
The number of sample-annotator pairs selected in one AL cycle.
If 'adaptive' is set, the `batch_size` is determined based on the
annotation performances and the parameter `epsilon`.
return_utilities : bool, default=False
If `True`, also return the utilities based on the query strategy.
Returns
-------
query_indices : np.ndarray of shape (batch_size, 2)
The `query_indices` indicate which candidate sample pairs are to be
queried is, i.e., which candidate sample is to be annotated by
which annotator, e.g., `query_indices[:, 0]` indicates the selected
candidate samples and `query_indices[:, 1]` indicates the
respectively selected annotators.
- If `candidates` is `None` or of shape `(n_candidates,)`, the
indexing of refers to samples in `X`.
- If `candidates` is of shape `(n_candidates, n_features)`, the
indexing refers to samples in `candidates`.
utilities: numpy.ndarray of shape (batch_size, n_samples,\
n_annotators) or numpy.ndarray of shape (batch_size,\
n_candidates, n_annotators)
The utilities of all candidate samples w.r.t. to the available
annotators after each selected sample of the batch, e.g.,
`utilities[0, :, j]` indicates the utilities used for selecting
the first sample-annotator-pair (with indices `query_indices[0]`).
- If `candidates` is `None` or of shape `(n_candidates,)`, the
indexing refers to samples in `X`.
- If `candidates` is of shape `(n_candidates, n_features)`, the
indexing refers to samples in `candidates`.
"""
check_type(clf, "clf", SkactivemlClassifier)
check_type(fit_clf, "fit_clf", bool)
target_classes = getattr(clf, "classes_", clf.classes)
query_target_spec = self._resolve_target_spec(
y, classes=target_classes
)
if query_target_spec is None:
clf._resolve_target_spec(y)
elif fit_clf:
_check_target_spec_capability(
type(clf).__name__,
query_target_spec,
clf._target_capabilities,
)
elif hasattr(clf, "target_spec_"):
_check_classifier_prediction_contract(
clf.target_spec_, query_target_spec
)
else:
clf._resolve_target_spec(y, classes=query_target_spec.classes)
# base check
(
X,
y,
candidates,
annotators,
_,
return_utilities,
) = super()._validate_data(
X,
y,
candidates,
annotators,
1,
return_utilities,
reset=True,
classes=target_classes,
target_spec=query_target_spec,
)
X_cand, mapping, A_cand = self._transform_cand_annot(
candidates, annotators, X, y
)
# Check whether epsilon is float in [0, 1].
check_scalar(
x=self.epsilon,
target_type=float,
name="epsilon",
min_val=0,
max_val=1,
)
# Check whether alpha is float in (0, 1).
check_scalar(
x=self.alpha,
target_type=float,
name="alpha",
min_val=0,
max_val=1,
min_inclusive=False,
max_inclusive=False,
)
n_annotators = y.shape[1]
# IEThresh requires all annotators to be available for a candidate.
# With default candidates, this retains only fully unlabeled samples.
A_cand = np.repeat(
np.all(A_cand, axis=1).reshape(-1, 1), n_annotators, axis=1
)
# Fit classifier and compute uncertainties on candidate samples.
if fit_clf:
if sample_weight is None:
clf = clone(clf).fit(X, y)
else:
clf = clone(clf).fit(X, y, sample_weight)
if hasattr(clf, "target_spec_"):
classifier_target_spec = clf.target_spec_
else:
classifier_target_spec = clf._resolve_target_spec(
y, classes=query_target_spec.classes
)
_check_classifier_prediction_contract(
classifier_target_spec, query_target_spec
)
P = clf.predict_proba(X_cand)
uncertainties = uncertainty_scores(probas=P, method="least_confident")
# Fit annotator model and compute performance estimates.
ie_model = IntervalEstimationAnnotModel(
classes=clf.classes_,
missing_label=clf.missing_label,
alpha=self.alpha,
mode="upper",
random_state=self.random_state_,
)
ie_model.fit(X=X, y=y, sample_weight=sample_weight)
A_perf = ie_model.A_perf_
# Compute utilities.
# combine the values of A_perf and uncertainties
A_perf = A_perf[:, 2] + 1
A_perf = A_perf[np.newaxis]
max_range = np.max(A_perf) + 1
rand_permutation = self.random_state_.permutation(len(uncertainties))
uncertainties[rand_permutation] = (
rankdata(uncertainties[rand_permutation], method="ordinal")
* max_range
)
uncertainties = np.tile(uncertainties, (n_annotators, 1)).T
utilities = uncertainties + A_perf
# exclude not available annotators
utilities[~A_cand] = np.nan
# Determine actual batch size.
if isinstance(batch_size, str) and batch_size != "adaptive":
raise ValueError(
"If `batch_size` is of type `string`, "
"it must equal `'adaptive'`."
)
elif batch_size == "adaptive":
required_perf = self.epsilon * np.max(A_perf)
actl_batch_size = int(np.sum(A_perf >= required_perf))
elif isinstance(batch_size, int):
actl_batch_size = batch_size
else:
raise TypeError(
f"`batch_size` is of type `{type(batch_size)}` "
f"but must equal `'adaptive'` or be of type "
f"`int`."
)
if mapping is not None:
w_utilities = utilities
utilities = np.full((len(X), n_annotators), np.nan)
utilities[mapping, :] = w_utilities
# Perform selection based on previously computed utilities.
return simple_batch(
utilities,
self.random_state_,
batch_size=actl_batch_size,
return_utilities=return_utilities,
)