from ..base import SingleAnnotatorPoolQueryStrategy
from ..utils import (
MISSING_LABEL,
check_random_state,
check_equal_missing_label,
is_labeled,
labeled_indices,
unlabeled_indices,
check_scalar,
simple_batch,
match_signature,
)
from math import ceil
import numpy as np
from joblib import Parallel, delayed, cpu_count
from sklearn import clone
import warnings
# Query arguments aligned to the candidate samples if `candidates` contains
# samples instead of indices.
_CANDIDATE_ALIGNED_KWARGS = ["sample_weight_candidates", "utility_weight"]
def _subset_query_kwargs(
query_kwargs, names, indices, n_aligned, n_aligned_name
):
"""Subset the sample-aligned arguments in `names` by `indices`.
Check lengths against `n_aligned`, using `n_aligned_name` in errors.
Return a new dictionary without changing caller arrays. Scalars are
forwarded for validation by the wrapped strategy.
"""
query_kwargs = dict(query_kwargs)
for name in names:
value = query_kwargs.get(name)
if np.ndim(value) == 0:
continue
value = np.asarray(value)
if len(value) != n_aligned:
raise ValueError(
f"`{name}` must have length `{n_aligned_name}` but "
f"{len(value)} != {n_aligned}."
)
query_kwargs[name] = value[indices]
return query_kwargs
class _TargetPreservingWrapper(SingleAnnotatorPoolQueryStrategy):
"""Base class for wrappers passing `y` unchanged to the wrapped strategy.
The wrapped strategy resolves the target type of `y`. A wrapper only
separates labeled from unlabeled samples, where each row of a
two-dimensional `y` is one sample.
"""
@property
def _target_capabilities(self):
return getattr(
self.query_strategy, "_target_capabilities", frozenset()
)
def _check_wrapped_strategy(self):
"""Check the wrapped strategy's type and missing label."""
if not isinstance(
self.query_strategy, SingleAnnotatorPoolQueryStrategy
):
raise TypeError(
f"`query_strategy` is of type `{type(self.query_strategy)}` "
f"but must be of type `SingleAnnotatorPoolQueryStrategy`."
)
check_equal_missing_label(
self.query_strategy.missing_label, self.missing_label
)
@staticmethod
def _mask_type(y):
"""Return the target type whose label masks have one entry per row."""
return "multi-label" if np.ndim(y) == 2 else "single-output"
[docs]
class SubSamplingWrapper(_TargetPreservingWrapper):
"""Sub-sampling Wrapper
This class implements a wrapper for single-annotator pool-based strategies
that randomly sub-samples a set of candidates before computing their
utilities. This is useful when the number of available candidates is too
large and a small subset of candidates is sufficient to select a good batch
for labeling. The number of candidates can be controlled using
`max_candidates` which supports an absolute number or a fraction of the
available candidates. Additionally, `exclude_non_subsample` provides an
option to mask all candidates that were not included in the subsample. This
can further improve the runtime for query strategies that utilize all
available unlabeled data in their selection.
Each row of a two-dimensional `y` is one sample, so it must be either
fully labeled or fully unlabeled.
Parameters
----------
query_strategy : skactiveml.base.SingleAnnotatorPoolQueryStrategy
The strategy used for computing the utilities of the candidate
sub-sample.
max_candidates : int or float, default=0.1
Determines the number of candidates. If `max_candidates` is an
integer, `max_candidates` is the maximum number of candidates whose
utilities are computed. If `max_candidates` is a float,
`max_candidates` is the fraction of the original number of candidates.
exclude_non_subsample : bool, default=False
- If `True`, unlabeled candidates in `X` and `y` are excluded which
are not part of the subsample. If `candidates` is an array-like of
shape `(n_candidates, n_features)`, all unlabeled data will be
removed from `X` and `y`.
- If `False`, `X` and `y` stay the same.
embed_samples_func : Callable or None, default=None
- If `embed_samples_func` is a `Callable`, it must accept the samples
`X` as input and return the sample-wise embeddings as an at least
two-dimensional array, preserving the number and order of samples,
whereas the remaining dimensions may change. It is applied to
`candidates`, too, if they are passed as samples instead of
indices, and to a sample set passed as the query argument
`X_eval`, such that all samples share a single feature space.
An estimator that is passed as an already fitted one, i.e., with a
query argument such as `fit_clf=False`, must have been fitted in
this embedded space, because `query_strategy` scores the embedded
samples with it.
- If `embed_samples_func` is None, no action is performed.
missing_label : scalar or string or np.nan or None, default=np.nan
Value to represent a missing label.
random_state : int or np.random.RandomState, default=None
The random state to use.
"""
def __init__(
self,
query_strategy=None,
max_candidates=0.1,
exclude_non_subsample=False,
embed_samples_func=None,
missing_label=MISSING_LABEL,
random_state=None,
):
super().__init__(
missing_label=missing_label,
random_state=random_state,
)
self.query_strategy = query_strategy
self.max_candidates = max_candidates
self.exclude_non_subsample = exclude_non_subsample
self.embed_samples_func = embed_samples_func
[docs]
@match_signature("query_strategy", "query")
def query(
self,
X,
y,
candidates=None,
batch_size=1,
return_utilities=False,
**query_kwargs,
):
"""Determines for which candidate samples labels are to be queried.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Training data set, usually complete, i.e., including the labeled
and unlabeled samples.
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
Labels of the training data set (possibly including unlabeled ones
indicated by `self.missing_label`). If `y` is two-dimensional, a
row `y[i]` must either contain only observed labels or only
`missing_label` values, i.e., no mixing within a row.
candidates : None or array-like of shape (n_candidates), dtype=int or\
array-like of shape (n_candidates, n_features), default=None
- If `candidates` is `None`, the unlabeled samples from `(X,y)` are
considered as `candidates`.
- If `candidates` is of shape `(n_candidates,)` and of type
`int`, `candidates` is considered as the indices of the
samples in `(X,y)`.
- If `candidates` is of shape `(n_candidates, ...)`, the
candidate samples are directly given in `candidates` (not
necessarily contained in `X`). This is not supported by all
query strategies.
batch_size : int, default=1
The number of samples to be selected in one AL cycle.
return_utilities : bool, default=False
If `True`, also return the utilities based on the query strategy.
**query_kwargs : dict-like
Further keyword arguments are passed to the `query` method of the
`query_strategy` object.
Returns
-------
query_indices : numpy.ndarray of shape (batch_size,)
The query indices indicate for which candidate sample a label is
to be queried, e.g., `query_indices[0]` indicates the first
selected sample.
- If `candidates` is `None` or of shape
`(n_candidates,)`, the indexing refers to the samples in
`X`.
- If `candidates` is of shape `(n_candidates, n_features)`,
the indexing refers to the samples in `candidates`.
utilities : numpy.ndarray of shape (batch_size, n_samples) or \
numpy.ndarray of shape (batch_size, n_candidates)
The utilities of samples after each selected sample of the batch,
e.g., `utilities[0]` indicates the utilities used for selecting
the first sample (with index `query_indices[0]`) of the batch.
Utilities for labeled samples will be set to np.nan.
- If `candidates` is `None` or of shape
`(n_candidates,)`, the indexing refers to the samples in
`X`.
- If `candidates` is of shape `(n_candidates, n_features)`,
the indexing refers to the samples in `candidates`.
"""
self._check_wrapped_strategy()
# A clone keeps results cached for one `X`, e.g., distances, out of a
# later query with other samples.
query_strategy = clone(self.query_strategy)
mask_type = self._mask_type(y)
X, y, candidates, batch_size, return_utilities = self._validate_data(
X,
y,
candidates,
batch_size,
return_utilities,
reset=True,
target_type=mask_type,
)
check_scalar(self.exclude_non_subsample, "exclude_non_subsample", bool)
is_lbld = is_labeled(
y=y,
missing_label=self.missing_label_,
target_type=mask_type,
)
seed_multiplier = int(is_lbld.sum() + 1)
max_candidates = self.max_candidates
if isinstance(self.max_candidates, int):
check_scalar(
self.max_candidates,
name="max_candidates",
target_type=int,
min_inclusive=True,
min_val=1,
)
elif isinstance(self.max_candidates, float):
check_scalar(
self.max_candidates,
name="max_candidates",
target_type=float,
min_inclusive=False,
max_inclusive=True,
min_val=0.0,
max_val=1.0,
)
else:
raise TypeError(
f"`max_candidates` is of type `{type(self.max_candidates)}`"
f" but must be in `[int, float]`."
)
if self.embed_samples_func is not None and not callable(
self.embed_samples_func
):
raise TypeError(
"`embed_samples_func` must be either a `Callable` or `None`."
)
random_state = check_random_state(self.random_state, seed_multiplier)
# subsampling with no explicit provided candidates
if candidates is None:
candidate_indices = unlabeled_indices(
y=y,
missing_label=self.missing_label_,
target_type=mask_type,
)
# transform max_candidates to int if a ratio is given
if isinstance(max_candidates, float):
max_candidates = ceil(
len(candidate_indices) * self.max_candidates
)
max_candidates = min(max_candidates, len(candidate_indices))
# subsample new candidates
new_candidates = random_state.choice(
a=candidate_indices, size=max_candidates, replace=False
)
# subsampling with provided explicit candidates
else:
# transform max_candidates to int if a ratio is given
if isinstance(max_candidates, float):
max_candidates = ceil(len(candidates) * self.max_candidates)
max_candidates = min(max_candidates, len(candidates))
if candidates.ndim == 1:
candidate_indices = candidates
# subsample new candidates
new_candidates = random_state.choice(
a=candidates, size=max_candidates, replace=False
)
else:
candidate_indices = range(len(candidates))
# subsample new candidates
new_candidate_indices = random_state.choice(
a=candidate_indices, size=max_candidates, replace=False
)
new_candidates = candidates[new_candidate_indices]
# check if to exclude unlabeled non-candidate training data
if self.exclude_non_subsample:
all_labeled = labeled_indices(
y=y,
missing_label=self.missing_label_,
target_type=mask_type,
)
if candidates is not None and candidates.ndim > 1:
subset_and_labeled_indices = all_labeled
else:
# ignore labeled candidates to avoid duplicate samples
all_labeled = np.setdiff1d(all_labeled, new_candidates)
subset_and_labeled_indices = np.concatenate(
[all_labeled, new_candidates]
)
sorted_idx = np.argsort(subset_and_labeled_indices)
subset_and_labeled_indices = subset_and_labeled_indices[sorted_idx]
new_X = X[subset_and_labeled_indices]
new_y = y[subset_and_labeled_indices]
# for explicitly provided candidates recalculate candidate indices
# that are passed to the wrapped query strategy
if candidates is None or candidates.ndim == 1:
new_candidates = np.flatnonzero(sorted_idx >= len(all_labeled))
else:
new_X = X
new_y = y
if candidates is not None and candidates.ndim > 1:
X_aligned_kwargs = ["sample_weight"]
# `utility_weight` weights the samples whose utilities are
# computed, which are the candidate samples in this case.
query_kwargs = _subset_query_kwargs(
query_kwargs,
_CANDIDATE_ALIGNED_KWARGS,
new_candidate_indices,
len(candidates),
"n_candidates",
)
else:
X_aligned_kwargs = ["sample_weight", "utility_weight"]
if self.exclude_non_subsample:
query_kwargs = _subset_query_kwargs(
query_kwargs,
X_aligned_kwargs,
subset_and_labeled_indices,
len(X),
"n_samples",
)
if self.embed_samples_func:
new_X = self._embed_samples(new_X, "X")
if candidates is not None and candidates.ndim > 1:
new_candidates = self._embed_samples(
new_candidates, "candidates"
)
# A keyword argument holding an independent sample set is
# embedded but never subset, since it is not aligned to `X`.
X_eval = query_kwargs.get("X_eval")
if X_eval is not None:
query_kwargs["X_eval"] = self._embed_samples(X_eval, "X_eval")
qs_output = query_strategy.query(
X=new_X,
y=new_y,
candidates=new_candidates,
batch_size=batch_size,
return_utilities=return_utilities,
**query_kwargs,
)
# unpack result of query strategy if needed
queried_indices = qs_output
utilities = None
if return_utilities:
queried_indices, utilities = qs_output
effective_batch_size = len(np.atleast_1d(queried_indices))
# retransform queried indices and utilities as if no training data was
# removed
if self.exclude_non_subsample and (
candidates is None or candidates.ndim == 1
):
# transform to original candidate indices
queried_indices = subset_and_labeled_indices[queried_indices]
# transform to original utilities shape
if utilities is not None:
new_utilities = np.full(
shape=(effective_batch_size, len(X)), fill_value=np.nan
)
transformed_new_candidates = subset_and_labeled_indices[
new_candidates
]
new_utilities[:, transformed_new_candidates] = utilities[
:, new_candidates
]
utilities = new_utilities
new_candidates = transformed_new_candidates
# transform indices if candidates was provided in the shape of
# (n_candidates, n_features)
if candidates is not None and candidates.ndim > 1:
new_queried_indices = new_candidate_indices[queried_indices]
else:
new_queried_indices = queried_indices
# transform utilities from subsampled shape to original utilities shape
if return_utilities:
if candidates is None or candidates.ndim == 1:
new_utilities = np.full(
shape=(effective_batch_size, len(X)), fill_value=np.nan
)
new_utilities[:, candidate_indices] = -np.inf
new_utilities[:, new_candidates] = utilities[:, new_candidates]
else:
new_utilities = np.full(
shape=(effective_batch_size, len(candidates)),
fill_value=np.nan,
)
new_utilities[:, candidate_indices] = -np.inf
new_utilities[:, new_candidate_indices] = utilities
if return_utilities:
return new_queried_indices, new_utilities
else:
return new_queried_indices
def _embed_samples(self, samples, name):
"""Embed `samples` by applying `embed_samples_func`.
Parameters
----------
samples : array-like of shape (n_samples, ...)
Samples to be embedded.
name : str
Name of `samples` being used for the error message.
Returns
-------
embedded_samples : array-like of shape (n_samples, ...)
The embedded samples in the order of `samples`.
"""
embedded_samples = self.embed_samples_func(samples)
if np.ndim(embedded_samples) < 2:
raise ValueError(
f"`embed_samples_func` returned a "
f"{np.ndim(embedded_samples)}-dimensional embedding for "
f"`{name}` but must return an at least two-dimensional one."
)
if len(embedded_samples) != len(samples):
raise ValueError(
f"`embed_samples_func` returned "
f"{len(embedded_samples)} samples for the {len(samples)} "
f"samples of `{name}` but must preserve the number and "
f"order of samples."
)
return embedded_samples
[docs]
class ParallelUtilityEstimationWrapper(_TargetPreservingWrapper):
"""Parallel Utility Estimation Wrapper
This class implements a wrapper for single-annotator pool-based strategies
such that utilities for candidates can be calculated in parallel. The main
assumption for this is that the utility computations are independent from
another. Therefore, only `batch_size=1` is supported.
Each row of a two-dimensional `y` is one sample, so it must be either
fully labeled or fully unlabeled.
Parameters
----------
query_strategy : skactiveml.base.SingleAnnotatorPoolQueryStrategy
The strategy used for computing the utilities of the candidates.
n_jobs : int, default=-1
Determines the number of maximum number of parallel utility
computations. If `n_jobs` is set to -1 (default), the number of
parallel computations is set to the number of available CPU cores are.
For further details refer to `n_jobs` in `joblib.Parallel`.
parallel_dict : dict-like, default=None
Further arguments that will be passed to `joblib.Parallel`. Note that,
`n_jobs` should not be set in `parallel_dict`.
missing_label : scalar or string or np.nan or None, default=np.nan
Value to represent a missing label.
random_state : int or np.random.RandomState, default=None
The random state to use.
"""
def __init__(
self,
query_strategy=None,
n_jobs=-1,
parallel_dict=None,
missing_label=MISSING_LABEL,
random_state=None,
):
super().__init__(
missing_label=missing_label,
random_state=random_state,
)
self.query_strategy = query_strategy
self.n_jobs = n_jobs
self.parallel_dict = parallel_dict
[docs]
@match_signature("query_strategy", "query")
def query(
self,
X,
y,
candidates=None,
batch_size=1,
return_utilities=False,
**query_kwargs,
):
"""Determines for which candidate samples labels are to be queried.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Training data set, usually complete, i.e., including the labeled
and unlabeled samples.
y : array-like of shape (n_samples,) or (n_samples, n_outputs)
Labels of the training data set (possibly including unlabeled ones
indicated by `self.missing_label`). If `y` is two-dimensional, a
row `y[i]` must either contain only observed labels or only
`missing_label` values, i.e., no mixing within a row.
candidates : None or array-like of shape (n_candidates), dtype=int or
array-like of shape (n_candidates, n_features), (default=None)
- If `candidates` is `None`, the unlabeled samples from `(X,y)` are
considered as `candidates`.
- If `candidates` is of shape `(n_candidates,)` and of type
`int`, `candidates` is considered as the indices of the
samples in `(X,y)`.
- If `candidates` is of shape `(n_candidates, ...)`, the
candidate samples are directly given in `candidates` (not
necessarily contained in `X`). This is not supported by all
query strategies.
batch_size : int, default=1
The number of samples to be selected in one AL cycle. For this
wrapper, only `batch_size=1` is supported.
return_utilities : bool, default=False
If `True`, also return the utilities based on the query strategy.
**query_kwargs : dict-like
Further keyword arguments are passed to the `query` method of the
`query_strategy` object.
Returns
-------
query_indices : numpy.ndarray of shape (batch_size,)
The query indices indicate for which candidate sample a label is
to be queried, e.g., `query_indices[0]` indicates the first
selected sample.
- If `candidates` is `None` or of shape
`(n_candidates,)`, the indexing refers to the samples in
`X`.
- If `candidates` is of shape `(n_candidates, n_features)`,
the indexing refers to the samples in `candidates`.
utilities : numpy.ndarray of shape (batch_size, n_samples) or \
numpy.ndarray of shape (batch_size, n_candidates)
The utilities of samples after each selected sample of the batch,
e.g., `utilities[0]` indicates the utilities used for selecting
the first sample (with index `query_indices[0]`) of the batch.
Utilities for labeled samples will be set to np.nan.
- If `candidates` is `None` or of shape
`(n_candidates,)`, the indexing refers to the samples in
`X`.
- If `candidates` is of shape `(n_candidates, n_features)`,
the indexing refers to the samples in `candidates`.
"""
self._check_wrapped_strategy()
# A clone keeps results cached for one `X`, e.g., distances, out of a
# later query with other samples.
query_strategy = clone(self.query_strategy)
mask_type = self._mask_type(y)
# Validate parameters.
X, y, candidates, batch_size, return_utilities = self._validate_data(
X,
y,
candidates,
batch_size,
return_utilities,
reset=True,
target_type=mask_type,
)
if batch_size != 1:
raise ValueError("`batch_size` must be set to 1.")
# Determine candidate samples for selection.
X_cand, mapping = self._transform_candidates(
candidates=candidates, X=X, y=y, target_type=mask_type
)
# Determine number of parallel jobs.
if self.parallel_dict is None:
parallel_dict = {}
elif isinstance(self.parallel_dict, dict):
parallel_dict = self.parallel_dict.copy()
if "n_jobs" in parallel_dict.keys():
warnings.warn(
f"`n_jobs` ({parallel_dict['n_jobs']}) "
"is specified in `parallel_dict`. "
f"This will be replaced with `n_jobs={self.n_jobs}`."
)
else:
raise TypeError(
f"`parallel_dict` is of type `{type(self.parallel_dict)}` "
f"but must be a dictionary or None."
)
parallel_dict["n_jobs"] = min(self.n_jobs, len(X_cand))
parallel_pool = Parallel(**parallel_dict)
def query_lambda_func(chunk_candidates, chunk_kwargs):
_, utilities = query_strategy.query(
X=X,
y=y,
candidates=chunk_candidates,
batch_size=1,
return_utilities=True,
**chunk_kwargs,
)
# Utilities of index candidates refer to the samples in `X`.
if mapping is None:
return utilities[0]
return utilities[0][chunk_candidates]
# Never split into more chunks than there are candidates.
if parallel_dict["n_jobs"] < 0:
n_chunks = min(cpu_count(), len(X_cand))
else:
n_chunks = parallel_dict["n_jobs"]
if mapping is None:
# Candidate-aligned arguments are split along with the candidate
# samples.
chunks = [
(
X_cand[positions],
_subset_query_kwargs(
query_kwargs,
_CANDIDATE_ALIGNED_KWARGS,
positions,
len(X_cand),
"n_candidates",
),
)
for positions in np.array_split(
np.arange(len(X_cand)), n_chunks
)
]
else:
# Index candidates keep the arguments aligned to `X` valid and
# are scored by the wrapped strategy as in a direct query.
chunks = [
(indices, query_kwargs)
for indices in np.array_split(mapping, n_chunks)
]
utilities_cand = np.concatenate(
parallel_pool(
delayed(query_lambda_func)(*chunk) for chunk in chunks
),
axis=0,
)
if mapping is None:
utilities = utilities_cand
else:
utilities = np.full(len(X), np.nan)
utilities[mapping] = utilities_cand
return simple_batch(
utilities,
self.random_state_,
batch_size=batch_size,
return_utilities=return_utilities,
)