copairs.map

copairs.map

Module to compute mAP-based metrics.

apply_fdr_correction(map_scores, method='fdr_bh')

Apply standard FDR correction across all tests.

Parameters:

• map_scores (DataFrame) –

  DataFrame containing mAP scores with a 'p_value' column.

• method (str, default: 'fdr_bh' ) –

  Multiple testing correction method (default: 'fdr_bh'). See statsmodels.stats.multitest.multipletests for options.

Returns:

• DataFrame –

  Input DataFrame with 'corrected_p_value' column added.

Source code in src/copairs/map/hierarchical_fdr.py

def apply_fdr_correction(
    map_scores: pd.DataFrame,
    method: str = "fdr_bh",
) -> pd.DataFrame:
    """Apply standard FDR correction across all tests.

    Parameters
    ----------
    map_scores : pd.DataFrame
        DataFrame containing mAP scores with a 'p_value' column.
    method : str, optional
        Multiple testing correction method (default: 'fdr_bh').
        See statsmodels.stats.multitest.multipletests for options.

    Returns
    -------
    pd.DataFrame
        Input DataFrame with 'corrected_p_value' column added.

    """
    map_scores = map_scores.copy()
    _, pvals_corrected, _, _ = _multipletests(map_scores["p_value"], method=method)
    map_scores["corrected_p_value"] = pvals_corrected
    return map_scores

apply_hierarchical_fdr_correction(map_scores, hierarchical_by, sameby)

Apply hierarchical FDR correction for grouped hypotheses.

Implements a two-stage testing procedure appropriate for dose-response data where only high doses are expected to be active:

• Stage 1: Use minimum p-value within each group defined by hierarchical_by, then apply BH correction at the group level. A group passes if any member is significant.
• Stage 2: For groups that pass Stage 1, apply BH correction to the individual tests within each group.

Parameters:

• map_scores (DataFrame) –

  DataFrame containing mAP scores with a 'p_value' column.

• hierarchical_by (list) –

  Metadata column(s) defining the group structure (e.g., ['compound']).

• sameby (list) –

  Metadata column(s) used for mAP calculation (e.g., ['compound', 'dose']).

Returns:

• DataFrame –

  Input DataFrame with additional columns: - corrected_p_value: BH-corrected p-value (1.0 for groups that didn't pass Stage 1). - stage1_p_value: Group-level p-value from Stage 1 (minimum p-value). - stage1_corrected_p_value: BH-corrected Stage 1 p-value. - stage1_significant: Whether the group passed Stage 1.

Raises:

• ValueError –

  If hierarchical_by is not a proper subset of sameby.

Notes

This method uses minimum p-value (rather than Simes) for Stage 1 aggregation. Min-p is appropriate for dose-response data where only high doses are expected to be active. Simes would penalize compounds for having inactive low doses, which is the expected biological behavior.

Source code in src/copairs/map/hierarchical_fdr.py

def apply_hierarchical_fdr_correction(
    map_scores: pd.DataFrame,
    hierarchical_by: List[str],
    sameby: List[str],
) -> pd.DataFrame:
    """Apply hierarchical FDR correction for grouped hypotheses.

    Implements a two-stage testing procedure appropriate for dose-response data
    where only high doses are expected to be active:

    - Stage 1: Use minimum p-value within each group defined by `hierarchical_by`,
      then apply BH correction at the group level. A group passes if any member
      is significant.
    - Stage 2: For groups that pass Stage 1, apply BH correction to the
      individual tests within each group.

    Parameters
    ----------
    map_scores : pd.DataFrame
        DataFrame containing mAP scores with a 'p_value' column.
    hierarchical_by : list
        Metadata column(s) defining the group structure (e.g., ['compound']).
    sameby : list
        Metadata column(s) used for mAP calculation (e.g., ['compound', 'dose']).

    Returns
    -------
    pd.DataFrame
        Input DataFrame with additional columns:
        - `corrected_p_value`: BH-corrected p-value (1.0 for groups that didn't pass Stage 1).
        - `stage1_p_value`: Group-level p-value from Stage 1 (minimum p-value).
        - `stage1_corrected_p_value`: BH-corrected Stage 1 p-value.
        - `stage1_significant`: Whether the group passed Stage 1.

    Raises
    ------
    ValueError
        If `hierarchical_by` is not a proper subset of `sameby`.

    Notes
    -----
    This method uses minimum p-value (rather than Simes) for Stage 1 aggregation.
    Min-p is appropriate for dose-response data where only high doses are expected
    to be active. Simes would penalize compounds for having inactive low doses,
    which is the expected biological behavior.

    """
    sameby = check_group_columns(sameby, "sameby", map_scores)
    hierarchical_by = check_group_columns(hierarchical_by, "hierarchical_by")
    check_hierarchical_by(hierarchical_by, sameby)

    logger.info("Applying hierarchical FDR correction...")
    map_scores = map_scores.copy()

    # Stage 1: Aggregate p-values to group level using minimum p-value
    # Min-p is appropriate for dose-response where only high doses are expected to be active
    stage1_pvals = map_scores.groupby(hierarchical_by, observed=True).agg(
        {"p_value": "min"}
    )
    stage1_pvals.columns = ["stage1_p_value"]

    # Apply BH correction at the group level
    reject_stage1, stage1_corrected, _, _ = _multipletests(
        stage1_pvals["stage1_p_value"], method="fdr_bh"
    )
    stage1_pvals["stage1_corrected_p_value"] = stage1_corrected
    stage1_pvals["stage1_significant"] = reject_stage1

    # Merge Stage 1 results back to map_scores
    map_scores = map_scores.merge(
        stage1_pvals.reset_index(), on=hierarchical_by, how="left"
    )

    # Stage 2: For groups that passed Stage 1, apply BH within each group
    # For groups that didn't pass, set corrected_p_value to 1.0
    map_scores["corrected_p_value"] = np.where(
        map_scores["p_value"].notna(), 1.0, np.nan
    )

    for group_key, group_df in map_scores.groupby(hierarchical_by, observed=True):
        passed = group_df["stage1_significant"].iloc[0]
        if pd.isna(passed) or not passed:
            # Group didn't pass Stage 1, skip
            continue

        group_indices = group_df.index
        group_pvals = group_df["p_value"].values

        if len(group_pvals) == 1:
            # Single test in group, no additional correction needed
            map_scores.loc[group_indices, "corrected_p_value"] = group_pvals[0]
        else:
            # Apply BH correction within the group
            _, group_corrected, _, _ = _multipletests(group_pvals, method="fdr_bh")
            map_scores.loc[group_indices, "corrected_p_value"] = group_corrected

    return map_scores

get_map_pvalue(ap_scores, sameby, null_size, seed=0, progress_bar=True, max_workers=None, cache_dir=None, pos_pairs=None, null='permutation', pairs=None, *, max_reference_profiles=16384, pool_memory_budget=4 * 1024 ** 3, key_table_memory_budget=256 * 1024 ** 2)

Compute mAP scores and p-values from AP scores.

This function groups AP scores by the specified columns, computes the mean Average Precision (mAP) for each group, and calculates p-values by comparing against the null distribution of the group's mAP chosen by null. A null mAP counts against the group when it is >= the group's mAP, so an mAP equal to an atom of the null (e.g. perfect retrieval) counts the atom; values within the scores' rounding count as ties (copairs.nulls.tie_thresholds).

Parameters:

• ap_scores (DataFrame) –

  DataFrame containing individual Average Precision (AP) scores and pair statistics (e.g., number of positive pairs n_pos_pairs and total pairs n_total_pairs).

• sameby (list or str) –

  Nonempty, distinct string column names present in the input.

• null_size (int) –

  Number of Monte Carlo draws. Zero requests scores only: p-values are NaN after input/layout validation, without sampling or null-only tables.

• seed (int or None, default: 0 ) –

  Reproducible seed; None resolves fresh entropy once per run. Joint and permutation accept integers in [0, 2**64); copied accepts any integer >= 0.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

• max_workers (int, default: None ) –

  Threads that compute the copied null's p-values or run the permutation null's groups. Default defined by concurrent.futures.ThreadPoolExecutor.

• cache_dir (str or Path, default: None ) –

  Location to save the cache of the copied null's AP samples.

• pos_pairs (ndarray, default: None ) –

  The (n_pairs, 2) positive pairs used to compute the AP scores, indexing the rows of the frame average_precision returned (its return_pairs=True gives them, as pairs["pos"]); ap_scores must keep that frame's index. Pairs whose counts disagree with n_pos_pairs raise. When omitted, they are taken from pairs or, failing that, from the pairs average_precision attached to ap_scores. Multi-member groups require actual positive pairs; counts alone do not identify their graph. Singletons can use their AP null without pairs. If a pair handle is available, explicit positive pairs must agree with it. Joint null only.

• null (str, default: 'permutation' ) –

  Null of the group mAP. "permutation" (default) relabels profiles (copairs.nulls.permutation): in each block of members ranked against the same references (e.g. a plate's replicates and controls), any of the block's profiles could have carried the group's label, and each draw recomputes the group's mAP from the observed similarities. All positive edges must stay inside the tested group; the graph need not be complete. Original metadata rules define the graph and reference pools and are not reapplied per draw. A singleton with external positives can use "joint", its AP null. Positive draw counts use Monte Carlo sampling with (hits + 1)/(draws + 1). It recomputes similarities, so APs computed with a distance that estimates its scale per batch ("seuclidean", "mahalanobis") raise. Multilabel pair dictionaries use negative-graph components, preserving member counts per component and other labels, with the original positive graph. Negatives follow the original negative/lone-pair structure; counts can change. Each group must contain one complete label. Overlapping reference sets are allowed for this explicitly distinct multilabel model. "joint" samples the members' APs together (copairs.nulls.joint): a positive pair inside the group is one random number in both members' rank lists, and other similarities are independent, which real profiles are not (members share references, and some profiles are close to many others), so it can give too small p-values for groups of many replicates. "copied" samples each member's AP null per (n_pos_pairs, n_total_pairs) configuration and the group's null averages them, so members with equal pair counts share one draw and the null does not narrow as replicates are added.

• pairs (dict, default: None ) –

  Dictionary with "pos", "neg" and "similarity" entries, as average_precision(..., return_pairs=True) returns them; their indices are rows of that call's frame, so ap_scores must keep its index. The permutation null needs them and, when not given, takes them from ap_scores.attrs["copairs_pairs"], where average_precision attaches them; frames that lost them (built, reloaded, pickled, or concatenated from several AP calls) need pairs= or another null. The joint null takes its positive pairs from pairs too, or from the attached pairs (see pos_pairs). Multilabel dictionaries also carry "label_col", "profile", "row_key", "keys" and the lazy "_lone" pair function; "label_col" selects the multilabel permutation path. Explicit pairs override attachments. Keep the original index, matching pairs and complete tested groups/labels; validation checks structure, not source identity or group completeness. Recompute AP when removing group members. Copies share the dictionary, features and arrays; keep them unchanged between AP and mAP. Custom similarities must return the same value for a pair regardless of batch size or contents.

• max_reference_profiles (int, default: 16384 ) –

  Maximum total reference profiles across a tested group's blocks; >= 0. For multilabel groups, component-union size minus its members; all components count. Pool-table estimates include members too. Checked before quadratic pool tables are allocated.

• pool_memory_budget (int, default: 4294967296 ) –

  Positive byte budget for estimated concurrent pool key/rank tables. One oversized pool may run alone. This does not cap total process RAM.

• key_table_memory_budget (int, default: 268435456 ) –

  Byte limit for optional cross-pool key reuse; zero disables it. Neither budget includes features, layouts or all temporary arrays.

Returns:

• DataFrame –

  DataFrame with the following columns: - Columns from sameby (group identifiers). - mean_average_precision: Mean AP score for each group. - mean_normalized_average_precision: Mean normalized AP score (scale-independent). - p_value: p-value comparing mAP to the null distribution. - indices: The group's rows, as positions in ap_scores after rows with a NaN AP or no positive pairs are dropped.

Raises:

• UnpairedException –

  If no row has a non-NaN AP, positive pairs and a value in every sameby column, e.g. an empty table.

• ValueError –

  Invalid grouping, pair dictionaries/counts, layout or resource arguments. max_workers must be a positive integer or None even for zero draws and joint sampling, which does not use that executor.

Source code in src/copairs/map/map.py

def get_map_pvalue(
    ap_scores: pd.DataFrame,
    sameby: List[str],
    null_size: int,
    seed: int | None = 0,
    progress_bar: bool = True,
    max_workers: Optional[int] = None,
    cache_dir: Optional[Union[str, Path]] = None,
    pos_pairs: Optional[np.ndarray] = None,
    null: str = "permutation",
    pairs: dict | None = None,
    *,
    max_reference_profiles: int = 16_384,
    pool_memory_budget: int = 4 * 1024**3,
    key_table_memory_budget: int = 256 * 1024**2,
) -> pd.DataFrame:
    """Compute mAP scores and p-values from AP scores.

    This function groups AP scores by the specified columns, computes the mean
    Average Precision (mAP) for each group, and calculates p-values by comparing
    against the null distribution of the group's mAP chosen by `null`. A null
    mAP counts against the group when it is ``>=`` the group's mAP, so an mAP
    equal to an atom of the null (e.g. perfect retrieval) counts the atom; values
    within the scores' rounding count as ties (`copairs.nulls.tie_thresholds`).

    Parameters
    ----------
    ap_scores : pd.DataFrame
        DataFrame containing individual Average Precision (AP) scores and pair statistics
        (e.g., number of positive pairs `n_pos_pairs` and total pairs `n_total_pairs`).
    sameby : list or str
        Nonempty, distinct string column names present in the input.
    null_size : int
        Number of Monte Carlo draws. Zero requests scores only: p-values are NaN
        after input/layout validation, without sampling or null-only tables.
    seed : int or None, default 0
        Reproducible seed; None resolves fresh entropy once per run. Joint and
        permutation accept integers in [0, 2**64); copied accepts any integer >= 0.
    progress_bar : bool
        Whether or not to show tqdm's progress bar.
    max_workers : int
        Threads that compute the copied null's p-values or run the
        permutation null's groups. Default defined by
        `concurrent.futures.ThreadPoolExecutor`.
    cache_dir : str or Path
        Location to save the cache of the copied null's AP samples.
    pos_pairs : np.ndarray, optional
        The `(n_pairs, 2)` positive pairs used to compute the AP scores,
        indexing the rows of the frame `average_precision` returned (its
        `return_pairs=True` gives them, as `pairs["pos"]`); `ap_scores` must keep
        that frame's index. Pairs whose counts disagree with `n_pos_pairs`
        raise. When omitted, they are taken from ``pairs`` or, failing that,
        from the pairs `average_precision` attached to ``ap_scores``. Multi-member
        groups require actual positive pairs; counts alone do not identify their
        graph. Singletons can use their AP null without pairs. If a pair handle
        is available, explicit positive pairs must agree with it. Joint null only.
    null : str
        Null of the group mAP. ``"permutation"`` (default) relabels profiles
        (`copairs.nulls.permutation`): in each block of members ranked against
        the same references (e.g. a plate's replicates and controls), any of
        the block's profiles could have carried the group's label, and each
        draw recomputes the group's mAP from the observed similarities. All
        positive edges must stay inside the tested group; the graph need not
        be complete. Original metadata rules define the graph and reference
        pools and are not reapplied per draw. A singleton with external
        positives can use ``"joint"``, its AP null. Positive draw counts use
        Monte Carlo sampling with (hits + 1)/(draws + 1). It recomputes similarities, so
        APs computed with a distance that estimates its scale per batch
        (``"seuclidean"``, ``"mahalanobis"``) raise. Multilabel pair dictionaries
        use negative-graph components, preserving member counts per component
        and other labels, with the original positive graph. Negatives follow
        the original negative/lone-pair structure; counts can change. Each
        group must contain one complete label. Overlapping reference sets are
        allowed for this explicitly distinct multilabel model. ``"joint"`` samples
        the members' APs together (`copairs.nulls.joint`): a positive pair
        inside the group is one random number in both members' rank lists,
        and other similarities are independent, which real profiles are not
        (members share references, and some profiles are close to many
        others), so it can give too small p-values for groups of many
        replicates. ``"copied"`` samples each member's AP null per
        `(n_pos_pairs, n_total_pairs)`
        configuration and the group's null averages them, so members with
        equal pair counts share one draw and the null does not narrow as
        replicates are added.
    pairs : dict, optional
        Dictionary with ``"pos"``, ``"neg"`` and ``"similarity"`` entries, as
        ``average_precision(..., return_pairs=True)`` returns them; their
        indices are rows of that call's frame, so ``ap_scores`` must keep its
        index. The permutation null needs them and, when not given, takes
        them from ``ap_scores.attrs["copairs_pairs"]``, where
        `average_precision` attaches them; frames that lost them (built,
        reloaded, pickled, or concatenated from several AP calls) need
        ``pairs=`` or another null. The joint null takes its positive pairs
        from ``pairs`` too, or from the attached pairs (see ``pos_pairs``).
        Multilabel dictionaries also carry ``"label_col"``, ``"profile"``,
        ``"row_key"``, ``"keys"`` and the lazy ``"_lone"`` pair function;
        ``"label_col"`` selects the multilabel permutation path.
        Explicit pairs override attachments. Keep the original index, matching
        pairs and complete tested groups/labels; validation checks structure,
        not source identity or group completeness. Recompute AP when removing
        group members. Copies share the dictionary, features and arrays; keep
        them unchanged between AP and mAP. Custom similarities must return the
        same value for a pair regardless of batch size or contents.
    max_reference_profiles : int, default 16384
        Maximum total reference profiles across a tested group's blocks; >= 0.
        For multilabel groups, component-union size minus its members; all
        components count. Pool-table estimates include members too.
        Checked before quadratic pool tables are allocated.
    pool_memory_budget : int, default 4294967296
        Positive byte budget for estimated concurrent pool key/rank tables.
        One oversized pool may run alone. This does not cap total process RAM.
    key_table_memory_budget : int, default 268435456
        Byte limit for optional cross-pool key reuse; zero disables it.
        Neither budget includes features, layouts or all temporary arrays.

    Returns
    -------
    pd.DataFrame
        DataFrame with the following columns:
        - Columns from `sameby` (group identifiers).
        - `mean_average_precision`: Mean AP score for each group.
        - `mean_normalized_average_precision`: Mean normalized AP score (scale-independent).
        - `p_value`: p-value comparing mAP to the null distribution.
        - `indices`: The group's rows, as positions in ap_scores after rows
          with a NaN AP or no positive pairs are dropped.

    Raises
    ------
    UnpairedException
        If no row has a non-NaN AP, positive pairs and a value in every
        `sameby` column, e.g. an empty table.
    ValueError
        Invalid grouping, pair dictionaries/counts, layout or resource arguments.
        ``max_workers`` must be a positive integer or None even for zero draws
        and joint sampling, which does not use that executor.

    """
    null = check_null(null)
    null_size = check_null_size(null_size)
    sameby = check_group_columns(sameby, "sameby", ap_scores)
    max_workers = check_max_workers(max_workers)
    resources = dict(
        zip(
            ("max_reference_profiles", "pool_memory_budget", "key_table_memory_budget"),
            permutation.check_resources(
                max_reference_profiles, pool_memory_budget, key_table_memory_budget
            ),
        )
    )
    if null != "copied":
        check_seed(seed)
    elif seed is not None and (int(seed) != seed or seed < 0):  # NumPy: any int >= 0
        raise ValueError(f"seed must be a non-negative integer, got {seed}")
    if null != "joint" and pos_pairs is not None:
        raise ValueError("pos_pairs is used by the joint null only")
    if null != "copied" and pairs is None:
        pairs = attached_pairs(ap_scores)
    if pairs is not None and null != "copied":
        if (
            not isinstance(pairs, dict)
            or not {"pos", "neg", "similarity"} <= pairs.keys()
        ):
            raise ValueError(
                "pairs must be a dictionary with pos, neg and similarity entries"
            )
        if pos_pairs is not None and not np.array_equal(
            check_pairs(pos_pairs, "pos_pairs"),
            check_pairs(pairs["pos"], "positive pairs"),
        ):
            raise ValueError(
                "pos_pairs must index the rows of the AP frame and match the selected handle's positive pairs"
            )
    if null == "joint" and pos_pairs is None and pairs is not None:
        pos_pairs = pairs["pos"]
    if null == "permutation" and pairs is None:
        raise ValueError(
            "null='permutation' (the default) needs the pairs the APs were "
            "computed from. average_precision attaches them to the frame it "
            "returns, and these AP scores have none: they were built, loaded "
            "from disk, pickled or concatenated from several AP calls. "
            "Both average_precision and multilabel.average_precision attach pairs. "
            "Recompute the APs, pass pairs=, or choose null='joint' "
            "or null='copied'."
        )
    if null == "permutation":
        check_recomputable(pairs["similarity"])

    if null == "permutation" or pos_pairs is not None:
        _check_row_index(ap_scores.index)

    # Filter out invalid or incomplete AP scores, remembering the original rows.
    # Rows missing a sameby value stay (the groupby drops them), so `indices`
    # keep their positions; the check counts only rows a group can take.
    valid = (
        ap_scores["average_precision"].notna() & (ap_scores["n_pos_pairs"] > 0)
    ).fillna(False)
    if not (valid & ap_scores[sameby].notna().all(axis=1)).any():
        raise UnpairedException(
            "Unable to find valid AP scores: no row has a non-NaN AP, positive "
            "pairs and a value in every sameby column."
        )
    missing_keys = valid & ~ap_scores[sameby].notna().all(axis=1)
    tested = ap_scores.loc[valid & ~missing_keys]
    counts = _normalize_pair_counts(tested)
    ap_values = tested["average_precision"].to_numpy(dtype=np.float64)
    if not np.isfinite(ap_values).all() or ((ap_values < 0) | (ap_values > 1)).any():
        raise ValueError("usable AP scores must be finite values in [0, 1]")
    if (~valid).any() or missing_keys.any():
        logger.warning(
            "Excluding %d rows with missing AP/no positive pairs and %d rows with missing group keys",
            int((~valid).sum()),
            int(missing_keys.sum()),
        )
    ap_scores = ap_scores[valid]
    profile = ap_scores.index.to_numpy()
    ap_scores = ap_scores.reset_index(drop=True)
    # Keep positional indices for rows without group keys, but never cast their
    # unused counts (which may be missing/invalid) or pass them into a sampler.
    normalized = np.zeros((len(ap_scores), 2), dtype=np.int64)
    normalized[(~missing_keys[valid]).to_numpy(dtype=bool)] = counts
    ap_scores[["n_pos_pairs", "n_total_pairs"]] = normalized

    # Average the APs in float64 and round the means to the APs' dtype: ties
    # carry the APs' rounding, and pandas averages float32 APs to float32 or
    # float64 depending on its version and the group keys. The reported mAP
    # is then the score the nulls count ties against, under any pandas.
    ap_dtype = getattr(
        ap_scores["average_precision"].dtype,
        "numpy_dtype",
        ap_scores["average_precision"].to_numpy().dtype,
    )
    score_dtype = ap_dtype if ap_dtype.kind == "f" else np.dtype(np.float64)
    ap_scores["average_precision"] = ap_scores["average_precision"].astype(np.float64)

    # Group by the specified metadata column(s) and calculate mean AP
    map_scores = ap_scores.groupby(sameby, observed=True, as_index=False).agg(
        {
            "average_precision": ["mean", lambda x: list(x.index)],
            "normalized_average_precision": "mean",
        }
    )
    map_scores.columns = sameby + [
        "mean_average_precision",
        "indices",
        "mean_normalized_average_precision",
    ]
    map_scores["mean_average_precision"] = map_scores["mean_average_precision"].astype(
        score_dtype
    )
    groups = map_scores["indices"].tolist()
    scores = map_scores["mean_average_precision"].to_numpy()
    copied = dict(
        null_size=null_size,
        seed=seed,
        cache_dir=cache_dir,
        progress_bar=progress_bar,
        max_workers=max_workers,
    )

    if null == "copied":
        map_scores["p_value"] = (
            np.full(len(groups), np.nan)
            if null_size == 0
            else _copied_pvalues(scores, groups, ap_scores, **copied)
        )
        return map_scores

    if null == "permutation":
        logger.info("Computing p-values...")
        arguments = (
            groups,
            profile,
            ap_scores["n_pos_pairs"].to_numpy(),
            ap_scores["n_total_pairs"].to_numpy(),
        )
        options = dict(
            observed=scores,
            progress_bar=progress_bar,
            max_workers=max_workers,
            **resources,
        )
        if "label_col" in pairs:
            pvalues = permutation.label_permutation_pvalues(
                *arguments, pairs, null_size, seed, **options
            )
        else:
            pvalues = permutation.permutation_pvalues(
                *arguments,
                pairs["pos"],
                pairs["neg"],
                pairs["similarity"],
                null_size,
                seed,
                **options,
            )
        map_scores["p_value"] = pvalues
        return map_scores

    num_pos = ap_scores["n_pos_pairs"].to_numpy().astype(np.int64)
    num_neg = ap_scores["n_total_pairs"].to_numpy().astype(np.int64) - num_pos
    logger.info("Computing p-values...")
    map_scores["p_value"] = joint.group_pvalues(
        scores, groups, num_pos, num_neg, null_size, seed, profile, pos_pairs,
        progress_bar=progress_bar,
    )  # fmt: skip
    return map_scores

mean_average_precision(ap_scores, sameby, null_size, threshold, seed=0, progress_bar=True, max_workers=None, cache_dir=None, pos_pairs=None, null='permutation', pairs=None, *, max_reference_profiles=16384, pool_memory_budget=4 * 1024 ** 3, key_table_memory_budget=256 * 1024 ** 2)

Calculate the Mean Average Precision (mAP) score and associated p-values.

This function computes the Mean Average Precision (mAP) score by grouping profiles based on the specified criteria (sameby). It calculates the significance of mAP scores by comparing them to a null distribution and performs multiple testing corrections using Benjamini-Hochberg FDR.

Per-call resource controls (max_reference_profiles, pool_memory_budget, key_table_memory_budget), pair dictionaries and supported layouts follow :func:get_map_pvalue. With null_size=0 statistical columns are NaN and significance flags are nullable booleans containing pd.NA; correction is skipped.

Parameters:

• ap_scores (DataFrame) –

  DataFrame containing individual Average Precision (AP) scores and pair statistics (e.g., number of positive pairs n_pos_pairs and total pairs n_total_pairs).

• sameby (list or str) –

  Metadata column(s) used to group profiles for mAP calculation.

• null_size (int) –

  Number of samples in the null distribution for significance testing.

• threshold (float) –

  p-value threshold for identifying significant MaP scores.

• seed (int or None, default: 0 ) –

  Reproducible seed; None requests fresh randomness once per run.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

• max_workers (int, default: None ) –

  Threads that compute the copied null's p-values or run the permutation null's groups. Default defined by concurrent.futures.ThreadPoolExecutor.

• cache_dir (str or Path, default: None ) –

  Location to save the cache of the copied null's AP samples.

• pos_pairs (ndarray, default: None ) –

  Positive pairs used to compute the AP scores, for null="joint"; see get_map_pvalue. Needed there when a group's positives are not its other members.

• null (str, default: 'permutation' ) –

  Null of the group mAP: "permutation" (default), "joint" or "copied"; see get_map_pvalue.

• pairs (dict, default: None ) –

  Pair dictionary of the AP scores, needed by null="permutation" and attached to AP frames; see get_map_pvalue.

Returns:

• DataFrame –

  DataFrame with the following columns: - mean_average_precision: Mean AP score for each group. - mean_normalized_average_precision: Mean normalized AP score (scale-independent). - p_value: p-value comparing mAP to the null distribution. - corrected_p_value: Adjusted p-value after multiple testing correction. - below_p: Boolean indicating if the p-value is below the threshold. - below_corrected_p: Boolean indicating if the corrected p-value is below the threshold.

Raises:

• UnpairedException –

  If no row has a non-NaN AP, positive pairs and a value in every sameby column, e.g. an empty table.

See Also

mean_average_precision_hierarchical : For hierarchical FDR correction with grouped data.

Source code in src/copairs/map/map.py

def mean_average_precision(
    ap_scores: pd.DataFrame,
    sameby: List[str],
    null_size: int,
    threshold: float,
    seed: int | None = 0,
    progress_bar: bool = True,
    max_workers: Optional[int] = None,
    cache_dir: Optional[Union[str, Path]] = None,
    pos_pairs: Optional[np.ndarray] = None,
    null: str = "permutation",
    pairs: dict | None = None,
    *,
    max_reference_profiles: int = 16_384,
    pool_memory_budget: int = 4 * 1024**3,
    key_table_memory_budget: int = 256 * 1024**2,
) -> pd.DataFrame:
    """Calculate the Mean Average Precision (mAP) score and associated p-values.

    This function computes the Mean Average Precision (mAP) score by grouping profiles
    based on the specified criteria (`sameby`). It calculates the significance of mAP
    scores by comparing them to a null distribution and performs multiple testing
    corrections using Benjamini-Hochberg FDR.

    Per-call resource controls (``max_reference_profiles``, ``pool_memory_budget``,
    ``key_table_memory_budget``), pair dictionaries and supported layouts follow
    :func:`get_map_pvalue`. With ``null_size=0`` statistical columns are NaN and
    significance flags are nullable booleans containing pd.NA; correction is skipped.

    Parameters
    ----------
    ap_scores : pd.DataFrame
        DataFrame containing individual Average Precision (AP) scores and pair statistics
        (e.g., number of positive pairs `n_pos_pairs` and total pairs `n_total_pairs`).
    sameby : list or str
        Metadata column(s) used to group profiles for mAP calculation.
    null_size : int
        Number of samples in the null distribution for significance testing.
    threshold : float
        p-value threshold for identifying significant MaP scores.
    seed : int or None, default 0
        Reproducible seed; None requests fresh randomness once per run.
    progress_bar : bool
        Whether or not to show tqdm's progress bar.
    max_workers : int
        Threads that compute the copied null's p-values or run the
        permutation null's groups. Default defined by
        `concurrent.futures.ThreadPoolExecutor`.
    cache_dir : str or Path
        Location to save the cache of the copied null's AP samples.
    pos_pairs : np.ndarray, optional
        Positive pairs used to compute the AP scores, for ``null="joint"``;
        see `get_map_pvalue`. Needed there when a group's positives are not
        its other members.
    null : str
        Null of the group mAP: ``"permutation"`` (default), ``"joint"`` or
        ``"copied"``; see `get_map_pvalue`.
    pairs : dict, optional
        Pair dictionary of the AP scores, needed by ``null="permutation"`` and
        attached to AP frames; see `get_map_pvalue`.

    Returns
    -------
    pd.DataFrame
        DataFrame with the following columns:
        - `mean_average_precision`: Mean AP score for each group.
        - `mean_normalized_average_precision`: Mean normalized AP score (scale-independent).
        - `p_value`: p-value comparing mAP to the null distribution.
        - `corrected_p_value`: Adjusted p-value after multiple testing correction.
        - `below_p`: Boolean indicating if the p-value is below the threshold.
        - `below_corrected_p`: Boolean indicating if the corrected p-value is below the threshold.

    Raises
    ------
    UnpairedException
        If no row has a non-NaN AP, positive pairs and a value in every
        `sameby` column, e.g. an empty table.

    See Also
    --------
    mean_average_precision_hierarchical : For hierarchical FDR correction with grouped data.

    """
    # Step 1: Compute mAP scores and p-values
    map_scores = get_map_pvalue(
        ap_scores=ap_scores,
        sameby=sameby,
        null_size=null_size,
        seed=seed,
        progress_bar=progress_bar,
        max_workers=max_workers,
        cache_dir=cache_dir,
        pos_pairs=pos_pairs,
        null=null,
        pairs=pairs,
        max_reference_profiles=max_reference_profiles,
        pool_memory_budget=pool_memory_budget,
        key_table_memory_budget=key_table_memory_budget,
    )

    if null_size == 0:
        map_scores["corrected_p_value"] = np.nan
    else:
        map_scores = apply_fdr_correction(map_scores)
    return _mark_significance(map_scores, threshold)

mean_average_precision_hierarchical(ap_scores, sameby, null_size, threshold, seed=0, hierarchical_by=None, progress_bar=True, max_workers=None, cache_dir=None, pos_pairs=None, null='permutation', pairs=None, *, max_reference_profiles=16384, pool_memory_budget=4 * 1024 ** 3, key_table_memory_budget=256 * 1024 ** 2)

Calculate the Mean Average Precision (mAP) score with hierarchical FDR correction.

This function computes the Mean Average Precision (mAP) score by grouping profiles based on the specified criteria (sameby). It applies hierarchical FDR correction appropriate for grouped hypothesis testing, such as dose-response data.

Per-call resource controls (max_reference_profiles, pool_memory_budget, key_table_memory_budget), pair dictionaries and supported layouts follow :func:get_map_pvalue. With null_size=0 all statistical columns, including stage-1 p-values, are NaN; significance flags contain pd.NA and correction is skipped. Supply hierarchical_by explicitly, preferably as a keyword.

Parameters:

• ap_scores (DataFrame) –

  DataFrame containing individual Average Precision (AP) scores and pair statistics (e.g., number of positive pairs n_pos_pairs and total pairs n_total_pairs).

• sameby (list or str) –

  Metadata column(s) used to group profiles for mAP calculation.

• null_size (int) –

  Number of samples in the null distribution for significance testing.

• threshold (float) –

  p-value threshold for identifying significant MaP scores.

• seed (int or None, default: 0 ) –

  Reproducible seed; None requests fresh randomness once per run.

• hierarchical_by (list, default: None ) –

  Metadata column(s) for hierarchical FDR correction. Enables two-stage testing:

  • Stage 1: Use minimum p-value within each group defined by hierarchical_by, then apply BH correction at the group level. A group passes if any member is significant.
  • Stage 2: For groups that pass Stage 1, apply BH correction to the individual tests within each group.

  This is designed for dose-response data where only high doses are expected to be active. The hierarchical_by columns must be a proper subset of sameby. For example, with sameby=['compound', 'dose'] and hierarchical_by=['compound'], mAP is calculated per compound×dose, but FDR correction accounts for the grouped structure.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

• max_workers (int, default: None ) –

  Threads that compute the copied null's p-values or run the permutation null's groups. Default defined by concurrent.futures.ThreadPoolExecutor.

• cache_dir (str or Path, default: None ) –

  Location to save the cache of the copied null's AP samples.

• pos_pairs (ndarray, default: None ) –

  Positive pairs used to compute the AP scores, for null="joint"; see get_map_pvalue. Needed there when a group's positives are not its other members.

• null (str, default: 'permutation' ) –

  Null of the group mAP: "permutation" (default), "joint" or "copied"; see get_map_pvalue.

• pairs (dict, default: None ) –

  Pair dictionary of the AP scores, needed by null="permutation" and attached to AP frames; see get_map_pvalue.

Returns:

• DataFrame –

  DataFrame with the following columns: - mean_average_precision: Mean AP score for each group. - mean_normalized_average_precision: Mean normalized AP score (scale-independent). - p_value: p-value comparing mAP to the null distribution. - corrected_p_value: Adjusted p-value after multiple testing correction. - below_p: Boolean indicating if the p-value is below the threshold. - below_corrected_p: Boolean indicating if the corrected p-value is below the threshold. - stage1_p_value: Group-level p-value from Stage 1 (minimum p-value). - stage1_corrected_p_value: BH-corrected Stage 1 p-value. - stage1_significant: Whether the group passed Stage 1.

Raises:

• UnpairedException –

  If no row has a non-NaN AP, positive pairs and a value in every sameby column, e.g. an empty table.

See Also

mean_average_precision : For standard BH FDR correction.

Source code in src/copairs/map/map.py

def mean_average_precision_hierarchical(
    ap_scores: pd.DataFrame,
    sameby: List[str],
    null_size: int,
    threshold: float,
    seed: int | None = 0,
    hierarchical_by: Optional[List[str]] = None,
    progress_bar: bool = True,
    max_workers: Optional[int] = None,
    cache_dir: Optional[Union[str, Path]] = None,
    pos_pairs: Optional[np.ndarray] = None,
    null: str = "permutation",
    pairs: dict | None = None,
    *,
    max_reference_profiles: int = 16_384,
    pool_memory_budget: int = 4 * 1024**3,
    key_table_memory_budget: int = 256 * 1024**2,
) -> pd.DataFrame:
    """Calculate the Mean Average Precision (mAP) score with hierarchical FDR correction.

    This function computes the Mean Average Precision (mAP) score by grouping profiles
    based on the specified criteria (`sameby`). It applies hierarchical FDR correction
    appropriate for grouped hypothesis testing, such as dose-response data.

    Per-call resource controls (``max_reference_profiles``, ``pool_memory_budget``,
    ``key_table_memory_budget``), pair dictionaries and supported layouts follow
    :func:`get_map_pvalue`. With ``null_size=0`` all statistical columns, including
    stage-1 p-values, are NaN; significance flags contain pd.NA and correction is skipped.
    Supply ``hierarchical_by`` explicitly, preferably as a keyword.

    Parameters
    ----------
    ap_scores : pd.DataFrame
        DataFrame containing individual Average Precision (AP) scores and pair statistics
        (e.g., number of positive pairs `n_pos_pairs` and total pairs `n_total_pairs`).
    sameby : list or str
        Metadata column(s) used to group profiles for mAP calculation.
    null_size : int
        Number of samples in the null distribution for significance testing.
    threshold : float
        p-value threshold for identifying significant MaP scores.
    seed : int or None, default 0
        Reproducible seed; None requests fresh randomness once per run.
    hierarchical_by : list
        Metadata column(s) for hierarchical FDR correction. Enables two-stage testing:

        - Stage 1: Use minimum p-value within each group defined by `hierarchical_by`,
          then apply BH correction at the group level. A group passes if any member
          is significant.
        - Stage 2: For groups that pass Stage 1, apply BH correction to the
          individual tests within each group.

        This is designed for dose-response data where only high doses are expected
        to be active. The `hierarchical_by` columns must be a proper subset of `sameby`.
        For example, with `sameby=['compound', 'dose']` and `hierarchical_by=['compound']`,
        mAP is calculated per compound×dose, but FDR correction accounts for the
        grouped structure.
    progress_bar : bool
        Whether or not to show tqdm's progress bar.
    max_workers : int
        Threads that compute the copied null's p-values or run the
        permutation null's groups. Default defined by
        `concurrent.futures.ThreadPoolExecutor`.
    cache_dir : str or Path
        Location to save the cache of the copied null's AP samples.
    pos_pairs : np.ndarray, optional
        Positive pairs used to compute the AP scores, for ``null="joint"``;
        see `get_map_pvalue`. Needed there when a group's positives are not
        its other members.
    null : str
        Null of the group mAP: ``"permutation"`` (default), ``"joint"`` or
        ``"copied"``; see `get_map_pvalue`.
    pairs : dict, optional
        Pair dictionary of the AP scores, needed by ``null="permutation"`` and
        attached to AP frames; see `get_map_pvalue`.

    Returns
    -------
    pd.DataFrame
        DataFrame with the following columns:
        - `mean_average_precision`: Mean AP score for each group.
        - `mean_normalized_average_precision`: Mean normalized AP score (scale-independent).
        - `p_value`: p-value comparing mAP to the null distribution.
        - `corrected_p_value`: Adjusted p-value after multiple testing correction.
        - `below_p`: Boolean indicating if the p-value is below the threshold.
        - `below_corrected_p`: Boolean indicating if the corrected p-value is below the threshold.
        - `stage1_p_value`: Group-level p-value from Stage 1 (minimum p-value).
        - `stage1_corrected_p_value`: BH-corrected Stage 1 p-value.
        - `stage1_significant`: Whether the group passed Stage 1.

    Raises
    ------
    UnpairedException
        If no row has a non-NaN AP, positive pairs and a value in every
        `sameby` column, e.g. an empty table.

    See Also
    --------
    mean_average_precision : For standard BH FDR correction.

    """
    if hierarchical_by is None:
        raise ValueError("hierarchical_by is required for hierarchical correction")
    sameby = check_group_columns(sameby, "sameby", ap_scores)
    hierarchical_by = check_group_columns(hierarchical_by, "hierarchical_by")
    check_hierarchical_by(hierarchical_by, sameby)
    # Step 1: Compute mAP scores and p-values
    map_scores = get_map_pvalue(
        ap_scores=ap_scores,
        sameby=sameby,
        null_size=null_size,
        seed=seed,
        progress_bar=progress_bar,
        max_workers=max_workers,
        cache_dir=cache_dir,
        pos_pairs=pos_pairs,
        null=null,
        pairs=pairs,
        max_reference_profiles=max_reference_profiles,
        pool_memory_budget=pool_memory_budget,
        key_table_memory_budget=key_table_memory_budget,
    )

    if null_size == 0:
        for column in (
            "corrected_p_value",
            "stage1_p_value",
            "stage1_corrected_p_value",
        ):
            map_scores[column] = np.nan
        map_scores["stage1_significant"] = pd.array(
            [pd.NA] * len(map_scores), dtype="boolean"
        )
    else:
        map_scores = apply_hierarchical_fdr_correction(
            map_scores, hierarchical_by, sameby
        )
    return _mark_significance(map_scores, threshold)

average_precision

Functions to compute average precision.

attach_pairs(frame, pairs)

Attach pairs to frame (in place) and return it.

Source code in src/copairs/map/average_precision.py

def attach_pairs(frame: pd.DataFrame, pairs) -> pd.DataFrame:
    """Attach ``pairs`` to ``frame`` (in place) and return it."""
    frame.attrs[PAIRS_ATTR] = _AttachedPairs(pairs)
    return frame

attached_pairs(frame)

Return the pairs an AP function attached to frame, or None.

Source code in src/copairs/map/average_precision.py

def attached_pairs(frame: pd.DataFrame):
    """Return the pairs an AP function attached to ``frame``, or None."""
    handle = frame.attrs.get(PAIRS_ATTR)
    return handle.pairs if isinstance(handle, _AttachedPairs) else None

average_precision(meta, feats, pos_sameby, pos_diffby, neg_sameby, neg_diffby, batch_size=20000, distance='cosine', progress_bar=True, return_pairs=False)

Calculate average precision (AP) scores for pairs of profiles based on their similarity.

This function identifies positive and negative pairs of profiles using metadata rules, computes their similarity scores, and calculates average precision scores for each profile. The results include the number of positive and total pairs for each profile.

Parameters:

• meta (DataFrame) –

  Metadata of the profiles, including columns used for defining pairs. This DataFrame should include the columns specified in pos_sameby, pos_diffby, neg_sameby, and neg_diffby.

• feats (ndarray) –

  Feature matrix representing the profiles, where rows correspond to profiles and columns to features.

• pos_sameby (list) –

  Metadata columns used to define positive pairs. Two profiles are considered a positive pair if they belong to the same group that is not a control group. For example, replicate profiles of the same compound are positive pairs and should share the same value in a column identifying compounds.

• pos_diffby (list) –

  Metadata columns used to differentiate positive pairs. Positive pairs do not need to differ in any metadata columns, so this is typically left empty. However, if necessary (e.g., to account for batch effects), you can specify columns such as batch identifiers.

• neg_sameby (list) –

  Metadata columns used to define negative pairs. Typically left empty, as profiles forming a negative pair (e.g., a compound and a DMSO/control) do not need to share any metadata values. This ensures comparisons are made without enforcing unnecessary constraints.

• neg_diffby (list) –

  Metadata columns used to differentiate negative pairs. Two profiles are considered a negative pair if one belongs to a compound group and the other to a DMSO/ control group. They must differ in specified metadata columns, such as those identifying the compound and the treatment index, to ensure comparisons are only made between compounds and DMSO controls (not between different compounds).

• batch_size (int, default: 20000 ) –

  The batch size for similarity computations to optimize memory usage. Default is 20000.

• distance (str or callable, default: 'cosine' ) –

  The distance function used for computing similarities. Default is "cosine". A custom similarity must return the same value for a pair regardless of batch size or contents.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

• return_pairs (bool, default: False ) –

  Also return the pair dictionary the APs were computed from, which null="permutation" in :func:copairs.map.mean_average_precision needs. They are attached to the returned frame either way, in frame.attrs["copairs_pairs"], and follow it through filtering and copies (not saving, pickling or concatenating with other AP calls' frames). The dictionary contains "pos" and "neg" arrays of shape (n, 2), indexed by source row, and a "similarity" callable mapping pairs to similarities. Keep the original AP index, matching pairs and complete tested groups; validation checks structure, not source identity or group completeness. Recompute AP when removing group members. Copies share the dictionary, arrays and features; keep them unchanged between AP calculation and testing. They keep feats and every pair (8 bytes per pair) alive with the frame, whichever null is used; drop them with frame.attrs.pop("copairs_pairs").

Returns:

• DataFrame –

  A DataFrame (with return_pairs, a (DataFrame, dict) tuple) containing the following columns: - 'average_precision': The calculated average precision score for each profile. - 'normalized_average_precision': The normalized AP score (scale-independent). - 'n_pos_pairs': The number of positive pairs for each profile. - 'n_total_pairs': The total number of pairs for each profile. - Additional metadata columns from the input.

Raises:

• UnpairedException –

  If no positive or negative pairs are found in the dataset.

Notes

• Positive Pair Rules:
  • Positive pairs are defined by pos_sameby (profiles share these metadata values) and optionally differentiated by pos_diffby (profiles must differ in these metadata values if specified).
• Negative Pair Rules:
  • Negative pairs are defined by neg_diffby (profiles differ in these metadata values) and optionally constrained by neg_sameby (profiles share these metadata values if specified).

Source code in src/copairs/map/average_precision.py

def average_precision(
    meta: pd.DataFrame,
    feats: pd.DataFrame,
    pos_sameby: List[str],
    pos_diffby: List[str],
    neg_sameby: List[str],
    neg_diffby: List[str],
    batch_size: int = 20000,
    distance: str = "cosine",
    progress_bar: bool = True,
    return_pairs: bool = False,
) -> pd.DataFrame:
    """Calculate average precision (AP) scores for pairs of profiles based on their similarity.

    This function identifies positive and negative pairs of profiles using  metadata
    rules, computes their similarity scores, and calculates average precision
    scores for each profile. The results include the number of positive and total pairs
    for each profile.

    Parameters
    ----------
    meta : pd.DataFrame
        Metadata of the profiles, including columns used for defining pairs.
        This DataFrame should include the columns specified in `pos_sameby`,
        `pos_diffby`, `neg_sameby`, and `neg_diffby`.

    feats : np.ndarray
        Feature matrix representing the profiles, where rows correspond to profiles
        and columns to features.

    pos_sameby : list
        Metadata columns used to define positive pairs. Two profiles are considered a
        positive pair if they belong to the same group that is not a control group.
        For example, replicate profiles of the same compound are positive pairs and
        should share the same value in a column identifying compounds.

    pos_diffby : list
        Metadata columns used to differentiate positive pairs. Positive pairs do not need
        to differ in any metadata columns, so this is typically left empty. However,
        if necessary (e.g., to account for batch effects), you can specify columns
        such as batch identifiers.

    neg_sameby : list
        Metadata columns used to define negative pairs. Typically left empty, as profiles
        forming a negative pair (e.g., a compound and a DMSO/control) do not need to
        share any metadata values. This ensures comparisons are made without enforcing
        unnecessary constraints.

    neg_diffby : list
        Metadata columns used to differentiate negative pairs. Two profiles are considered
        a negative pair if one belongs to a compound group and the other to a DMSO/
        control group. They must differ in specified metadata columns, such as those
        identifying the compound and the treatment index, to ensure comparisons are
        only made between compounds and DMSO controls (not between different compounds).

    batch_size : int
        The batch size for similarity computations to optimize memory usage.
        Default is 20000.

    distance : str or callable
        The distance function used for computing similarities. Default is "cosine".
        A custom similarity must return the same value for a pair regardless
        of batch size or contents.

    progress_bar : bool
        Whether or not to show tqdm's progress bar.

    return_pairs : bool
        Also return the pair dictionary the APs were computed from, which
        ``null="permutation"`` in :func:`copairs.map.mean_average_precision`
        needs. They are attached to the returned frame either way, in
        ``frame.attrs["copairs_pairs"]``, and follow it through filtering
        and copies (not saving, pickling or concatenating with other AP
        calls' frames). The dictionary contains ``"pos"`` and ``"neg"`` arrays
        of shape (n, 2), indexed by source row, and a ``"similarity"`` callable
        mapping pairs to similarities. Keep the original AP index, matching
        pairs and complete tested groups; validation checks structure, not source
        identity or group completeness. Recompute AP when removing group members.
        Copies share the dictionary, arrays and features; keep them unchanged
        between AP calculation and testing. They keep ``feats`` and every pair
        (8 bytes per pair) alive with the frame, whichever null is used;
        drop them with ``frame.attrs.pop("copairs_pairs")``.

    Returns
    -------
    pd.DataFrame
        A DataFrame (with ``return_pairs``, a ``(DataFrame, dict)``
        tuple) containing the following columns:
        - 'average_precision': The calculated average precision score for each profile.
        - 'normalized_average_precision': The normalized AP score (scale-independent).
        - 'n_pos_pairs': The number of positive pairs for each profile.
        - 'n_total_pairs': The total number of pairs for each profile.
        - Additional metadata columns from the input.

    Raises
    ------
    UnpairedException
        If no positive or negative pairs are found in the dataset.

    Notes
    -----
    - Positive Pair Rules:
        * Positive pairs are defined by `pos_sameby` (profiles share these metadata values)
          and optionally differentiated by `pos_diffby` (profiles must differ in these metadata values if specified).
    - Negative Pair Rules:
        * Negative pairs are defined by `neg_diffby` (profiles differ in these metadata values)
          and optionally constrained by `neg_sameby` (profiles share these metadata values if specified).
    """
    # Combine all metadata columns needed for pair definitions
    columns = flatten_str_list(pos_sameby, pos_diffby, neg_sameby, neg_diffby)

    # Validate and filter metadata to ensure the required columns are present and usable
    meta, columns = evaluate_and_filter(meta, columns)
    validate_pipeline_input(meta, feats, columns)

    # Get the distance function for similarity calculations (e.g., cosine)
    similarity_fn = compute.get_similarity_fn(distance, progress_bar=progress_bar)

    # Reset metadata index for consistent indexing
    meta = meta.reset_index(drop=True).copy()

    logger.info("Indexing metadata...")

    # Identify positive pairs based on `pos_sameby` and `pos_diffby`
    logger.info("Finding positive pairs...")
    pos_pairs = find_pairs(meta, sameby=pos_sameby, diffby=pos_diffby)
    if len(pos_pairs) == 0:
        raise UnpairedException("Unable to find positive pairs.")

    # Identify negative pairs based on `neg_sameby` and `neg_diffby`
    logger.info("Finding negative pairs...")
    neg_pairs = find_pairs(meta, sameby=neg_sameby, diffby=neg_diffby)
    if len(neg_pairs) == 0:
        raise UnpairedException("Unable to find negative pairs.")

    # Compute similarities for positive pairs
    logger.info("Computing positive similarities...")
    pos_sims = similarity_fn(feats, pos_pairs, batch_size)

    # Compute similarities for negative pairs
    logger.info("Computing negative similarities...")
    neg_sims = similarity_fn(feats, neg_pairs, batch_size)

    # Build rank lists for calculating average precision
    logger.info("Building rank lists...")
    paired_ix, rel_k_list, counts = build_rank_lists(
        pos_pairs, neg_pairs, pos_sims, neg_sims
    )

    # Compute average precision scores and associated configurations
    logger.info("Computing average precision...")
    ap_scores, null_confs = compute.ap_contiguous(rel_k_list, counts)

    # Add AP scores and pair counts to the metadata DataFrame
    logger.info("Creating result DataFrame...")
    meta["n_pos_pairs"] = 0
    meta["n_total_pairs"] = 0
    meta.loc[paired_ix, "average_precision"] = ap_scores
    meta.loc[paired_ix, "n_pos_pairs"] = null_confs[:, 0]
    meta.loc[paired_ix, "n_total_pairs"] = null_confs[:, 1]

    # Compute normalized AP scores
    logger.info("Computing normalized average precision...")
    meta["normalized_average_precision"] = np.nan
    # Compute normalized scores for profiles with pairs
    M = null_confs[:, 0]  # n_pos_pairs
    L = null_confs[:, 1]  # n_total_pairs
    N = L - M  # n_neg_pairs
    normalized_scores = normalize_ap(ap_scores, M, N)
    meta.loc[paired_ix, "normalized_average_precision"] = normalized_scores

    logger.info("Finished.")
    pairs = {
        "pos": pos_pairs,
        "neg": neg_pairs,
        "similarity": _FeatureSimilarity(feats, distance, batch_size),
    }
    attach_pairs(meta, pairs)
    return (meta, pairs) if return_pairs else meta

build_rank_lists(pos_pairs, neg_pairs, pos_sims, neg_sims)

Build rank lists for calculating average precision.

This function processes positive and negative pairs along with their similarity scores to construct rank lists and determine unique profile indices with their associated counts.

Parameters:

• pos_pairs (ndarray) –

  Array of positive pair indices, where each pair is represented as a pair of integers.

• neg_pairs (ndarray) –

  Array of negative pair indices, where each pair is represented as a pair of integers.

• pos_sims (ndarray) –

  Array of similarity scores for positive pairs.

• neg_sims (ndarray) –

  Array of similarity scores for negative pairs.

Returns:

• paired_ix ( ndarray ) –

  Unique indices of profiles that appear in the rank lists.

• rel_k_list ( ndarray ) –

  Array of relevance labels (1 for positive pairs, 0 for negative pairs) sorted by decreasing similarity within each profile.

• counts ( ndarray ) –

  Array of counts indicating how many times each profile index appears in the rank lists.

Source code in src/copairs/map/average_precision.py

def build_rank_lists(
    pos_pairs: np.ndarray,
    neg_pairs: np.ndarray,
    pos_sims: np.ndarray,
    neg_sims: np.ndarray,
):
    """Build rank lists for calculating average precision.

    This function processes positive and negative pairs along with their similarity scores
    to construct rank lists and determine unique profile indices with their associated counts.

    Parameters
    ----------
    pos_pairs : np.ndarray
        Array of positive pair indices, where each pair is represented as a pair of integers.

    neg_pairs : np.ndarray
        Array of negative pair indices, where each pair is represented as a pair of integers.

    pos_sims : np.ndarray
        Array of similarity scores for positive pairs.

    neg_sims : np.ndarray
        Array of similarity scores for negative pairs.

    Returns
    -------
    paired_ix : np.ndarray
        Unique indices of profiles that appear in the rank lists.

    rel_k_list : np.ndarray
        Array of relevance labels (1 for positive pairs, 0 for negative pairs) sorted by
        decreasing similarity within each profile.

    counts : np.ndarray
        Array of counts indicating how many times each profile index appears in the rank lists.
    """
    # Combine relevance labels: 1 for positive pairs, 0 for negative pairs
    labels = np.concatenate(
        [
            np.ones(pos_pairs.size, dtype=np.uint32),
            np.zeros(neg_pairs.size, dtype=np.uint32),
        ]
    )

    # Flatten positive and negative pair indices for ranking
    ix = np.concatenate([pos_pairs.ravel(), neg_pairs.ravel()])

    # Expand similarity scores to match the flattened pair indices
    sim_all = np.concatenate([np.repeat(pos_sims, 2), np.repeat(neg_sims, 2)])

    # Sort by index (lexicographical order) and then by similarity (descending)
    # `1 - sim_all` ensures higher similarity values appear first, prioritizing
    # pairs with stronger similarity scores for ranking.
    # `ix` acts as a secondary criterion, ensuring consistent ordering of pairs
    # with equal similarity scores by their indices (lexicographical order).
    ix_sort = np.lexsort([1 - sim_all, ix])

    # Create the rank list of relevance labels sorted by similarity and index
    rel_k_list = labels[ix_sort]

    # Find unique profile indices and count their occurrences in the pairs
    paired_ix, counts = np.unique(ix, return_counts=True)

    return paired_ix, rel_k_list, counts.astype(np.uint32)

check_recomputable(similarity)

Raise if similarity cannot recompute its AP stage's similarities.

The permutation null ranks profiles by similarities it recomputes in its own batches of pairs; a metric that estimates its scale from each batch gives other values there than in the AP stage.

Source code in src/copairs/map/average_precision.py

def check_recomputable(similarity) -> None:
    """Raise if ``similarity`` cannot recompute its AP stage's similarities.

    The permutation null ranks profiles by similarities it recomputes in its
    own batches of pairs; a metric that estimates its scale from each batch
    gives other values there than in the AP stage.
    """
    if (
        isinstance(similarity, _FeatureSimilarity)
        and isinstance(similarity.distance, str)
        and similarity.distance in compute.BATCH_DEPENDENT_METRICS
    ):
        raise ValueError(
            f"distance={similarity.distance!r} estimates its scale from each "
            "batch of pairs, so the permutation null cannot recompute the "
            "similarities the APs were ranked by. Pass null='joint' or "
            "null='copied', which use the APs as computed."
        )

p_values(dframe, null_size, seed, progress_bar=True)

Compute p-values for average precision scores based on a null distribution.

This function calculates the p-values for each profile in the input DataFrame, comparing their average precision scores (average_precision) against a null distribution generated for their specific configurations (number of positive and total pairs). Profiles with no positive pairs are excluded from the p-value calculation.

Parameters:

• dframe (DataFrame) –

  A DataFrame containing the following columns: - average_precision: The AP scores for each profile. - n_pos_pairs: Number of positive pairs for each profile. - n_total_pairs: Total number of pairs (positive + negative) for each profile.

• null_size (int) –

  The number of samples to generate in the null distribution for significance testing.

• seed (int) –

  Random seed for reproducibility of the null distribution.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

Returns:

• ndarray –

  An array of p-values for each profile in the DataFrame. Profiles with no positive pairs will have NaN as their p-value.

Source code in src/copairs/map/average_precision.py

def p_values(
    dframe: pd.DataFrame, null_size: int, seed: int, progress_bar: bool = True
) -> np.ndarray:
    """Compute p-values for average precision scores based on a null distribution.

    This function calculates the p-values for each profile in the input DataFrame,
    comparing their average precision scores (`average_precision`) against a null
    distribution generated for their specific configurations (number of positive
    and total pairs). Profiles with no positive pairs are excluded from the p-value calculation.

    Parameters
    ----------
    dframe : pd.DataFrame
        A DataFrame containing the following columns:
        - `average_precision`: The AP scores for each profile.
        - `n_pos_pairs`: Number of positive pairs for each profile.
        - `n_total_pairs`: Total number of pairs (positive + negative) for each profile.
    null_size : int
        The number of samples to generate in the null distribution for significance testing.
    seed : int
        Random seed for reproducibility of the null distribution.
    progress_bar : bool
        Whether or not to show tqdm's progress bar.

    Returns
    -------
    np.ndarray
        An array of p-values for each profile in the DataFrame. Profiles with no positive
        pairs will have NaN as their p-value.
    """
    # Create a mask to filter profiles with at least one positive pair
    mask = dframe["n_pos_pairs"] > 0

    # Initialize the p-values array with NaN for all profiles
    pvals = np.full(len(dframe), np.nan, dtype=np.float32)

    # Extract the average precision scores and null configurations for valid profiles
    scores = dframe.loc[mask, "average_precision"].values
    null_confs = dframe.loc[mask, ["n_pos_pairs", "n_total_pairs"]].values

    # Compute p-values for profiles with valid configurations using the null distribution
    pvals[mask] = compute.p_values(scores, null_confs, null_size, seed, progress_bar)

    # Return the array of p-values, including NaN for invalid profiles
    return pvals

filter

Functions to support query-like syntax when finding the matches.

apply_filters(df, query_list)

Combine and apply query filters to a DataFrame.

This function takes a list of query expressions and applies them to a DataFrame to filter its rows. If no query expressions are provided, the original DataFrame is returned unchanged.

Parameters:

• df (DataFrame) –

  The DataFrame to which the filters will be applied.

• query_list (List[str]) –

  A list of query expressions (e.g., "column_name > 5"). These expressions should follow the syntax supported by pd.DataFrame.query.

Returns:

• DataFrame –

  The DataFrame filtered based on the provided query expressions.

Raises:

• ValueError: –
  • If the combined query results in an empty DataFrame.
  • If the combined query expression is invalid.

Source code in src/copairs/map/filter.py

def apply_filters(df: pd.DataFrame, query_list: List[str]) -> pd.DataFrame:
    """Combine and apply query filters to a DataFrame.

    This function takes a list of query expressions and applies them to a DataFrame
    to filter its rows. If no query expressions are provided, the original DataFrame
    is returned unchanged.

    Parameters
    ----------
    df : pd.DataFrame
        The DataFrame to which the filters will be applied.
    query_list : List[str]
        A list of query expressions (e.g., "column_name > 5"). These expressions
        should follow the syntax supported by `pd.DataFrame.query`.

    Returns
    -------
    pd.DataFrame
        The DataFrame filtered based on the provided query expressions.

    Raises
    ------
    ValueError:
        - If the combined query results in an empty DataFrame.
        - If the combined query expression is invalid.
    """
    # If no queries are provided, return the original DataFrame unchanged
    if not query_list:
        return df

    # Combine the query expressions into a single string using logical AND (&)
    combined_query = " & ".join(f"({query})" for query in query_list)

    try:
        # Apply the combined query to filter the DataFrame
        df_filtered = df.query(combined_query)

        # Raise an error if the filtered DataFrame is empty
        if df_filtered.empty:
            raise ValueError(f"No data matched the query: {combined_query}")
    except Exception as e:
        # Handle any issues with the query expression and provide feedback
        raise ValueError(
            f"Invalid combined query expression: {combined_query}. Error: {e}"
        )

    # Return the filtered DataFrame
    return df_filtered

evaluate_and_filter(df, columns)

Evaluate query filters and filter the metadata DataFrame based on specified columns.

This function processes column specifications, extracts any filter conditions, applies these conditions to the metadata DataFrame, and returns the filtered metadata along with the updated list of columns.

Parameters:

• df (DataFrame) –

  The metadata DataFrame containing information about profiles to be filtered.

• columns (List[str]) –

  A list of metadata column names.

Returns:

• Tuple[DataFrame, List[str]] –
  • The filtered metadata DataFrame.
  • The updated list of columns after processing any filter specifications.

Source code in src/copairs/map/filter.py

def evaluate_and_filter(
    df: pd.DataFrame, columns: List[str]
) -> Tuple[pd.DataFrame, List[str]]:
    """Evaluate query filters and filter the metadata DataFrame based on specified columns.

    This function processes column specifications, extracts any filter conditions,
    applies these conditions to the metadata DataFrame, and returns the filtered metadata
    along with the updated list of columns.

    Parameters
    ----------
    df : pd.DataFrame
        The metadata DataFrame containing information about profiles to be filtered.
    columns : List[str]
        A list of metadata column names.

    Returns
    -------
    Tuple[pd.DataFrame, List[str]]
        - The filtered metadata DataFrame.
        - The updated list of columns after processing any filter specifications.
    """
    # Extract query filters from the column specifications
    query_list, columns = extract_filters(columns, df.columns)

    # Apply the extracted filters to the metadata DataFrame
    df = apply_filters(df, query_list)

    # Return the filtered metadata DataFrame and the updated list of columns
    return df, columns

extract_filters(columns, df_columns)

Extract and validate query filters from selected metadata columns.

Parameters:

• columns (List[str]) –

  A list of selected metadata column names or query expressions. Query expressions should follow a valid syntax (e.g., "metadata_column > 5" or "metadata_column == 'value'").

• df_columns (List[str]) –

  All available metadata column names to validate against.

Returns:

• Tuple[List[str], List[str]] –
  • queries_to_eval: A list of valid query expressions to evaluate.
  • parsed_cols: A list of valid metadata column names extracted from the input columns.

Raises:

• ValueError: –
  • If a metadata column or query expression is invalid (e.g., references a non-existent column).
  • If duplicate queries are found for the same metadata column.

Source code in src/copairs/map/filter.py

def extract_filters(
    columns: List[str], df_columns: List[str]
) -> Tuple[List[str], List[str]]:
    """Extract and validate query filters from selected metadata columns.

    Parameters
    ----------
    columns : List[str]
        A list of selected metadata column names or query expressions. Query expressions
        should follow a valid syntax (e.g., "metadata_column > 5" or "metadata_column == 'value'").
    df_columns : List[str]
        All available metadata column names to validate against.

    Returns
    -------
    Tuple[List[str], List[str]]
        - `queries_to_eval`: A list of valid query expressions to evaluate.
        - `parsed_cols`: A list of valid metadata column names extracted from the input `columns`.

    Raises
    ------
    ValueError:
        - If a metadata column or query expression is invalid (e.g., references a non-existent column).
        - If duplicate queries are found for the same metadata column.
    """
    # Initialize lists to store parsed metadata column names and query expressions
    parsed_cols = []
    queries_to_eval = []

    # Iterate through each entry in the selected metadata columns
    for col in columns:
        if col in df_columns:
            # If the entry is a valid metadata column name, add it to parsed_cols
            parsed_cols.append(col)
            continue

        # Use regex to extract metadata column names from query expressions
        column_names = re.findall(r"(\w+)\s*[=<>!]+", col)

        # Validate the extracted metadata column names against all available metadata columns
        valid_column_names = [col for col in column_names if col in df_columns]
        if not valid_column_names:
            raise ValueError(f"Invalid query or metadata column name: {col}")

        # Add valid query expressions and associated metadata columns
        queries_to_eval.append(col)
        parsed_cols.extend(valid_column_names)

        # Check for duplicate metadata columns in the parsed list
        if len(parsed_cols) != len(set(parsed_cols)):
            raise ValueError(f"Duplicate queries for column: {col}")

    # Return the queries to evaluate and the parsed metadata column names
    return queries_to_eval, parsed_cols

flatten_str_list(*args)

Create a single list with all the params given.

Source code in src/copairs/map/filter.py

def flatten_str_list(*args):
    """Create a single list with all the params given."""
    columns = set()
    for col in args:
        if isinstance(col, str):
            columns.add(col)
        elif isinstance(col, dict):
            columns.update(itertools.chain.from_iterable(col.values()))
        else:
            columns.update(col)
    columns = list(columns)
    return columns

validate_pipeline_input(meta, feats, columns)

Validate the metadata and features for consistency and completeness.

Parameters:

• meta (DataFrame) –

  The metadata DataFrame describing the profiles.

• feats (ndarray) –

  The feature matrix where rows correspond to profiles in the metadata.

• columns (List[str]) –

  List of column names in the metadata to validate for null values.

Raises:

• ValueError: –
  • If any of the specified metadata columns contain null values.
  • If the number of rows in the metadata and features are not equal.
  • If the feature matrix contains null values.

Source code in src/copairs/map/filter.py

def validate_pipeline_input(
    meta: pd.DataFrame, feats: np.ndarray, columns: List[str]
) -> None:
    """Validate the metadata and features for consistency and completeness.

    Parameters
    ----------
    meta : pd.DataFrame
        The metadata DataFrame describing the profiles.
    feats : np.ndarray
        The feature matrix where rows correspond to profiles in the metadata.
    columns : List[str]
        List of column names in the metadata to validate for null values.

    Raises
    ------
    ValueError:
        - If any of the specified metadata columns contain null values.
        - If the number of rows in the metadata and features are not equal.
        - If the feature matrix contains null values.
    """
    # Check for null values in the specified metadata columns
    if meta[columns].isna().any(axis=None):
        raise ValueError("metadata columns should not have null values.")

    # Check if the number of rows in metadata matches the feature matrix
    if len(meta) != len(feats):
        raise ValueError("Metadata and features must have the same number of rows.")

    # Check for null values in the feature matrix
    if np.isnan(feats).any():
        raise ValueError("features should not have null values.")

hierarchical_fdr

Hierarchical FDR correction for grouped hypothesis testing.

apply_fdr_correction(map_scores, method='fdr_bh')

Apply standard FDR correction across all tests.

Parameters:

• map_scores (DataFrame) –

  DataFrame containing mAP scores with a 'p_value' column.

• method (str, default: 'fdr_bh' ) –

  Multiple testing correction method (default: 'fdr_bh'). See statsmodels.stats.multitest.multipletests for options.

Returns:

• DataFrame –

  Input DataFrame with 'corrected_p_value' column added.

Source code in src/copairs/map/hierarchical_fdr.py

def apply_fdr_correction(
    map_scores: pd.DataFrame,
    method: str = "fdr_bh",
) -> pd.DataFrame:
    """Apply standard FDR correction across all tests.

    Parameters
    ----------
    map_scores : pd.DataFrame
        DataFrame containing mAP scores with a 'p_value' column.
    method : str, optional
        Multiple testing correction method (default: 'fdr_bh').
        See statsmodels.stats.multitest.multipletests for options.

    Returns
    -------
    pd.DataFrame
        Input DataFrame with 'corrected_p_value' column added.

    """
    map_scores = map_scores.copy()
    _, pvals_corrected, _, _ = _multipletests(map_scores["p_value"], method=method)
    map_scores["corrected_p_value"] = pvals_corrected
    return map_scores

apply_hierarchical_fdr_correction(map_scores, hierarchical_by, sameby)

Apply hierarchical FDR correction for grouped hypotheses.

Implements a two-stage testing procedure appropriate for dose-response data where only high doses are expected to be active:

• Stage 1: Use minimum p-value within each group defined by hierarchical_by, then apply BH correction at the group level. A group passes if any member is significant.
• Stage 2: For groups that pass Stage 1, apply BH correction to the individual tests within each group.

Parameters:

• map_scores (DataFrame) –

  DataFrame containing mAP scores with a 'p_value' column.

• hierarchical_by (list) –

  Metadata column(s) defining the group structure (e.g., ['compound']).

• sameby (list) –

  Metadata column(s) used for mAP calculation (e.g., ['compound', 'dose']).

Returns:

• DataFrame –

  Input DataFrame with additional columns: - corrected_p_value: BH-corrected p-value (1.0 for groups that didn't pass Stage 1). - stage1_p_value: Group-level p-value from Stage 1 (minimum p-value). - stage1_corrected_p_value: BH-corrected Stage 1 p-value. - stage1_significant: Whether the group passed Stage 1.

Raises:

• ValueError –

  If hierarchical_by is not a proper subset of sameby.

Notes

This method uses minimum p-value (rather than Simes) for Stage 1 aggregation. Min-p is appropriate for dose-response data where only high doses are expected to be active. Simes would penalize compounds for having inactive low doses, which is the expected biological behavior.

Source code in src/copairs/map/hierarchical_fdr.py

def apply_hierarchical_fdr_correction(
    map_scores: pd.DataFrame,
    hierarchical_by: List[str],
    sameby: List[str],
) -> pd.DataFrame:
    """Apply hierarchical FDR correction for grouped hypotheses.

    Implements a two-stage testing procedure appropriate for dose-response data
    where only high doses are expected to be active:

    - Stage 1: Use minimum p-value within each group defined by `hierarchical_by`,
      then apply BH correction at the group level. A group passes if any member
      is significant.
    - Stage 2: For groups that pass Stage 1, apply BH correction to the
      individual tests within each group.

    Parameters
    ----------
    map_scores : pd.DataFrame
        DataFrame containing mAP scores with a 'p_value' column.
    hierarchical_by : list
        Metadata column(s) defining the group structure (e.g., ['compound']).
    sameby : list
        Metadata column(s) used for mAP calculation (e.g., ['compound', 'dose']).

    Returns
    -------
    pd.DataFrame
        Input DataFrame with additional columns:
        - `corrected_p_value`: BH-corrected p-value (1.0 for groups that didn't pass Stage 1).
        - `stage1_p_value`: Group-level p-value from Stage 1 (minimum p-value).
        - `stage1_corrected_p_value`: BH-corrected Stage 1 p-value.
        - `stage1_significant`: Whether the group passed Stage 1.

    Raises
    ------
    ValueError
        If `hierarchical_by` is not a proper subset of `sameby`.

    Notes
    -----
    This method uses minimum p-value (rather than Simes) for Stage 1 aggregation.
    Min-p is appropriate for dose-response data where only high doses are expected
    to be active. Simes would penalize compounds for having inactive low doses,
    which is the expected biological behavior.

    """
    sameby = check_group_columns(sameby, "sameby", map_scores)
    hierarchical_by = check_group_columns(hierarchical_by, "hierarchical_by")
    check_hierarchical_by(hierarchical_by, sameby)

    logger.info("Applying hierarchical FDR correction...")
    map_scores = map_scores.copy()

    # Stage 1: Aggregate p-values to group level using minimum p-value
    # Min-p is appropriate for dose-response where only high doses are expected to be active
    stage1_pvals = map_scores.groupby(hierarchical_by, observed=True).agg(
        {"p_value": "min"}
    )
    stage1_pvals.columns = ["stage1_p_value"]

    # Apply BH correction at the group level
    reject_stage1, stage1_corrected, _, _ = _multipletests(
        stage1_pvals["stage1_p_value"], method="fdr_bh"
    )
    stage1_pvals["stage1_corrected_p_value"] = stage1_corrected
    stage1_pvals["stage1_significant"] = reject_stage1

    # Merge Stage 1 results back to map_scores
    map_scores = map_scores.merge(
        stage1_pvals.reset_index(), on=hierarchical_by, how="left"
    )

    # Stage 2: For groups that passed Stage 1, apply BH within each group
    # For groups that didn't pass, set corrected_p_value to 1.0
    map_scores["corrected_p_value"] = np.where(
        map_scores["p_value"].notna(), 1.0, np.nan
    )

    for group_key, group_df in map_scores.groupby(hierarchical_by, observed=True):
        passed = group_df["stage1_significant"].iloc[0]
        if pd.isna(passed) or not passed:
            # Group didn't pass Stage 1, skip
            continue

        group_indices = group_df.index
        group_pvals = group_df["p_value"].values

        if len(group_pvals) == 1:
            # Single test in group, no additional correction needed
            map_scores.loc[group_indices, "corrected_p_value"] = group_pvals[0]
        else:
            # Apply BH correction within the group
            _, group_corrected, _, _ = _multipletests(group_pvals, method="fdr_bh")
            map_scores.loc[group_indices, "corrected_p_value"] = group_corrected

    return map_scores

check_group_columns(columns, name, frame=None)

Return a nonempty list of distinct grouping column names, or raise.

Source code in src/copairs/map/hierarchical_fdr.py

def check_group_columns(columns, name, frame=None):
    """Return a nonempty list of distinct grouping column names, or raise."""
    if isinstance(columns, str):
        columns = [columns]
    elif not isinstance(columns, (list, tuple, pd.Index)):
        raise ValueError(
            f"{name} must be a column name or a nonempty list of column names"
        )
    columns = list(columns)
    if not columns or any(
        not isinstance(column, str) or not column for column in columns
    ):
        raise ValueError(f"{name} must contain nonempty string column names")
    if len(set(columns)) != len(columns):
        raise ValueError(f"{name} must contain distinct column names")
    if frame is not None:
        for column in columns:
            if list(frame.columns).count(column) != 1:
                raise ValueError(
                    f"{name} column {column!r} must occur exactly once in the input"
                )
    return columns

check_hierarchical_by(hierarchical_by, sameby)

Raise ValueError unless hierarchical_by is a nonempty proper subset.

Source code in src/copairs/map/hierarchical_fdr.py

def check_hierarchical_by(hierarchical_by: list[str], sameby: list[str]) -> None:
    """Raise ValueError unless ``hierarchical_by`` is a nonempty proper subset."""
    hierarchical_by = check_group_columns(hierarchical_by, "hierarchical_by")
    sameby = check_group_columns(sameby, "sameby")
    if not set(hierarchical_by).issubset(set(sameby)):
        raise ValueError(
            f"hierarchical_by columns {hierarchical_by} must be a subset of "
            f"sameby columns {sameby}"
        )

    if set(hierarchical_by) == set(sameby):
        raise ValueError(
            f"hierarchical_by columns {hierarchical_by} must be a proper subset of "
            f"sameby columns {sameby}. If they are equal, use standard correction "
            f"by not specifying hierarchical_by."
        )

map

Functions to compute mean average precision.

get_map_pvalue(ap_scores, sameby, null_size, seed=0, progress_bar=True, max_workers=None, cache_dir=None, pos_pairs=None, null='permutation', pairs=None, *, max_reference_profiles=16384, pool_memory_budget=4 * 1024 ** 3, key_table_memory_budget=256 * 1024 ** 2)

Compute mAP scores and p-values from AP scores.

This function groups AP scores by the specified columns, computes the mean Average Precision (mAP) for each group, and calculates p-values by comparing against the null distribution of the group's mAP chosen by null. A null mAP counts against the group when it is >= the group's mAP, so an mAP equal to an atom of the null (e.g. perfect retrieval) counts the atom; values within the scores' rounding count as ties (copairs.nulls.tie_thresholds).

Parameters:

• ap_scores (DataFrame) –

  DataFrame containing individual Average Precision (AP) scores and pair statistics (e.g., number of positive pairs n_pos_pairs and total pairs n_total_pairs).

• sameby (list or str) –

  Nonempty, distinct string column names present in the input.

• null_size (int) –

  Number of Monte Carlo draws. Zero requests scores only: p-values are NaN after input/layout validation, without sampling or null-only tables.

• seed (int or None, default: 0 ) –

  Reproducible seed; None resolves fresh entropy once per run. Joint and permutation accept integers in [0, 2**64); copied accepts any integer >= 0.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

• max_workers (int, default: None ) –

  Threads that compute the copied null's p-values or run the permutation null's groups. Default defined by concurrent.futures.ThreadPoolExecutor.

• cache_dir (str or Path, default: None ) –

  Location to save the cache of the copied null's AP samples.

• pos_pairs (ndarray, default: None ) –

  The (n_pairs, 2) positive pairs used to compute the AP scores, indexing the rows of the frame average_precision returned (its return_pairs=True gives them, as pairs["pos"]); ap_scores must keep that frame's index. Pairs whose counts disagree with n_pos_pairs raise. When omitted, they are taken from pairs or, failing that, from the pairs average_precision attached to ap_scores. Multi-member groups require actual positive pairs; counts alone do not identify their graph. Singletons can use their AP null without pairs. If a pair handle is available, explicit positive pairs must agree with it. Joint null only.

• null (str, default: 'permutation' ) –

  Null of the group mAP. "permutation" (default) relabels profiles (copairs.nulls.permutation): in each block of members ranked against the same references (e.g. a plate's replicates and controls), any of the block's profiles could have carried the group's label, and each draw recomputes the group's mAP from the observed similarities. All positive edges must stay inside the tested group; the graph need not be complete. Original metadata rules define the graph and reference pools and are not reapplied per draw. A singleton with external positives can use "joint", its AP null. Positive draw counts use Monte Carlo sampling with (hits + 1)/(draws + 1). It recomputes similarities, so APs computed with a distance that estimates its scale per batch ("seuclidean", "mahalanobis") raise. Multilabel pair dictionaries use negative-graph components, preserving member counts per component and other labels, with the original positive graph. Negatives follow the original negative/lone-pair structure; counts can change. Each group must contain one complete label. Overlapping reference sets are allowed for this explicitly distinct multilabel model. "joint" samples the members' APs together (copairs.nulls.joint): a positive pair inside the group is one random number in both members' rank lists, and other similarities are independent, which real profiles are not (members share references, and some profiles are close to many others), so it can give too small p-values for groups of many replicates. "copied" samples each member's AP null per (n_pos_pairs, n_total_pairs) configuration and the group's null averages them, so members with equal pair counts share one draw and the null does not narrow as replicates are added.

• pairs (dict, default: None ) –

  Dictionary with "pos", "neg" and "similarity" entries, as average_precision(..., return_pairs=True) returns them; their indices are rows of that call's frame, so ap_scores must keep its index. The permutation null needs them and, when not given, takes them from ap_scores.attrs["copairs_pairs"], where average_precision attaches them; frames that lost them (built, reloaded, pickled, or concatenated from several AP calls) need pairs= or another null. The joint null takes its positive pairs from pairs too, or from the attached pairs (see pos_pairs). Multilabel dictionaries also carry "label_col", "profile", "row_key", "keys" and the lazy "_lone" pair function; "label_col" selects the multilabel permutation path. Explicit pairs override attachments. Keep the original index, matching pairs and complete tested groups/labels; validation checks structure, not source identity or group completeness. Recompute AP when removing group members. Copies share the dictionary, features and arrays; keep them unchanged between AP and mAP. Custom similarities must return the same value for a pair regardless of batch size or contents.

• max_reference_profiles (int, default: 16384 ) –

  Maximum total reference profiles across a tested group's blocks; >= 0. For multilabel groups, component-union size minus its members; all components count. Pool-table estimates include members too. Checked before quadratic pool tables are allocated.

• pool_memory_budget (int, default: 4294967296 ) –

  Positive byte budget for estimated concurrent pool key/rank tables. One oversized pool may run alone. This does not cap total process RAM.

• key_table_memory_budget (int, default: 268435456 ) –

  Byte limit for optional cross-pool key reuse; zero disables it. Neither budget includes features, layouts or all temporary arrays.

Returns:

• DataFrame –

  DataFrame with the following columns: - Columns from sameby (group identifiers). - mean_average_precision: Mean AP score for each group. - mean_normalized_average_precision: Mean normalized AP score (scale-independent). - p_value: p-value comparing mAP to the null distribution. - indices: The group's rows, as positions in ap_scores after rows with a NaN AP or no positive pairs are dropped.

Raises:

• UnpairedException –

  If no row has a non-NaN AP, positive pairs and a value in every sameby column, e.g. an empty table.

• ValueError –

  Invalid grouping, pair dictionaries/counts, layout or resource arguments. max_workers must be a positive integer or None even for zero draws and joint sampling, which does not use that executor.

Source code in src/copairs/map/map.py

def get_map_pvalue(
    ap_scores: pd.DataFrame,
    sameby: List[str],
    null_size: int,
    seed: int | None = 0,
    progress_bar: bool = True,
    max_workers: Optional[int] = None,
    cache_dir: Optional[Union[str, Path]] = None,
    pos_pairs: Optional[np.ndarray] = None,
    null: str = "permutation",
    pairs: dict | None = None,
    *,
    max_reference_profiles: int = 16_384,
    pool_memory_budget: int = 4 * 1024**3,
    key_table_memory_budget: int = 256 * 1024**2,
) -> pd.DataFrame:
    """Compute mAP scores and p-values from AP scores.

    This function groups AP scores by the specified columns, computes the mean
    Average Precision (mAP) for each group, and calculates p-values by comparing
    against the null distribution of the group's mAP chosen by `null`. A null
    mAP counts against the group when it is ``>=`` the group's mAP, so an mAP
    equal to an atom of the null (e.g. perfect retrieval) counts the atom; values
    within the scores' rounding count as ties (`copairs.nulls.tie_thresholds`).

    Parameters
    ----------
    ap_scores : pd.DataFrame
        DataFrame containing individual Average Precision (AP) scores and pair statistics
        (e.g., number of positive pairs `n_pos_pairs` and total pairs `n_total_pairs`).
    sameby : list or str
        Nonempty, distinct string column names present in the input.
    null_size : int
        Number of Monte Carlo draws. Zero requests scores only: p-values are NaN
        after input/layout validation, without sampling or null-only tables.
    seed : int or None, default 0
        Reproducible seed; None resolves fresh entropy once per run. Joint and
        permutation accept integers in [0, 2**64); copied accepts any integer >= 0.
    progress_bar : bool
        Whether or not to show tqdm's progress bar.
    max_workers : int
        Threads that compute the copied null's p-values or run the
        permutation null's groups. Default defined by
        `concurrent.futures.ThreadPoolExecutor`.
    cache_dir : str or Path
        Location to save the cache of the copied null's AP samples.
    pos_pairs : np.ndarray, optional
        The `(n_pairs, 2)` positive pairs used to compute the AP scores,
        indexing the rows of the frame `average_precision` returned (its
        `return_pairs=True` gives them, as `pairs["pos"]`); `ap_scores` must keep
        that frame's index. Pairs whose counts disagree with `n_pos_pairs`
        raise. When omitted, they are taken from ``pairs`` or, failing that,
        from the pairs `average_precision` attached to ``ap_scores``. Multi-member
        groups require actual positive pairs; counts alone do not identify their
        graph. Singletons can use their AP null without pairs. If a pair handle
        is available, explicit positive pairs must agree with it. Joint null only.
    null : str
        Null of the group mAP. ``"permutation"`` (default) relabels profiles
        (`copairs.nulls.permutation`): in each block of members ranked against
        the same references (e.g. a plate's replicates and controls), any of
        the block's profiles could have carried the group's label, and each
        draw recomputes the group's mAP from the observed similarities. All
        positive edges must stay inside the tested group; the graph need not
        be complete. Original metadata rules define the graph and reference
        pools and are not reapplied per draw. A singleton with external
        positives can use ``"joint"``, its AP null. Positive draw counts use
        Monte Carlo sampling with (hits + 1)/(draws + 1). It recomputes similarities, so
        APs computed with a distance that estimates its scale per batch
        (``"seuclidean"``, ``"mahalanobis"``) raise. Multilabel pair dictionaries
        use negative-graph components, preserving member counts per component
        and other labels, with the original positive graph. Negatives follow
        the original negative/lone-pair structure; counts can change. Each
        group must contain one complete label. Overlapping reference sets are
        allowed for this explicitly distinct multilabel model. ``"joint"`` samples
        the members' APs together (`copairs.nulls.joint`): a positive pair
        inside the group is one random number in both members' rank lists,
        and other similarities are independent, which real profiles are not
        (members share references, and some profiles are close to many
        others), so it can give too small p-values for groups of many
        replicates. ``"copied"`` samples each member's AP null per
        `(n_pos_pairs, n_total_pairs)`
        configuration and the group's null averages them, so members with
        equal pair counts share one draw and the null does not narrow as
        replicates are added.
    pairs : dict, optional
        Dictionary with ``"pos"``, ``"neg"`` and ``"similarity"`` entries, as
        ``average_precision(..., return_pairs=True)`` returns them; their
        indices are rows of that call's frame, so ``ap_scores`` must keep its
        index. The permutation null needs them and, when not given, takes
        them from ``ap_scores.attrs["copairs_pairs"]``, where
        `average_precision` attaches them; frames that lost them (built,
        reloaded, pickled, or concatenated from several AP calls) need
        ``pairs=`` or another null. The joint null takes its positive pairs
        from ``pairs`` too, or from the attached pairs (see ``pos_pairs``).
        Multilabel dictionaries also carry ``"label_col"``, ``"profile"``,
        ``"row_key"``, ``"keys"`` and the lazy ``"_lone"`` pair function;
        ``"label_col"`` selects the multilabel permutation path.
        Explicit pairs override attachments. Keep the original index, matching
        pairs and complete tested groups/labels; validation checks structure,
        not source identity or group completeness. Recompute AP when removing
        group members. Copies share the dictionary, features and arrays; keep
        them unchanged between AP and mAP. Custom similarities must return the
        same value for a pair regardless of batch size or contents.
    max_reference_profiles : int, default 16384
        Maximum total reference profiles across a tested group's blocks; >= 0.
        For multilabel groups, component-union size minus its members; all
        components count. Pool-table estimates include members too.
        Checked before quadratic pool tables are allocated.
    pool_memory_budget : int, default 4294967296
        Positive byte budget for estimated concurrent pool key/rank tables.
        One oversized pool may run alone. This does not cap total process RAM.
    key_table_memory_budget : int, default 268435456
        Byte limit for optional cross-pool key reuse; zero disables it.
        Neither budget includes features, layouts or all temporary arrays.

    Returns
    -------
    pd.DataFrame
        DataFrame with the following columns:
        - Columns from `sameby` (group identifiers).
        - `mean_average_precision`: Mean AP score for each group.
        - `mean_normalized_average_precision`: Mean normalized AP score (scale-independent).
        - `p_value`: p-value comparing mAP to the null distribution.
        - `indices`: The group's rows, as positions in ap_scores after rows
          with a NaN AP or no positive pairs are dropped.

    Raises
    ------
    UnpairedException
        If no row has a non-NaN AP, positive pairs and a value in every
        `sameby` column, e.g. an empty table.
    ValueError
        Invalid grouping, pair dictionaries/counts, layout or resource arguments.
        ``max_workers`` must be a positive integer or None even for zero draws
        and joint sampling, which does not use that executor.

    """
    null = check_null(null)
    null_size = check_null_size(null_size)
    sameby = check_group_columns(sameby, "sameby", ap_scores)
    max_workers = check_max_workers(max_workers)
    resources = dict(
        zip(
            ("max_reference_profiles", "pool_memory_budget", "key_table_memory_budget"),
            permutation.check_resources(
                max_reference_profiles, pool_memory_budget, key_table_memory_budget
            ),
        )
    )
    if null != "copied":
        check_seed(seed)
    elif seed is not None and (int(seed) != seed or seed < 0):  # NumPy: any int >= 0
        raise ValueError(f"seed must be a non-negative integer, got {seed}")
    if null != "joint" and pos_pairs is not None:
        raise ValueError("pos_pairs is used by the joint null only")
    if null != "copied" and pairs is None:
        pairs = attached_pairs(ap_scores)
    if pairs is not None and null != "copied":
        if (
            not isinstance(pairs, dict)
            or not {"pos", "neg", "similarity"} <= pairs.keys()
        ):
            raise ValueError(
                "pairs must be a dictionary with pos, neg and similarity entries"
            )
        if pos_pairs is not None and not np.array_equal(
            check_pairs(pos_pairs, "pos_pairs"),
            check_pairs(pairs["pos"], "positive pairs"),
        ):
            raise ValueError(
                "pos_pairs must index the rows of the AP frame and match the selected handle's positive pairs"
            )
    if null == "joint" and pos_pairs is None and pairs is not None:
        pos_pairs = pairs["pos"]
    if null == "permutation" and pairs is None:
        raise ValueError(
            "null='permutation' (the default) needs the pairs the APs were "
            "computed from. average_precision attaches them to the frame it "
            "returns, and these AP scores have none: they were built, loaded "
            "from disk, pickled or concatenated from several AP calls. "
            "Both average_precision and multilabel.average_precision attach pairs. "
            "Recompute the APs, pass pairs=, or choose null='joint' "
            "or null='copied'."
        )
    if null == "permutation":
        check_recomputable(pairs["similarity"])

    if null == "permutation" or pos_pairs is not None:
        _check_row_index(ap_scores.index)

    # Filter out invalid or incomplete AP scores, remembering the original rows.
    # Rows missing a sameby value stay (the groupby drops them), so `indices`
    # keep their positions; the check counts only rows a group can take.
    valid = (
        ap_scores["average_precision"].notna() & (ap_scores["n_pos_pairs"] > 0)
    ).fillna(False)
    if not (valid & ap_scores[sameby].notna().all(axis=1)).any():
        raise UnpairedException(
            "Unable to find valid AP scores: no row has a non-NaN AP, positive "
            "pairs and a value in every sameby column."
        )
    missing_keys = valid & ~ap_scores[sameby].notna().all(axis=1)
    tested = ap_scores.loc[valid & ~missing_keys]
    counts = _normalize_pair_counts(tested)
    ap_values = tested["average_precision"].to_numpy(dtype=np.float64)
    if not np.isfinite(ap_values).all() or ((ap_values < 0) | (ap_values > 1)).any():
        raise ValueError("usable AP scores must be finite values in [0, 1]")
    if (~valid).any() or missing_keys.any():
        logger.warning(
            "Excluding %d rows with missing AP/no positive pairs and %d rows with missing group keys",
            int((~valid).sum()),
            int(missing_keys.sum()),
        )
    ap_scores = ap_scores[valid]
    profile = ap_scores.index.to_numpy()
    ap_scores = ap_scores.reset_index(drop=True)
    # Keep positional indices for rows without group keys, but never cast their
    # unused counts (which may be missing/invalid) or pass them into a sampler.
    normalized = np.zeros((len(ap_scores), 2), dtype=np.int64)
    normalized[(~missing_keys[valid]).to_numpy(dtype=bool)] = counts
    ap_scores[["n_pos_pairs", "n_total_pairs"]] = normalized

    # Average the APs in float64 and round the means to the APs' dtype: ties
    # carry the APs' rounding, and pandas averages float32 APs to float32 or
    # float64 depending on its version and the group keys. The reported mAP
    # is then the score the nulls count ties against, under any pandas.
    ap_dtype = getattr(
        ap_scores["average_precision"].dtype,
        "numpy_dtype",
        ap_scores["average_precision"].to_numpy().dtype,
    )
    score_dtype = ap_dtype if ap_dtype.kind == "f" else np.dtype(np.float64)
    ap_scores["average_precision"] = ap_scores["average_precision"].astype(np.float64)

    # Group by the specified metadata column(s) and calculate mean AP
    map_scores = ap_scores.groupby(sameby, observed=True, as_index=False).agg(
        {
            "average_precision": ["mean", lambda x: list(x.index)],
            "normalized_average_precision": "mean",
        }
    )
    map_scores.columns = sameby + [
        "mean_average_precision",
        "indices",
        "mean_normalized_average_precision",
    ]
    map_scores["mean_average_precision"] = map_scores["mean_average_precision"].astype(
        score_dtype
    )
    groups = map_scores["indices"].tolist()
    scores = map_scores["mean_average_precision"].to_numpy()
    copied = dict(
        null_size=null_size,
        seed=seed,
        cache_dir=cache_dir,
        progress_bar=progress_bar,
        max_workers=max_workers,
    )

    if null == "copied":
        map_scores["p_value"] = (
            np.full(len(groups), np.nan)
            if null_size == 0
            else _copied_pvalues(scores, groups, ap_scores, **copied)
        )
        return map_scores

    if null == "permutation":
        logger.info("Computing p-values...")
        arguments = (
            groups,
            profile,
            ap_scores["n_pos_pairs"].to_numpy(),
            ap_scores["n_total_pairs"].to_numpy(),
        )
        options = dict(
            observed=scores,
            progress_bar=progress_bar,
            max_workers=max_workers,
            **resources,
        )
        if "label_col" in pairs:
            pvalues = permutation.label_permutation_pvalues(
                *arguments, pairs, null_size, seed, **options
            )
        else:
            pvalues = permutation.permutation_pvalues(
                *arguments,
                pairs["pos"],
                pairs["neg"],
                pairs["similarity"],
                null_size,
                seed,
                **options,
            )
        map_scores["p_value"] = pvalues
        return map_scores

    num_pos = ap_scores["n_pos_pairs"].to_numpy().astype(np.int64)
    num_neg = ap_scores["n_total_pairs"].to_numpy().astype(np.int64) - num_pos
    logger.info("Computing p-values...")
    map_scores["p_value"] = joint.group_pvalues(
        scores, groups, num_pos, num_neg, null_size, seed, profile, pos_pairs,
        progress_bar=progress_bar,
    )  # fmt: skip
    return map_scores

mean_average_precision(ap_scores, sameby, null_size, threshold, seed=0, progress_bar=True, max_workers=None, cache_dir=None, pos_pairs=None, null='permutation', pairs=None, *, max_reference_profiles=16384, pool_memory_budget=4 * 1024 ** 3, key_table_memory_budget=256 * 1024 ** 2)

Calculate the Mean Average Precision (mAP) score and associated p-values.

This function computes the Mean Average Precision (mAP) score by grouping profiles based on the specified criteria (sameby). It calculates the significance of mAP scores by comparing them to a null distribution and performs multiple testing corrections using Benjamini-Hochberg FDR.

Per-call resource controls (max_reference_profiles, pool_memory_budget, key_table_memory_budget), pair dictionaries and supported layouts follow :func:get_map_pvalue. With null_size=0 statistical columns are NaN and significance flags are nullable booleans containing pd.NA; correction is skipped.

Parameters:

• ap_scores (DataFrame) –

  DataFrame containing individual Average Precision (AP) scores and pair statistics (e.g., number of positive pairs n_pos_pairs and total pairs n_total_pairs).

• sameby (list or str) –

  Metadata column(s) used to group profiles for mAP calculation.

• null_size (int) –

  Number of samples in the null distribution for significance testing.

• threshold (float) –

  p-value threshold for identifying significant MaP scores.

• seed (int or None, default: 0 ) –

  Reproducible seed; None requests fresh randomness once per run.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

• max_workers (int, default: None ) –

  Threads that compute the copied null's p-values or run the permutation null's groups. Default defined by concurrent.futures.ThreadPoolExecutor.

• cache_dir (str or Path, default: None ) –

  Location to save the cache of the copied null's AP samples.

• pos_pairs (ndarray, default: None ) –

  Positive pairs used to compute the AP scores, for null="joint"; see get_map_pvalue. Needed there when a group's positives are not its other members.

• null (str, default: 'permutation' ) –

  Null of the group mAP: "permutation" (default), "joint" or "copied"; see get_map_pvalue.

• pairs (dict, default: None ) –

  Pair dictionary of the AP scores, needed by null="permutation" and attached to AP frames; see get_map_pvalue.

Returns:

• DataFrame –

  DataFrame with the following columns: - mean_average_precision: Mean AP score for each group. - mean_normalized_average_precision: Mean normalized AP score (scale-independent). - p_value: p-value comparing mAP to the null distribution. - corrected_p_value: Adjusted p-value after multiple testing correction. - below_p: Boolean indicating if the p-value is below the threshold. - below_corrected_p: Boolean indicating if the corrected p-value is below the threshold.

Raises:

• UnpairedException –

  If no row has a non-NaN AP, positive pairs and a value in every sameby column, e.g. an empty table.

See Also

mean_average_precision_hierarchical : For hierarchical FDR correction with grouped data.

Source code in src/copairs/map/map.py

def mean_average_precision(
    ap_scores: pd.DataFrame,
    sameby: List[str],
    null_size: int,
    threshold: float,
    seed: int | None = 0,
    progress_bar: bool = True,
    max_workers: Optional[int] = None,
    cache_dir: Optional[Union[str, Path]] = None,
    pos_pairs: Optional[np.ndarray] = None,
    null: str = "permutation",
    pairs: dict | None = None,
    *,
    max_reference_profiles: int = 16_384,
    pool_memory_budget: int = 4 * 1024**3,
    key_table_memory_budget: int = 256 * 1024**2,
) -> pd.DataFrame:
    """Calculate the Mean Average Precision (mAP) score and associated p-values.

    This function computes the Mean Average Precision (mAP) score by grouping profiles
    based on the specified criteria (`sameby`). It calculates the significance of mAP
    scores by comparing them to a null distribution and performs multiple testing
    corrections using Benjamini-Hochberg FDR.

    Per-call resource controls (``max_reference_profiles``, ``pool_memory_budget``,
    ``key_table_memory_budget``), pair dictionaries and supported layouts follow
    :func:`get_map_pvalue`. With ``null_size=0`` statistical columns are NaN and
    significance flags are nullable booleans containing pd.NA; correction is skipped.

    Parameters
    ----------
    ap_scores : pd.DataFrame
        DataFrame containing individual Average Precision (AP) scores and pair statistics
        (e.g., number of positive pairs `n_pos_pairs` and total pairs `n_total_pairs`).
    sameby : list or str
        Metadata column(s) used to group profiles for mAP calculation.
    null_size : int
        Number of samples in the null distribution for significance testing.
    threshold : float
        p-value threshold for identifying significant MaP scores.
    seed : int or None, default 0
        Reproducible seed; None requests fresh randomness once per run.
    progress_bar : bool
        Whether or not to show tqdm's progress bar.
    max_workers : int
        Threads that compute the copied null's p-values or run the
        permutation null's groups. Default defined by
        `concurrent.futures.ThreadPoolExecutor`.
    cache_dir : str or Path
        Location to save the cache of the copied null's AP samples.
    pos_pairs : np.ndarray, optional
        Positive pairs used to compute the AP scores, for ``null="joint"``;
        see `get_map_pvalue`. Needed there when a group's positives are not
        its other members.
    null : str
        Null of the group mAP: ``"permutation"`` (default), ``"joint"`` or
        ``"copied"``; see `get_map_pvalue`.
    pairs : dict, optional
        Pair dictionary of the AP scores, needed by ``null="permutation"`` and
        attached to AP frames; see `get_map_pvalue`.

    Returns
    -------
    pd.DataFrame
        DataFrame with the following columns:
        - `mean_average_precision`: Mean AP score for each group.
        - `mean_normalized_average_precision`: Mean normalized AP score (scale-independent).
        - `p_value`: p-value comparing mAP to the null distribution.
        - `corrected_p_value`: Adjusted p-value after multiple testing correction.
        - `below_p`: Boolean indicating if the p-value is below the threshold.
        - `below_corrected_p`: Boolean indicating if the corrected p-value is below the threshold.

    Raises
    ------
    UnpairedException
        If no row has a non-NaN AP, positive pairs and a value in every
        `sameby` column, e.g. an empty table.

    See Also
    --------
    mean_average_precision_hierarchical : For hierarchical FDR correction with grouped data.

    """
    # Step 1: Compute mAP scores and p-values
    map_scores = get_map_pvalue(
        ap_scores=ap_scores,
        sameby=sameby,
        null_size=null_size,
        seed=seed,
        progress_bar=progress_bar,
        max_workers=max_workers,
        cache_dir=cache_dir,
        pos_pairs=pos_pairs,
        null=null,
        pairs=pairs,
        max_reference_profiles=max_reference_profiles,
        pool_memory_budget=pool_memory_budget,
        key_table_memory_budget=key_table_memory_budget,
    )

    if null_size == 0:
        map_scores["corrected_p_value"] = np.nan
    else:
        map_scores = apply_fdr_correction(map_scores)
    return _mark_significance(map_scores, threshold)

mean_average_precision_hierarchical(ap_scores, sameby, null_size, threshold, seed=0, hierarchical_by=None, progress_bar=True, max_workers=None, cache_dir=None, pos_pairs=None, null='permutation', pairs=None, *, max_reference_profiles=16384, pool_memory_budget=4 * 1024 ** 3, key_table_memory_budget=256 * 1024 ** 2)

Calculate the Mean Average Precision (mAP) score with hierarchical FDR correction.

This function computes the Mean Average Precision (mAP) score by grouping profiles based on the specified criteria (sameby). It applies hierarchical FDR correction appropriate for grouped hypothesis testing, such as dose-response data.

Per-call resource controls (max_reference_profiles, pool_memory_budget, key_table_memory_budget), pair dictionaries and supported layouts follow :func:get_map_pvalue. With null_size=0 all statistical columns, including stage-1 p-values, are NaN; significance flags contain pd.NA and correction is skipped. Supply hierarchical_by explicitly, preferably as a keyword.

Parameters:

• ap_scores (DataFrame) –

  DataFrame containing individual Average Precision (AP) scores and pair statistics (e.g., number of positive pairs n_pos_pairs and total pairs n_total_pairs).

• sameby (list or str) –

  Metadata column(s) used to group profiles for mAP calculation.

• null_size (int) –

  Number of samples in the null distribution for significance testing.

• threshold (float) –

  p-value threshold for identifying significant MaP scores.

• seed (int or None, default: 0 ) –

  Reproducible seed; None requests fresh randomness once per run.

• hierarchical_by (list, default: None ) –

  Metadata column(s) for hierarchical FDR correction. Enables two-stage testing:

  • Stage 1: Use minimum p-value within each group defined by hierarchical_by, then apply BH correction at the group level. A group passes if any member is significant.
  • Stage 2: For groups that pass Stage 1, apply BH correction to the individual tests within each group.

  This is designed for dose-response data where only high doses are expected to be active. The hierarchical_by columns must be a proper subset of sameby. For example, with sameby=['compound', 'dose'] and hierarchical_by=['compound'], mAP is calculated per compound×dose, but FDR correction accounts for the grouped structure.

• progress_bar (bool, default: True ) –

  Whether or not to show tqdm's progress bar.

• max_workers (int, default: None ) –

  Threads that compute the copied null's p-values or run the permutation null's groups. Default defined by concurrent.futures.ThreadPoolExecutor.

• cache_dir (str or Path, default: None ) –

  Location to save the cache of the copied null's AP samples.

• pos_pairs (ndarray, default: None ) –

  Positive pairs used to compute the AP scores, for null="joint"; see get_map_pvalue. Needed there when a group's positives are not its other members.

• null (str, default: 'permutation' ) –

  Null of the group mAP: "permutation" (default), "joint" or "copied"; see get_map_pvalue.

• pairs (dict, default: None ) –

  Pair dictionary of the AP scores, needed by null="permutation" and attached to AP frames; see get_map_pvalue.

Returns:

• DataFrame –

  DataFrame with the following columns: - mean_average_precision: Mean AP score for each group. - mean_normalized_average_precision: Mean normalized AP score (scale-independent). - p_value: p-value comparing mAP to the null distribution. - corrected_p_value: Adjusted p-value after multiple testing correction. - below_p: Boolean indicating if the p-value is below the threshold. - below_corrected_p: Boolean indicating if the corrected p-value is below the threshold. - stage1_p_value: Group-level p-value from Stage 1 (minimum p-value). - stage1_corrected_p_value: BH-corrected Stage 1 p-value. - stage1_significant: Whether the group passed Stage 1.

Raises:

• UnpairedException –

  If no row has a non-NaN AP, positive pairs and a value in every sameby column, e.g. an empty table.

See Also

mean_average_precision : For standard BH FDR correction.

Source code in src/copairs/map/map.py

def mean_average_precision_hierarchical(
    ap_scores: pd.DataFrame,
    sameby: List[str],
    null_size: int,
    threshold: float,
    seed: int | None = 0,
    hierarchical_by: Optional[List[str]] = None,
    progress_bar: bool = True,
    max_workers: Optional[int] = None,
    cache_dir: Optional[Union[str, Path]] = None,
    pos_pairs: Optional[np.ndarray] = None,
    null: str = "permutation",
    pairs: dict | None = None,
    *,
    max_reference_profiles: int = 16_384,
    pool_memory_budget: int = 4 * 1024**3,
    key_table_memory_budget: int = 256 * 1024**2,
) -> pd.DataFrame:
    """Calculate the Mean Average Precision (mAP) score with hierarchical FDR correction.

    This function computes the Mean Average Precision (mAP) score by grouping profiles
    based on the specified criteria (`sameby`). It applies hierarchical FDR correction
    appropriate for grouped hypothesis testing, such as dose-response data.

    Per-call resource controls (``max_reference_profiles``, ``pool_memory_budget``,
    ``key_table_memory_budget``), pair dictionaries and supported layouts follow
    :func:`get_map_pvalue`. With ``null_size=0`` all statistical columns, including
    stage-1 p-values, are NaN; significance flags contain pd.NA and correction is skipped.
    Supply ``hierarchical_by`` explicitly, preferably as a keyword.

    Parameters
    ----------
    ap_scores : pd.DataFrame
        DataFrame containing individual Average Precision (AP) scores and pair statistics
        (e.g., number of positive pairs `n_pos_pairs` and total pairs `n_total_pairs`).
    sameby : list or str
        Metadata column(s) used to group profiles for mAP calculation.
    null_size : int
        Number of samples in the null distribution for significance testing.
    threshold : float
        p-value threshold for identifying significant MaP scores.
    seed : int or None, default 0
        Reproducible seed; None requests fresh randomness once per run.
    hierarchical_by : list
        Metadata column(s) for hierarchical FDR correction. Enables two-stage testing:

        - Stage 1: Use minimum p-value within each group defined by `hierarchical_by`,
          then apply BH correction at the group level. A group passes if any member
          is significant.
        - Stage 2: For groups that pass Stage 1, apply BH correction to the
          individual tests within each group.

        This is designed for dose-response data where only high doses are expected
        to be active. The `hierarchical_by` columns must be a proper subset of `sameby`.
        For example, with `sameby=['compound', 'dose']` and `hierarchical_by=['compound']`,
        mAP is calculated per compound×dose, but FDR correction accounts for the
        grouped structure.
    progress_bar : bool
        Whether or not to show tqdm's progress bar.
    max_workers : int
        Threads that compute the copied null's p-values or run the
        permutation null's groups. Default defined by
        `concurrent.futures.ThreadPoolExecutor`.
    cache_dir : str or Path
        Location to save the cache of the copied null's AP samples.
    pos_pairs : np.ndarray, optional
        Positive pairs used to compute the AP scores, for ``null="joint"``;
        see `get_map_pvalue`. Needed there when a group's positives are not
        its other members.
    null : str
        Null of the group mAP: ``"permutation"`` (default), ``"joint"`` or
        ``"copied"``; see `get_map_pvalue`.
    pairs : dict, optional
        Pair dictionary of the AP scores, needed by ``null="permutation"`` and
        attached to AP frames; see `get_map_pvalue`.

    Returns
    -------
    pd.DataFrame
        DataFrame with the following columns:
        - `mean_average_precision`: Mean AP score for each group.
        - `mean_normalized_average_precision`: Mean normalized AP score (scale-independent).
        - `p_value`: p-value comparing mAP to the null distribution.
        - `corrected_p_value`: Adjusted p-value after multiple testing correction.
        - `below_p`: Boolean indicating if the p-value is below the threshold.
        - `below_corrected_p`: Boolean indicating if the corrected p-value is below the threshold.
        - `stage1_p_value`: Group-level p-value from Stage 1 (minimum p-value).
        - `stage1_corrected_p_value`: BH-corrected Stage 1 p-value.
        - `stage1_significant`: Whether the group passed Stage 1.

    Raises
    ------
    UnpairedException
        If no row has a non-NaN AP, positive pairs and a value in every
        `sameby` column, e.g. an empty table.

    See Also
    --------
    mean_average_precision : For standard BH FDR correction.

    """
    if hierarchical_by is None:
        raise ValueError("hierarchical_by is required for hierarchical correction")
    sameby = check_group_columns(sameby, "sameby", ap_scores)
    hierarchical_by = check_group_columns(hierarchical_by, "hierarchical_by")
    check_hierarchical_by(hierarchical_by, sameby)
    # Step 1: Compute mAP scores and p-values
    map_scores = get_map_pvalue(
        ap_scores=ap_scores,
        sameby=sameby,
        null_size=null_size,
        seed=seed,
        progress_bar=progress_bar,
        max_workers=max_workers,
        cache_dir=cache_dir,
        pos_pairs=pos_pairs,
        null=null,
        pairs=pairs,
        max_reference_profiles=max_reference_profiles,
        pool_memory_budget=pool_memory_budget,
        key_table_memory_budget=key_table_memory_budget,
    )

    if null_size == 0:
        for column in (
            "corrected_p_value",
            "stage1_p_value",
            "stage1_corrected_p_value",
        ):
            map_scores[column] = np.nan
        map_scores["stage1_significant"] = pd.array(
            [pd.NA] * len(map_scores), dtype="boolean"
        )
    else:
        map_scores = apply_hierarchical_fdr_correction(
            map_scores, hierarchical_by, sameby
        )
    return _mark_significance(map_scores, threshold)

silent_thread_map(fn, *iterables, **kwargs)

Map iterables and kwargs to a function.

Parameters:

• fn (callable) –

  Function to map over iterables.

• *iterables (tuple, default: () ) –

  Iterables to map over.

• **kwargs (dict, default: {} ) –

  Additional keyword arguments. Accepts: - max_workers : int, optional Maximum number of workers [default: min(32, cpu_count() + 4)]. - chunksize : int, optional Size of chunks for each worker [default: 1].

Source code in src/copairs/map/map.py

def silent_thread_map(fn, *iterables, **kwargs):
    """Map iterables and kwargs to a function.

    Parameters
    ----------
    fn : callable
        Function to map over iterables.
    *iterables : tuple
        Iterables to map over.
    **kwargs : dict
        Additional keyword arguments. Accepts:
        - max_workers : int, optional
            Maximum number of workers [default: min(32, cpu_count() + 4)].
        - chunksize : int, optional
            Size of chunks for each worker [default: 1].
    """
    # Based on tqdm's original implementation for consistency
    # (github.com/tqdm/tqdm/blob/0ed5d7f18fa3153834cbac0aa57e8092b217cc16/tqdm/contrib/concurrent.py#L29).

    kwargs = kwargs.copy()
    max_workers = kwargs.pop("max_workers", min(32, (cpu_count() or 1) + 4))
    chunksize = kwargs.pop("chunksize", 1)
    with ThreadPoolExecutor(max_workers=max_workers) as ex:
        return list(ex.map(fn, *iterables, chunksize=chunksize, **kwargs))

multilabel

Functions to compute mAP with multilabel support.

average_precision(meta, feats, pos_sameby, pos_diffby, neg_sameby, neg_diffby, multilabel_col, batch_size=20000, distance='cosine', progress_bar=True, return_pairs=False)

Compute average precision with multilabel support.

Returns normalized_average_precision in addition to average_precision. Each row is one (profile, label) pair: the profile's AP for that label.

The pair dictionary the APs were computed from, which null="permutation" in :func:copairs.map.mean_average_precision needs, are attached to the returned frame as frame.attrs["copairs_pairs"], as in :func:copairs.map.average_precision; return_pairs=True also returns them, as a (DataFrame, dict) tuple. Entries "pos" and "neg" are positive AP-row and negative source-profile pairs; "similarity" maps source-profile pairs to similarities. "profile" and "row_key" map AP rows to source profiles and labels in "keys"; "label_col" names the label column and selects multilabel permutation. "_lone" is a lazy function returning pairs sharing only one label (or None for unsupported negative rules). These entries keep feats, every positive and negative pair (8 bytes per pair) and each AP row's profile and label (12 bytes per row) alive with the frame, whichever null is used; drop them with frame.attrs.pop("copairs_pairs"). Preserve the original index, matching pairs and all holders of each tested label; validation checks structure, not source identity or label completeness. Filter whole labels or recompute AP. Copies share features and pair/mapping arrays; keep them unchanged between AP and mAP testing. Saved tables contain results only; later permutation testing needs the original pair dictionary or AP recomputation. The multilabel permutation null uses connected components of the negative graph, with fixed draws and member counts per component, keeping other labels fixed. Its exchangeability assumption is distinct from the single-label reference-block model.

See Also

copairs.map.average_precision : Average precision without multilabel support.

Source code in src/copairs/map/multilabel.py

def average_precision(
    meta: pd.DataFrame,
    feats: pd.DataFrame,
    pos_sameby: List[str],
    pos_diffby: List[str],
    neg_sameby: List[str],
    neg_diffby: List[str],
    multilabel_col,
    batch_size=20000,
    distance="cosine",
    progress_bar: bool = True,
    return_pairs: bool = False,
) -> pd.DataFrame:
    """
    Compute average precision with multilabel support.

    Returns normalized_average_precision in addition to average_precision.
    Each row is one (profile, label) pair: the profile's AP for that label.

    The pair dictionary the APs were computed from, which
    ``null="permutation"`` in :func:`copairs.map.mean_average_precision`
    needs, are attached to the returned frame as
    ``frame.attrs["copairs_pairs"]``, as in
    :func:`copairs.map.average_precision`; ``return_pairs=True`` also returns
    them, as a ``(DataFrame, dict)`` tuple. Entries ``"pos"`` and ``"neg"``
    are positive AP-row and negative source-profile pairs; ``"similarity"``
    maps source-profile pairs to similarities. ``"profile"`` and ``"row_key"``
    map AP rows to source profiles and labels in ``"keys"``; ``"label_col"``
    names the label column and selects multilabel permutation. ``"_lone"``
    is a lazy function returning pairs sharing only one label (or None for
    unsupported negative rules). These entries keep ``feats``, every
    positive and negative pair (8 bytes per pair) and each AP row's profile
    and label (12 bytes per row) alive with the frame, whichever null is
    used; drop them with ``frame.attrs.pop("copairs_pairs")``. Preserve the
    original index, matching pairs and all holders of each tested label;
    validation checks structure, not source identity or label completeness.
    Filter whole labels or recompute AP. Copies share features and pair/mapping
    arrays; keep them unchanged between AP and mAP testing. Saved tables contain
    results only; later permutation testing needs the original pair dictionary
    or AP recomputation. The multilabel permutation null uses
    connected components of the negative graph, with fixed draws and member
    counts per component, keeping other labels fixed. Its exchangeability
    assumption is distinct from the single-label reference-block model.

    See Also
    --------
    copairs.map.average_precision : Average precision without multilabel support.
    """
    columns = flatten_str_list(pos_sameby, pos_diffby, neg_sameby, neg_diffby)
    meta, columns = evaluate_and_filter(meta, columns)
    validate_pipeline_input(meta, feats, columns)
    distance_fn = compute.get_similarity_fn(distance, progress_bar=progress_bar)
    # Critical!, otherwise the indexing wont work
    meta = meta.reset_index(drop=True).copy()

    logger.info("Indexing metadata...")

    logger.info("Finding positive pairs...")
    pos_pairs, keys, pos_counts = find_pairs_multilabel(
        meta, sameby=pos_sameby, diffby=pos_diffby, multilabel_col=multilabel_col
    )
    if len(pos_pairs) == 0:
        raise UnpairedException("Unable to find positive pairs.")

    logger.info("Finding negative pairs...")
    neg_pairs = find_pairs_multilabel(
        meta, sameby=neg_sameby, diffby=neg_diffby, multilabel_col=multilabel_col
    )
    if len(neg_pairs) == 0:
        raise UnpairedException("Unable to find any negative pairs.")

    logger.info("Dropping dups in negative pairs...")
    neg_pairs = np.unique(neg_pairs, axis=0)

    logger.info("Computing positive similarities...")
    pos_sims = distance_fn(feats, pos_pairs, batch_size)

    logger.info("Computing negative similarities...")
    neg_sims = distance_fn(feats, neg_pairs, batch_size)

    logger.info("Computing AP per label...")
    negs_for = _create_neg_query_solver(neg_pairs, neg_sims, len(meta))
    ap_scores_list, null_confs_list, ix_list = _build_rank_lists_multi(
        pos_pairs, pos_sims, pos_counts, negs_for
    )

    logger.info("Creating result DataFrame...")
    results = []
    "Here the positive pairs are per-item inside multilabel_col"
    # TODO Check if multi-label key is necessary
    for i, key in enumerate(keys):
        # Compute normalized AP for this label group
        M = null_confs_list[i][:, 0]  # n_pos_pairs
        L = null_confs_list[i][:, 1]  # n_total_pairs
        N = L - M  # n_neg_pairs
        normalized_scores = normalize_ap(ap_scores_list[i], M, N)

        result = pd.DataFrame(
            {
                "average_precision": ap_scores_list[i],
                "normalized_average_precision": normalized_scores,
                "n_pos_pairs": null_confs_list[i][:, 0],
                "n_total_pairs": null_confs_list[i][:, 1],
                "ix": ix_list[i],
                multilabel_col: key,
            }
        )
        results.append(result)
    results = pd.concat(results).reset_index(drop=True)
    # The merged frame keeps these rows' positions as its index, so row i of
    # the AP frame is profile ix[i] for label keys[row_key[i]].
    profile = results["ix"].to_numpy().astype(np.int64)
    row_key = np.repeat(np.arange(len(keys), dtype=np.int32), list(map(len, ix_list)))
    pos_rows = _row_pairs(pos_pairs, pos_counts, profile, row_key)
    lone = None
    sameby, other = _columns(neg_sameby), _columns(neg_diffby)
    if multilabel_col in other and multilabel_col not in sameby:
        other = [c for c in other if c != multilabel_col]
        if set(_columns(pos_sameby)) == {multilabel_col, *sameby} and set(
            _columns(pos_diffby)
        ) == set(other):
            # The negative rules without the label are the positive rules.
            lone = partial(_lone_of_positives, pos_rows, profile, row_key)
        else:
            cols = list(dict.fromkeys([multilabel_col, *sameby, *other]))
            lone = partial(_lone_pairs, meta[cols], multilabel_col, sameby, other, keys)
    meta = meta.drop(multilabel_col, axis=1)
    results = meta.merge(results, right_on="ix", left_index=True).drop("ix", axis=1)
    results["n_pos_pairs"] = results["n_pos_pairs"].fillna(0).astype(np.uint32)
    results["n_total_pairs"] = results["n_total_pairs"].fillna(0).astype(np.uint32)
    pairs = {
        "pos": pos_rows,
        "neg": np.asarray(neg_pairs),
        "similarity": _FeatureSimilarity(feats, distance, batch_size),
        "profile": profile,
        "row_key": row_key,
        "label_col": multilabel_col,
        "keys": np.asarray(keys),
        "_lone": lone,
    }
    attach_pairs(results, pairs)
    logger.info("Finished.")
    return (results, pairs) if return_pairs else results

normalization

Functions for normalizing Average Precision scores.

compute_normalized_ap_scores(ap_scores, null_confs)

Compute both raw and normalized Average Precision scores.

Parameters:

• ap_scores (ndarray) –

  Array of raw Average Precision scores.

• null_confs (ndarray) –

  Array of configurations where each row is [n_pos_pairs, n_total_pairs].

Returns:

• ap_scores ( ndarray ) –

  The original raw AP scores.

• normalized_ap_scores ( ndarray ) –

  The normalized AP scores.

Source code in src/copairs/map/normalization.py

def compute_normalized_ap_scores(
    ap_scores: np.ndarray, null_confs: np.ndarray
) -> Tuple[np.ndarray, np.ndarray]:
    """Compute both raw and normalized Average Precision scores.

    Parameters
    ----------
    ap_scores : np.ndarray
        Array of raw Average Precision scores.
    null_confs : np.ndarray
        Array of configurations where each row is [n_pos_pairs, n_total_pairs].

    Returns
    -------
    ap_scores : np.ndarray
        The original raw AP scores.
    normalized_ap_scores : np.ndarray
        The normalized AP scores.
    """
    # Extract M (positive pairs) and compute N (negative pairs)
    M = null_confs[:, 0].astype(int)
    L = null_confs[:, 1].astype(int)
    N = L - M

    # Compute normalized scores
    normalized_ap_scores = normalize_ap(ap_scores, M, N)

    return ap_scores, normalized_ap_scores

expected_ap(M, N)

Compute the expected Average Precision under random ranking.

This implements the exact finite-sample formula for expected AP when items are randomly ranked.

Parameters:

• M (int) –

  Number of positive items (relevant documents).

• N (int) –

  Number of negative items (irrelevant documents).

Returns:

• float –

  The expected Average Precision under random ranking.

Notes

Formula: E[AP] = (1/L) × [(M-1)/(L-1) × (L - H_L) + H_L] where L = M + N and H_L is the L-th harmonic number.

Source code in src/copairs/map/normalization.py

def expected_ap(M: int, N: int) -> float:
    """Compute the expected Average Precision under random ranking.

    This implements the exact finite-sample formula for expected AP when
    items are randomly ranked.

    Parameters
    ----------
    M : int
        Number of positive items (relevant documents).
    N : int
        Number of negative items (irrelevant documents).

    Returns
    -------
    float
        The expected Average Precision under random ranking.

    Notes
    -----
    Formula: E[AP] = (1/L) × [(M-1)/(L-1) × (L - H_L) + H_L]
    where L = M + N and H_L is the L-th harmonic number.
    """
    L = M + N

    # Handle edge cases
    if L < 1 or M < 0 or N < 0:
        raise ValueError(f"Invalid inputs: M={M}, N={N}")
    if L == 1:
        return 1.0 if M == 1 else 0.0
    if M == 0:
        return 0.0
    if M == L:  # All items are positive
        return 1.0

    # Compute the L-th harmonic number
    H_L = harmonic_number(L)

    # Apply the exact formula
    mu0 = (1.0 / L) * (((M - 1.0) / (L - 1.0)) * (L - H_L) + H_L)

    return mu0

harmonic_number(n)

Compute the n-th harmonic number H_n = Σ(1/k) for k=1 to n.

Parameters:

• n (int) –

  The index of the harmonic number to compute.

Returns:

• float –

  The n-th harmonic number.

Source code in src/copairs/map/normalization.py

def harmonic_number(n: int) -> float:
    """Compute the n-th harmonic number H_n = Σ(1/k) for k=1 to n.

    Parameters
    ----------
    n : int
        The index of the harmonic number to compute.

    Returns
    -------
    float
        The n-th harmonic number.
    """
    if n <= 0:
        return 0.0
    return sum(1.0 / k for k in range(1, n + 1))

normalize_ap(ap, M, N, eps=1e-10)

Normalize Average Precision scores to be scale-independent.

Computes the normalized AP as (AP - μ₀) / (1 - μ₀) where μ₀ is the expected AP under random ranking.

Parameters:

• ap (float or ndarray) –

  The Average Precision score(s) to normalize.

• M (int or ndarray) –

  Number of positive items for each AP score.

• N (int or ndarray) –

  Number of negative items for each AP score.

• eps (float, default: 1e-10 ) –

  Small epsilon to avoid division by zero when μ₀ ≈ 1.

Returns:

• float or ndarray –

  The normalized Average Precision score(s).

Notes

• Normalized AP = 0 when performance equals random chance
• Normalized AP = 1 when performance is perfect
• Negative values indicate worse-than-random performance

Source code in src/copairs/map/normalization.py

def normalize_ap(
    ap: Union[float, np.ndarray],
    M: Union[int, np.ndarray],
    N: Union[int, np.ndarray],
    eps: float = 1e-10,
) -> Union[float, np.ndarray]:
    """Normalize Average Precision scores to be scale-independent.

    Computes the normalized AP as (AP - μ₀) / (1 - μ₀) where μ₀ is the
    expected AP under random ranking.

    Parameters
    ----------
    ap : float or np.ndarray
        The Average Precision score(s) to normalize.
    M : int or np.ndarray
        Number of positive items for each AP score.
    N : int or np.ndarray
        Number of negative items for each AP score.
    eps : float
        Small epsilon to avoid division by zero when μ₀ ≈ 1.

    Returns
    -------
    float or np.ndarray
        The normalized Average Precision score(s).

    Notes
    -----
    - Normalized AP = 0 when performance equals random chance
    - Normalized AP = 1 when performance is perfect
    - Negative values indicate worse-than-random performance
    """
    # Handle scalar or array inputs
    is_scalar = np.isscalar(ap)

    ap = np.atleast_1d(ap)
    M = np.atleast_1d(M)
    N = np.atleast_1d(N)

    # Validate that all arrays have compatible lengths
    lengths = [
        len(ap),
        len(M) if len(M) > 1 else len(ap),
        len(N) if len(N) > 1 else len(ap),
    ]
    if len(set(lengths)) > 1:
        raise ValueError(
            f"Array lengths must match: ap={len(ap)}, M={len(M)}, N={len(N)}"
        )

    # Compute expected AP for each configuration
    mu0 = np.zeros_like(ap, dtype=float)
    for i in range(len(ap)):
        M_i = M[i] if len(M) > 1 else M[0]
        N_i = N[i] if len(N) > 1 else N[0]
        mu0[i] = expected_ap(int(M_i), int(N_i))

    # Normalize: (AP - μ₀) / (1 - μ₀)
    # Use eps to avoid division by zero when μ₀ ≈ 1
    denominator = np.maximum(1 - mu0, eps)
    normalized = (ap - mu0) / denominator

    # Clip to [-1, 1] range to handle numerical edge cases
    normalized = np.clip(normalized, -1.0, 1.0)

    return float(normalized[0]) if is_scalar else normalized