Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions backend/app/api/v1/endpoints/results.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ def _compute_dimensional_breakdown(
experiment_id: UUID,
dimension: str,
db,
base_alpha: float = 0.05,
base_alpha: float,
value_source: Optional[ValueSource] = None,
) -> DimensionalBreakdownResponse:
"""
Expand All @@ -210,6 +210,11 @@ def _compute_dimensional_breakdown(
response lists first, so per variant the rows add up to that metric's
headline. DimensionalAnalysisService computes the statistics.

``base_alpha`` is 1 minus the confidence level the results are computed
at, and has no default. Each segment is tested at ``base_alpha`` divided
by the number of segments (Bonferroni); the heterogeneity test across the
segments (``detect_hte``) at ``base_alpha`` itself.

``value_source`` is passed to ``segment_rows`` when given: the
balanced-groups route passes ``stored_strata``. ``/results`` passes
none, so its breakdown reads each user's value as ``segment_rows``
Expand Down Expand Up @@ -275,7 +280,7 @@ def _compute_dimensional_breakdown(
segment_results = dim_service.compute_segment_results(
segments=segments, base_alpha=base_alpha
)
has_hte = dim_service.detect_hte(segment_results)
has_hte = dim_service.detect_hte(segment_results, alpha=base_alpha)
adjusted_alpha = (
dim_service.get_adjusted_alpha(base_alpha, len(segments))
if segments
Expand All @@ -284,9 +289,10 @@ def _compute_dimensional_breakdown(

hte_warning = (
(
f"Heterogeneous treatment effects detected across '{dimension}' segments. "
"Results may differ significantly between groups — "
"investigate per-segment effects before shipping."
"The first treatment's difference from the control varies between "
f"'{dimension}' segments by more than chance usually produces. "
"Treat this as a lead to confirm with a follow-up experiment, "
"not as a result."
)
if has_hte
else None
Expand Down
9 changes: 8 additions & 1 deletion backend/app/core/stats_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,13 @@
#: None, so treatments all significantly worse than a control at 0 (for a
#: proportion, only when lower is better) are no longer "no significant
#: difference".
#: 1.12.0 (#1969): a breakdown's ``has_heterogeneous_effects`` is true only
#: when a test that the first treatment's difference from the control, in
#: percentage points, is the same in every tested segment rejects at 1 minus
#: the confidence level (``DimensionalAnalysisService.detect_hte``). It no
#: longer fires on a spread or a change of sign of the lifts, nor on a
#: chi-squared test of the converting users alone at a fixed 0.05; the
#: ``hte_warning`` text changes with it.
#:
#: The rule: an engine-changing PR bumps ``ENGINE_VERSION`` to the next minor
#: and appends it to ``RELEASED_ENGINE_VERSIONS`` ("frozen on main", in
Expand All @@ -77,7 +84,7 @@
#: path stamping ``engine_version`` returns, caches or persists: results and
#: its snapshots, Bayesian, CUPED, sequential, and the bandit's state, history
#: and status. A revert of an engine-changing PR bumps again.
ENGINE_VERSION: str = "1.11.0"
ENGINE_VERSION: str = "1.12.0"

#: Mask that keeps the derived seed inside the signed 64-bit range.
_SEED_MASK = (1 << 63) - 1
Expand Down
11 changes: 9 additions & 2 deletions backend/app/schemas/dimensional.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,15 @@ class DimensionalBreakdownResponse(BaseModel):
has_heterogeneous_effects: bool = Field(
...,
description=(
"True when a chi-squared test detects significant heterogeneity "
"in treatment effects across segments."
"True when a test that the first treatment's difference from the "
"control, in percentage points, is the same in every tested segment "
"rejects at 1 minus the confidence level the results were computed "
"at. The first treatment is the non-control variant whose name "
"sorts first. A segment is tested only when, in both the control "
"and that treatment, its users times the pooled conversion rate, "
"and times one minus that rate, are each at least 10; with fewer "
"than two tested segments it is false. An exploratory signal, not "
"a confirmed finding."
),
)
hte_warning: Optional[str] = Field(
Expand Down
184 changes: 101 additions & 83 deletions backend/app/services/dimensional_analysis_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,52 @@

import logging
import math
import numbers
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Sequence, Tuple

import numpy as np
from scipy import stats

from backend.app.services import interaction_analysis

logger = logging.getLogger(__name__)

#: A segment enters the heterogeneity test only when, in both the control and
#: the tested treatment, its users times the pooled conversion rate, and its
#: users times one minus that rate, are each at least this
#: (``hte_tested_segments``). The seeded simulations that hold it are in
#: ``backend/tests/unit/stats_validation/test_golden_hte_false_positive_rate.py``.
HTE_MIN_EXPECTED_PER_CELL = 10


def hte_tested_segments(rows: Sequence[Sequence[int]]) -> List[int]:
"""The segments the heterogeneity test uses, as indexes into ``rows``.

Each row is one segment's ``(control users, control conversions,
treatment users, treatment conversions)``. The candidates are the
segments with at least one user in both arms, and the pooled rate is
their conversions over their users, both arms together. A candidate is
tested when, in both arms, its users ``n`` give ``n * rate`` and
``n * (1 - rate)`` of at least ``HTE_MIN_EXPECTED_PER_CELL``.

The comparison is made in integers, ``n * conversions >= floor * users``,
so a segment exactly at the floor is tested whatever floating point would
round ``n * rate`` to. A pooled rate of 0 or 1 tests nothing.
"""
counts = [tuple(int(value) for value in row) for row in rows]
candidates = [i for i, row in enumerate(counts) if row[0] > 0 and row[2] > 0]
users = sum(counts[i][0] + counts[i][2] for i in candidates)
conversions = sum(counts[i][1] + counts[i][3] for i in candidates)
floor = HTE_MIN_EXPECTED_PER_CELL * users
return [
i
for i in candidates
if all(
n * conversions >= floor and n * (users - conversions) >= floor
for n in (counts[i][0], counts[i][2])
)
]


# ---------------------------------------------------------------------------
# Data classes (returned by the service)
Expand Down Expand Up @@ -188,102 +226,82 @@ def compute_segment_results(

return results

def detect_hte(self, segment_results: List[SegmentResult]) -> bool:
def detect_hte(self, segment_results: List[SegmentResult], *, alpha: float) -> bool:
"""
Detect heterogeneous treatment effects (HTE) across segments.

Uses a chi-squared-like test on the treatment conversion rates across
segments. Returns True when at least two segments differ significantly
in their observed treatment effect.
Whether the treatment effect differs between segments (#1969).

Tests whether the first treatment's difference from the control, in
percentage points, is the same in every tested segment, and returns
True only when that is rejected at ``alpha``. The test is
``interaction_analysis.interaction_test`` (#219): the additive model
(one baseline per segment, one common difference) fitted by binomial
maximum likelihood, and Pearson's X^2 against it on J - 1 degrees of
freedom, J being the number of tested segments.

The control is the variant flagged ``is_control``, and the treatment
the first other variant of the first segment. The route lists the
control first and then the variants by name, so it is the
non-control variant whose name sorts first. Both are looked up by id
in every segment. A segment is tested when it passes
``hte_tested_segments``; ``other`` and ``unknown`` are segments like
any other. With fewer than two tested segments (no control, no
treatment, too few users or conversions) the answer is False, so
False means either "tested, alike" or "too little data to tell".

Args:
segment_results: Output of compute_segment_results.
alpha: The significance level, finite and strictly between 0 and
1: the experiment's ``1 - confidence_level``, never the
per-segment Bonferroni threshold. Required, so a caller
cannot silently test at a default.

Returns:
True when HTE is detected, False otherwise.
A Python ``bool``.

Raises:
ValueError: ``alpha`` is not a finite number between 0 and 1.
interaction_analysis.InteractionFitError: the fit failed. Never
caught here: the breakdown fails as a whole (a 500), never
with a partial or guessed answer.
"""
if len(segment_results) < 2:
if not (
isinstance(alpha, numbers.Real)
and math.isfinite(alpha)
and 0.0 < alpha < 1.0
):
raise ValueError(
f"alpha must be a finite number between 0 and 1, got {alpha!r}"
)
if not segment_results:
return False

first = segment_results[0].variants
control_id = next((v.variant_id for v in first if v.is_control), None)
treatment_id = next((v.variant_id for v in first if not v.is_control), None)
if control_id is None or treatment_id is None:
return False

# Gather (control_total, control_conv, treatment_total, treatment_conv)
# for each segment — use the first non-control variant as "treatment".
segment_data: List[Tuple[int, int, int, int]] = []
rows: List[Tuple[int, int, int, int]] = []
for seg in segment_results:
ctrl = next((v for v in seg.variants if v.is_control), None)
treat = next((v for v in seg.variants if not v.is_control), None)
if ctrl is None or treat is None:
continue
segment_data.append(
by_id = {v.variant_id: v for v in seg.variants}
control = by_id.get(control_id)
treatment = by_id.get(treatment_id)
rows.append(
(
ctrl.sample_size,
ctrl.conversions or 0,
treat.sample_size,
treat.conversions or 0,
control.sample_size if control else 0,
(control.conversions or 0) if control else 0,
treatment.sample_size if treatment else 0,
(treatment.conversions or 0) if treatment else 0,
)
)

if len(segment_data) < 2:
return False

# Build a 2×N contingency table (rows: control/treatment, cols: segments)
# and run chi-squared test. High chi-squared = heterogeneous effects.
ctrl_conv = np.array([d[1] for d in segment_data], dtype=float)
ctrl_total = np.array([d[0] for d in segment_data], dtype=float)
treat_conv = np.array([d[3] for d in segment_data], dtype=float)
treat_total = np.array([d[2] for d in segment_data], dtype=float)

# A variant can have no users in a segment (every variant is listed),
# so divide only where the total is positive; the rate is 0 elsewhere.
ctrl_rate = np.divide(
ctrl_conv,
ctrl_total,
out=np.zeros_like(ctrl_conv, dtype=float),
where=ctrl_total > 0,
)
treat_rate = np.divide(
treat_conv,
treat_total,
out=np.zeros_like(treat_conv, dtype=float),
where=treat_total > 0,
)
lift = treat_rate - ctrl_rate

# Chi-squared test on treatment conversion counts across segments
# If variances in conversion rates are large relative to their means, HTE detected.
# Create a 2xN contingency table for chi-squared
contingency = np.array(
[
[int(c) for c in treat_conv],
[int(c) for c in ctrl_conv],
]
)

# Only test if cells are non-trivially small
total_treat = treat_total.sum()
total_ctrl = ctrl_total.sum()
if total_treat == 0 or total_ctrl == 0:
tested = hte_tested_segments(rows)
if len(tested) < 2:
return False

try:
chi2_stat, p_value, dof, _ = stats.chi2_contingency(contingency)
# Use 0.05 as the HTE detection threshold
if p_value < 0.05 and dof > 0:
return True
except Exception as exc:
logger.debug("HTE chi-squared test failed: %s", exc)

# Secondary heuristic: compare lift variance to mean lift
if len(lift) >= 2:
lift_std = float(np.std(lift, ddof=1))
lift_mean = float(np.abs(np.mean(lift)))
# High coefficient of variation in lift across segments → HTE
if lift_mean > 0 and (lift_std / lift_mean) > 0.5:
return True
# Segments with opposing signs (positive and negative lift) → HTE
if any(l > 0 for l in lift) and any(l < 0 for l in lift):
return True

return False
n = [[rows[i][0] for i in tested], [rows[i][2] for i in tested]]
x = [[rows[i][1] for i in tested], [rows[i][3] for i in tested]]
result = interaction_analysis.interaction_test(n, x)
return bool(result.p_value < alpha)

# ------------------------------------------------------------------
# Private helpers
Expand Down
33 changes: 30 additions & 3 deletions backend/app/services/interaction_analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,20 @@
users and at least ``MIN_EXPECTED_PER_CELL`` expected converters and
non-converters at the table's pooled rate (``floor_reason``). Below that
floor the chi-squared approximation is not calibrated on skewed allocations.

The segment breakdown's heterogeneity flag (#1969,
``dimensional_analysis_service.detect_hte``) runs ``interaction_test`` on a
2 x J table of segments, with a lower floor of its own:
``HTE_MIN_EXPECTED_PER_CELL`` (10) expected converters and non-converters in
each arm of a segment, at the pooled rate, and at least one user per arm. A
breakdown's segments are far smaller than an experiment's arms; under this
module's floor the reported case would never be tested. Its seeded
simulations (``backend/tests/unit/stats_validation/
test_golden_hte_false_positive_rate.py``) pin the share flagged at alpha .05:
108 of 2,000 tables on the reported case (4 segments of 90 users per arm,
the same lift in each). The test is mildly liberal in one corner, many equal
small segments at a high base rate: 22 segments of 50 users per arm at 50%
flag 121 of 2,000, and 21 of 50 users at 30% -> 38% flag 139 of 2,000.
"""

from dataclasses import dataclass
Expand Down Expand Up @@ -56,8 +70,11 @@
class InteractionFitError(Exception):
"""The fit did not reach the maximum-likelihood estimate.

Never caught: it reaches the route's ``unexpected_failure`` (a 500). The
fit below cannot produce it on a table that passes the floor; it is an
Never caught: it reaches the calling route's ``unexpected_failure`` (a
500). The fit below cannot produce it on a table that passes its caller's
floor: ``floor_reason`` for #219's pair analysis, or
``dimensional_analysis_service.hte_tested_segments`` for the segment
breakdown's heterogeneity flag (#1969; see the module docstring). It is an
assertion, kept so that a defect is a 500 and never a wrong answer.
"""

Expand Down Expand Up @@ -222,7 +239,17 @@ def pearson_statistic_batch(
def interaction_test(n: Sequence, x: Sequence) -> InteractionTest:
"""Pearson's X^2 test of the additive model on one 2 x J table.

Call it only on a table ``floor_reason`` accepts.
Call it only on a table that passes its caller's floor. #219's pair
analysis passes tables ``floor_reason`` accepts (``MIN_USERS_PER_CELL``
users and ``MIN_EXPECTED_PER_CELL`` expected converters and
non-converters in every cell). The segment breakdown's heterogeneity flag
(#1969, ``dimensional_analysis_service.detect_hte``) passes the segments
``dimensional_analysis_service.hte_tested_segments`` keeps
(``HTE_MIN_EXPECTED_PER_CELL`` expected converters and non-converters in
each arm of a segment, and at least one user per arm). The module
docstring says why the breakdown's floor is lower, and the seeded
simulations behind it are in ``backend/tests/unit/stats_validation/
test_golden_hte_false_positive_rate.py``.

Raises:
InteractionFitError: the fit failed (a defect; never caught).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ def test_the_rows_are_the_stored_groups_and_add_up_to_every_user(
assigned = _assigned(db_session, experiment)
assert total == assigned, f"Σ {total} != {assigned:,}"
assert sum(c for row in rows.values() for _, c in row.values()) == 7
assert body["engine_version"] == ENGINE_VERSION == "1.11.0"
assert body["engine_version"] == ENGINE_VERSION == "1.12.0"
assert body["groups"]["dimension"] == "tier"
assert body["groups"]["is_exploratory"] is True
assert body["users_outside_blocks"] == 0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1827,9 +1827,9 @@ def spy(self, segments, *args, **kwargs):
original_hte = DimensionalAnalysisService.detect_hte
hte_rows: List[List[str]] = []

def hte_spy(self, segment_results):
def hte_spy(self, segment_results, **kwargs):
hte_rows.append([r.segment_value for r in segment_results])
return original_hte(self, segment_results)
return original_hte(self, segment_results, **kwargs)

monkeypatch.setattr(DimensionalAnalysisService, "detect_hte", hte_spy)
response = _get_results(viewer_client, exp)
Expand Down
7 changes: 4 additions & 3 deletions backend/tests/unit/api/test_dimensional_analysis_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,10 @@ def _make_breakdown_response(

hte_warning = (
(
f"Heterogeneous treatment effects detected across '{dimension}' segments. "
"Results may differ significantly between groups — "
"investigate per-segment effects before shipping."
"The first treatment's difference from the control varies between "
f"'{dimension}' segments by more than chance usually produces. "
"Treat this as a lead to confirm with a follow-up experiment, "
"not as a result."
)
if has_hte
else None
Expand Down
Loading
Loading