Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion experiments/active/2026-05-09-recently-liked-blender-v2.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Experiment: recently_liked Blender V2
Created: 2026-05-09
Status: active - implementation delegated
Status: frozen - passive watch only
Owner: ceo
Measure after: after each variant reaches at least 1,000 assigned mature users

Expand All @@ -15,6 +15,7 @@ The v1 failure is therefore treated as an experiment-design failure, not proof t
- 2026-05-09 CEO decision from [FFM-1093](/FFM/issues/FFM-1093): open a redesigned v2 A/B instead of immediately re-enabling the v1 weights.
- CTO implementation delegated in Paperclip to build LR-stratified assignment, high-volume-skipper handling, and explicit sample gates before the experiment can be read.
- Analyst measurement delegated in Paperclip and blocked on deployment.
- 2026-07-23T18:05:54Z freeze applied in code via `RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN = True`: existing assignment rows remain readable, but new/unassigned mature users use control weights without creating new experiment assignments. Rollback path: set the freeze constant to `False` and redeploy.

## Metrics Before

Expand Down
5 changes: 5 additions & 0 deletions src/recommendations/blender_experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
RECENTLY_LIKED_BLENDER_V2_CONTROL = "control"
RECENTLY_LIKED_BLENDER_V2_TREATMENT = "treatment"
RECENTLY_LIKED_BLENDER_V2_EXCLUDED = "excluded_high_volume_skipper"
RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN = True
RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN_AT = "2026-07-23T18:05:54Z"
RECENTLY_LIKED_BLENDER_V2_SAMPLE_GATE_PER_VARIANT = 1000
RECENTLY_LIKED_BLENDER_V2_MATURE_MIN_MEMES_SENT = 100
RECENTLY_LIKED_BLENDER_V2_SKIPPER_MIN_REACTIONS_7D = 50
Expand Down Expand Up @@ -335,6 +337,9 @@ async def get_or_assign_recently_liked_blender_v2_variant(user_id: int) -> str:
if assignment is not None:
return assignment["variant"]

if RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN:
return RECENTLY_LIKED_BLENDER_V2_CONTROL

metrics = await get_recent_7d_lr_assignment_metrics(user_id)
if metrics is None:
return RECENTLY_LIKED_BLENDER_V2_CONTROL
Expand Down
40 changes: 40 additions & 0 deletions tests/recommendations/test_blender_experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
MATURE_BLENDER_CONTROL_WEIGHTS,
MATURE_BLENDER_TREATMENT_WEIGHTS,
RECENTLY_LIKED_BLENDER_V2_CONTROL,
RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN_AT,
RECENTLY_LIKED_BLENDER_V2_EXCLUDED,
RECENTLY_LIKED_BLENDER_V2_EXPERIMENT_ID,
RECENTLY_LIKED_BLENDER_V2_SAMPLE_GATE_PER_VARIANT,
Expand Down Expand Up @@ -176,6 +177,37 @@ async def test_existing_recently_liked_blender_v2_assignment_wins():
get_assignment.assert_awaited_once_with(101, RECENTLY_LIKED_BLENDER_V2_EXPERIMENT_ID)


@pytest.mark.asyncio
async def test_frozen_recently_liked_blender_v2_does_not_assign_new_user():
with (
patch(
"src.recommendations.blender_experiments.get_experiment_assignment",
new_callable=AsyncMock,
return_value=None,
) as get_assignment,
patch(
"src.recommendations.blender_experiments.get_recent_7d_lr_assignment_metrics",
new_callable=AsyncMock,
) as get_metrics,
patch(
"src.recommendations.blender_experiments.assign_experiment",
new_callable=AsyncMock,
) as assign,
patch(
"src.recommendations.blender_experiments.get_experiment_variant",
new_callable=AsyncMock,
) as get_variant,
):
variant = await get_or_assign_recently_liked_blender_v2_variant(202)

assert variant == RECENTLY_LIKED_BLENDER_V2_CONTROL
assert RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN_AT == "2026-07-23T18:05:54Z"
get_assignment.assert_awaited_once_with(202, RECENTLY_LIKED_BLENDER_V2_EXPERIMENT_ID)
get_metrics.assert_not_awaited()
assign.assert_not_awaited()
get_variant.assert_not_awaited()


@pytest.mark.asyncio
async def test_recently_liked_blender_v2_race_rereads_winning_assignment():
with (
Expand All @@ -184,6 +216,10 @@ async def test_recently_liked_blender_v2_race_rereads_winning_assignment():
new_callable=AsyncMock,
return_value=None,
),
patch(
"src.recommendations.blender_experiments.RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN",
False,
),
patch(
"src.recommendations.blender_experiments.get_recent_7d_lr_assignment_metrics",
new_callable=AsyncMock,
Expand Down Expand Up @@ -218,6 +254,10 @@ async def test_recently_liked_blender_v2_defers_assignment_without_real_boundari
new_callable=AsyncMock,
return_value=None,
),
patch(
"src.recommendations.blender_experiments.RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN",
False,
),
patch(
"src.recommendations.blender_experiments.get_recent_7d_lr_assignment_metrics",
new_callable=AsyncMock,
Expand Down
Loading