diff --git a/experiments/active/2026-05-09-recently-liked-blender-v2.md b/experiments/active/2026-05-09-recently-liked-blender-v2.md index 8d09058e..6eaef116 100644 --- a/experiments/active/2026-05-09-recently-liked-blender-v2.md +++ b/experiments/active/2026-05-09-recently-liked-blender-v2.md @@ -1,6 +1,6 @@ # Experiment: recently_liked Blender V2 Created: 2026-05-09 -Status: active - implementation delegated +Status: frozen - passive watch only Owner: ceo Measure after: after each variant reaches at least 1,000 assigned mature users @@ -15,6 +15,7 @@ The v1 failure is therefore treated as an experiment-design failure, not proof t - 2026-05-09 CEO decision from [FFM-1093](/FFM/issues/FFM-1093): open a redesigned v2 A/B instead of immediately re-enabling the v1 weights. - CTO implementation delegated in Paperclip to build LR-stratified assignment, high-volume-skipper handling, and explicit sample gates before the experiment can be read. - Analyst measurement delegated in Paperclip and blocked on deployment. +- 2026-07-23T18:05:54Z freeze applied in code via `RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN = True`: existing assignment rows remain readable, but new/unassigned mature users use control weights without creating new experiment assignments. Rollback path: set the freeze constant to `False` and redeploy. ## Metrics Before diff --git a/src/recommendations/blender_experiments.py b/src/recommendations/blender_experiments.py index 8d9f6842..15f3c563 100644 --- a/src/recommendations/blender_experiments.py +++ b/src/recommendations/blender_experiments.py @@ -21,6 +21,8 @@ RECENTLY_LIKED_BLENDER_V2_CONTROL = "control" RECENTLY_LIKED_BLENDER_V2_TREATMENT = "treatment" RECENTLY_LIKED_BLENDER_V2_EXCLUDED = "excluded_high_volume_skipper" +RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN = True +RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN_AT = "2026-07-23T18:05:54Z" RECENTLY_LIKED_BLENDER_V2_SAMPLE_GATE_PER_VARIANT = 1000 RECENTLY_LIKED_BLENDER_V2_MATURE_MIN_MEMES_SENT = 100 RECENTLY_LIKED_BLENDER_V2_SKIPPER_MIN_REACTIONS_7D = 50 @@ -335,6 +337,9 @@ async def get_or_assign_recently_liked_blender_v2_variant(user_id: int) -> str: if assignment is not None: return assignment["variant"] + if RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN: + return RECENTLY_LIKED_BLENDER_V2_CONTROL + metrics = await get_recent_7d_lr_assignment_metrics(user_id) if metrics is None: return RECENTLY_LIKED_BLENDER_V2_CONTROL diff --git a/tests/recommendations/test_blender_experiments.py b/tests/recommendations/test_blender_experiments.py index 206693c3..c32b8e75 100644 --- a/tests/recommendations/test_blender_experiments.py +++ b/tests/recommendations/test_blender_experiments.py @@ -6,6 +6,7 @@ MATURE_BLENDER_CONTROL_WEIGHTS, MATURE_BLENDER_TREATMENT_WEIGHTS, RECENTLY_LIKED_BLENDER_V2_CONTROL, + RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN_AT, RECENTLY_LIKED_BLENDER_V2_EXCLUDED, RECENTLY_LIKED_BLENDER_V2_EXPERIMENT_ID, RECENTLY_LIKED_BLENDER_V2_SAMPLE_GATE_PER_VARIANT, @@ -176,6 +177,37 @@ async def test_existing_recently_liked_blender_v2_assignment_wins(): get_assignment.assert_awaited_once_with(101, RECENTLY_LIKED_BLENDER_V2_EXPERIMENT_ID) +@pytest.mark.asyncio +async def test_frozen_recently_liked_blender_v2_does_not_assign_new_user(): + with ( + patch( + "src.recommendations.blender_experiments.get_experiment_assignment", + new_callable=AsyncMock, + return_value=None, + ) as get_assignment, + patch( + "src.recommendations.blender_experiments.get_recent_7d_lr_assignment_metrics", + new_callable=AsyncMock, + ) as get_metrics, + patch( + "src.recommendations.blender_experiments.assign_experiment", + new_callable=AsyncMock, + ) as assign, + patch( + "src.recommendations.blender_experiments.get_experiment_variant", + new_callable=AsyncMock, + ) as get_variant, + ): + variant = await get_or_assign_recently_liked_blender_v2_variant(202) + + assert variant == RECENTLY_LIKED_BLENDER_V2_CONTROL + assert RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN_AT == "2026-07-23T18:05:54Z" + get_assignment.assert_awaited_once_with(202, RECENTLY_LIKED_BLENDER_V2_EXPERIMENT_ID) + get_metrics.assert_not_awaited() + assign.assert_not_awaited() + get_variant.assert_not_awaited() + + @pytest.mark.asyncio async def test_recently_liked_blender_v2_race_rereads_winning_assignment(): with ( @@ -184,6 +216,10 @@ async def test_recently_liked_blender_v2_race_rereads_winning_assignment(): new_callable=AsyncMock, return_value=None, ), + patch( + "src.recommendations.blender_experiments.RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN", + False, + ), patch( "src.recommendations.blender_experiments.get_recent_7d_lr_assignment_metrics", new_callable=AsyncMock, @@ -218,6 +254,10 @@ async def test_recently_liked_blender_v2_defers_assignment_without_real_boundari new_callable=AsyncMock, return_value=None, ), + patch( + "src.recommendations.blender_experiments.RECENTLY_LIKED_BLENDER_V2_ENROLLMENT_FROZEN", + False, + ), patch( "src.recommendations.blender_experiments.get_recent_7d_lr_assignment_metrics", new_callable=AsyncMock,