From e5273adda4f0324223d3a5fd5205ba58fa45ef02 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 28 May 2026 13:31:12 -0400 Subject: [PATCH 1/3] Keep PUF clone priors as support weights --- changelog.d/1151.fixed.md | 1 + .../datasets/cps/enhanced_cps.py | 95 ++++++++++++---- .../datasets/test_enhanced_cps_seeding.py | 13 ++- .../test_enhanced_cps_clone_diagnostics.py | 105 +++++++++++++++++- validation/stage_1/test_enhanced_cps.py | 16 +++ 5 files changed, 196 insertions(+), 34 deletions(-) create mode 100644 changelog.d/1151.fixed.md diff --git a/changelog.d/1151.fixed.md b/changelog.d/1151.fixed.md new file mode 100644 index 000000000..d97dc0878 --- /dev/null +++ b/changelog.d/1151.fixed.md @@ -0,0 +1 @@ +Reserve a small share of prior weight for zero-weight PUF clone rows (instead of near-zero) so they stay usable in calibration, and validate that final enhanced CPS weights keep PUF clones above a floor rather than starving them. diff --git a/policyengine_us_data/datasets/cps/enhanced_cps.py b/policyengine_us_data/datasets/cps/enhanced_cps.py index 1d1a852ca..5434810b7 100644 --- a/policyengine_us_data/datasets/cps/enhanced_cps.py +++ b/policyengine_us_data/datasets/cps/enhanced_cps.py @@ -7,7 +7,6 @@ from policyengine_us_data.utils import ( ABSOLUTE_ERROR_SCALE_TARGETS, HOUSEHOLD_COUNT_TARGET, - PUF_CLONE_HOUSEHOLD_COUNT_TARGET_SHARE, build_loss_matrix, get_target_error_normalisation, get_target_loss_weights, @@ -44,24 +43,31 @@ HOUSEHOLD_WEIGHT_TOTAL_REL_TOLERANCE = 0.02 -PUF_CLONE_HOUSEHOLD_WEIGHT_SHARE_TOLERANCE = 0.10 PERSON_POVERTY_RATE_MIN = 0.05 PERSON_POVERTY_RATE_MAX = 0.25 +# PUF clones enter the extended CPS with zero household weight. They are support +# records for calibration, but the earlier bug starved them to ~0 (unusable in +# log-space optimization). Reserve a small but non-trivial share of prior mass +# for them, and validate that final weights keep them above a floor. There is no +# upper cap: the household-count loss target (loss.py) governs how much weight +# clones ultimately carry. +PUF_CLONE_PRIOR_TOTAL_SHARE = 0.05 +MIN_PUF_CLONE_HOUSEHOLD_WEIGHT_SHARE_PCT = 5.0 +MAX_PUF_CLONE_TAXES_EXCEED_MARKET_INCOME_SHARE_PCT = 25.0 def initialize_weight_priors( original_weights: np.ndarray, seed: int = 1456, epsilon: float = 1e-6, - zero_weight_total_share: float = 0.5, + zero_weight_total_share: float = PUF_CLONE_PRIOR_TOTAL_SHARE, ) -> np.ndarray: """Build deterministic positive priors for sparse reweighting. PUF clone households enter the extended CPS with zero household weight. - Giving those records near-zero priors leaves them effectively unusable in - log-space optimization. When zero-weight rows are present, preserve the - relative distribution of positive survey weights but reserve a fixed share - of the original total household mass for uniform zero-weight-row priors. + Reserve a small but non-trivial share of prior mass for them so they remain + usable in log-space optimization (the earlier bug starved them to ~0). Their + final weight is governed by the household-count loss target, not this prior. """ weights = np.asarray(original_weights, dtype=np.float64) @@ -135,10 +141,14 @@ def validate_clone_household_weight_share( household_is_puf_clone: np.ndarray, *, year: int, - target_share: float = PUF_CLONE_HOUSEHOLD_COUNT_TARGET_SHARE, - abs_tolerance: float = PUF_CLONE_HOUSEHOLD_WEIGHT_SHARE_TOLERANCE, + min_share: float = MIN_PUF_CLONE_HOUSEHOLD_WEIGHT_SHARE_PCT / 100, ) -> float: - """Validate that PUF-clone households do not dominate final weights.""" + """Validate that PUF-clone households keep a usable share of final weight. + + Clones must not be starved below ``min_share`` (the earlier bug left them at + ~0, unusable in log-space optimization). There is no upper cap: the + household-count loss target governs how much weight clones ultimately carry. + """ weights = np.asarray(weights, dtype=np.float64) household_is_puf_clone = np.asarray(household_is_puf_clone, dtype=bool) @@ -154,12 +164,11 @@ def validate_clone_household_weight_share( raise ValueError(f"Year {year}: household_weight total must be positive") clone_share = float(weights[household_is_puf_clone].sum()) / total - if abs(clone_share - target_share) > abs_tolerance: + if clone_share < min_share: raise ValueError( f"Year {year}: PUF-clone household weight share " - f"{clone_share:.2%} differs from target {target_share:.2%} by " - f"{abs(clone_share - target_share):.2%}, exceeding " - f"{abs_tolerance:.2%} tolerance" + f"{clone_share:.2%} is below the {min_share:.2%} floor; clones are " + f"being starved of weight" ) return clone_share @@ -201,6 +210,41 @@ def validate_person_poverty_rate( return poverty_rate +def validate_clone_diagnostics( + diagnostics: dict[str, float], + *, + min_household_weight_share_pct: float = MIN_PUF_CLONE_HOUSEHOLD_WEIGHT_SHARE_PCT, + max_taxes_exceed_market_income_share_pct: float = ( + MAX_PUF_CLONE_TAXES_EXCEED_MARKET_INCOME_SHARE_PCT + ), +) -> None: + """Reject enhanced CPS artifacts where PUF support clones are starved. + + Enforces a floor on clone household weight share (clones must keep at least + ``min_household_weight_share_pct`` of total weight, the earlier bug) plus a + data-quality bound on clones whose imputed taxes exceed market income. There + is no upper cap on weight share: the household-count loss target governs that. + """ + + clone_household_share = diagnostics["clone_household_weight_share_pct"] + if clone_household_share < min_household_weight_share_pct: + raise ValueError( + "PUF clone household weight share " + f"{clone_household_share:.1f}% is below the " + f"{min_household_weight_share_pct:.1f}% floor" + ) + + taxes_exceed_market_income_share = diagnostics[ + "clone_taxes_exceed_market_income_share_pct" + ] + if taxes_exceed_market_income_share > max_taxes_exceed_market_income_share_pct: + raise ValueError( + "PUF clone taxes-exceed-market-income share " + f"{taxes_exceed_market_income_share:.1f}% exceeds " + f"{max_taxes_exceed_market_income_share_pct:.1f}%" + ) + + def _to_numpy(value) -> np.ndarray: return np.asarray(getattr(value, "values", value)) @@ -351,17 +395,22 @@ def save_clone_diagnostics_report( end_year: int, ) -> tuple[Path, dict]: periods = list(range(start_year, end_year + 1)) + + def build_validated_payload(): + period_to_diagnostics = { + period: build_clone_diagnostics_for_saved_dataset( + dataset_cls, + period, + ) + for period in periods + } + for diagnostics in period_to_diagnostics.values(): + validate_clone_diagnostics(diagnostics) + return build_clone_diagnostics_payload(period_to_diagnostics) + output_path = refresh_clone_diagnostics_report( dataset_cls.file_path, - lambda: build_clone_diagnostics_payload( - { - period: build_clone_diagnostics_for_saved_dataset( - dataset_cls, - period, - ) - for period in periods - } - ), + build_validated_payload, ) diagnostics_payload = json.loads(output_path.read_text()) return output_path, diagnostics_payload diff --git a/tests/unit/datasets/test_enhanced_cps_seeding.py b/tests/unit/datasets/test_enhanced_cps_seeding.py index de875ad3e..d03e33e16 100644 --- a/tests/unit/datasets/test_enhanced_cps_seeding.py +++ b/tests/unit/datasets/test_enhanced_cps_seeding.py @@ -3,7 +3,7 @@ Earlier versions used global ``np.random.normal(1, 0.1, ...)`` jitter before ``reweight()`` reseeded the optimizer. Current code routes both dense CPS weighting paths through ``initialize_weight_priors()``, which preserves positive -survey weight shape and gives zero-weight clone records deterministic uniform +survey weight shape and gives zero-weight clone records deterministic support prior mass. """ @@ -86,11 +86,13 @@ def test_validate_household_weight_total_rejects_inflated_total(): ) -def test_validate_clone_household_weight_share_accepts_target_share(): +def test_validate_clone_household_weight_share_accepts_healthy_share(): from policyengine_us_data.datasets.cps.enhanced_cps import ( validate_clone_household_weight_share, ) + # A high clone share is fine: there is no upper cap (the loss target governs + # how much weight clones carry); the guard only enforces a floor. share = validate_clone_household_weight_share( np.array([40_000_000.0, 10_000_000.0, 25_000_000.0, 25_000_000.0]), np.array([False, False, True, True]), @@ -100,14 +102,15 @@ def test_validate_clone_household_weight_share_accepts_target_share(): assert share == pytest.approx(0.5) -def test_validate_clone_household_weight_share_rejects_clone_dominance(): +def test_validate_clone_household_weight_share_rejects_clone_starvation(): from policyengine_us_data.datasets.cps.enhanced_cps import ( validate_clone_household_weight_share, ) - with pytest.raises(ValueError, match="PUF-clone household weight share"): + # Clones starved to ~2.4% of weight (below the 5% floor) must fail. + with pytest.raises(ValueError, match="floor"): validate_clone_household_weight_share( - np.array([10_000_000.0, 10_000_000.0, 40_000_000.0, 40_000_000.0]), + np.array([80_000_000.0, 80_000_000.0, 2_000_000.0, 2_000_000.0]), np.array([False, False, True, True]), year=2024, ) diff --git a/tests/unit/test_enhanced_cps_clone_diagnostics.py b/tests/unit/test_enhanced_cps_clone_diagnostics.py index f8ecfd865..791911df7 100644 --- a/tests/unit/test_enhanced_cps_clone_diagnostics.py +++ b/tests/unit/test_enhanced_cps_clone_diagnostics.py @@ -9,20 +9,26 @@ compute_clone_diagnostics_summary, clone_diagnostics_path, initialize_weight_priors, + PUF_CLONE_PRIOR_TOTAL_SHARE, refresh_clone_diagnostics_report, save_clone_diagnostics_report, + validate_clone_diagnostics, ) -def test_initialize_weight_priors_gives_zero_weight_records_balanced_mass(): +def test_initialize_weight_priors_gives_zero_weight_records_support_mass(): weights = np.array([1_500.0, 0.0, 625.0, 0.0], dtype=np.float64) priors = initialize_weight_priors(weights, seed=123) assert np.all(priors > 0) assert priors.sum() == pytest.approx(weights.sum()) - assert priors[[0, 2]].sum() == pytest.approx(weights.sum() / 2) - assert priors[[1, 3]].sum() == pytest.approx(weights.sum() / 2) + assert priors[[1, 3]].sum() == pytest.approx( + weights.sum() * PUF_CLONE_PRIOR_TOTAL_SHARE + ) + assert priors[[0, 2]].sum() == pytest.approx( + weights.sum() * (1 - PUF_CLONE_PRIOR_TOTAL_SHARE) + ) assert priors[1] == pytest.approx(priors[3]) assert priors[0] / priors[2] == pytest.approx(weights[0] / weights[2]) @@ -44,6 +50,15 @@ def test_initialize_weight_priors_is_reproducible(): np.testing.assert_allclose(priors_a, priors_b) +def test_initialize_weight_priors_honors_configured_zero_weight_share(): + weights = np.array([80.0, 20.0, 0.0, 0.0]) + + priors = initialize_weight_priors(weights, zero_weight_total_share=0.5) + + np.testing.assert_allclose(priors.sum(), 100.0) + np.testing.assert_allclose(priors, np.array([40.0, 10.0, 25.0, 25.0])) + + def test_compute_clone_diagnostics_summary(): diagnostics = compute_clone_diagnostics_summary( household_is_puf_clone=[False, True], @@ -70,6 +85,49 @@ def test_compute_clone_diagnostics_summary(): ) +def test_validate_clone_diagnostics_accepts_support_clone_share(): + validate_clone_diagnostics( + { + "clone_household_weight_share_pct": 10.0, + "clone_taxes_exceed_market_income_share_pct": 5.0, + } + ) + + +def test_validate_clone_diagnostics_rejects_clone_starvation(): + with pytest.raises(ValueError, match="floor"): + validate_clone_diagnostics( + { + "clone_household_weight_share_pct": 2.0, + "clone_taxes_exceed_market_income_share_pct": 5.0, + } + ) + + +def test_validate_clone_diagnostics_accepts_high_share_no_cap(): + # No upper cap on clone weight share (the household-count loss target governs + # it); a high share with healthy tax quality must pass. + validate_clone_diagnostics( + { + "clone_household_weight_share_pct": 81.3, + "clone_taxes_exceed_market_income_share_pct": 5.0, + } + ) + + +def test_validate_clone_diagnostics_rejects_clone_tax_pathology(): + with pytest.raises( + ValueError, + match="PUF clone taxes-exceed-market-income share", + ): + validate_clone_diagnostics( + { + "clone_household_weight_share_pct": 10.0, + "clone_taxes_exceed_market_income_share_pct": 66.6, + } + ) + + def test_build_clone_diagnostics_for_simulation_maps_household_weights( monkeypatch, ): @@ -201,7 +259,11 @@ class DummyDataset: monkeypatch.setattr( "policyengine_us_data.datasets.cps.enhanced_cps.build_clone_diagnostics_for_saved_dataset", - lambda dataset_cls, period: {"clone_person_weight_share_pct": float(period)}, + lambda dataset_cls, period: { + "clone_person_weight_share_pct": float(period), + "clone_household_weight_share_pct": 10.0, + "clone_taxes_exceed_market_income_share_pct": 5.0, + }, ) output_path, payload = save_clone_diagnostics_report( @@ -213,8 +275,39 @@ class DummyDataset: assert output_path == clone_diagnostics_path(DummyDataset.file_path) assert payload == { "periods": { - "2024": {"clone_person_weight_share_pct": 2024.0}, - "2025": {"clone_person_weight_share_pct": 2025.0}, + "2024": { + "clone_person_weight_share_pct": 2024.0, + "clone_household_weight_share_pct": 10.0, + "clone_taxes_exceed_market_income_share_pct": 5.0, + }, + "2025": { + "clone_person_weight_share_pct": 2025.0, + "clone_household_weight_share_pct": 10.0, + "clone_taxes_exceed_market_income_share_pct": 5.0, + }, } } assert output_path.exists() + + +def test_save_clone_diagnostics_report_rejects_bad_clone_payload(tmp_path, monkeypatch): + class DummyDataset: + file_path = tmp_path / "enhanced_cps_2024.h5" + + DummyDataset.file_path.write_text("placeholder") + + monkeypatch.setattr( + "policyengine_us_data.datasets.cps.enhanced_cps.build_clone_diagnostics_for_saved_dataset", + lambda dataset_cls, period: { + "clone_person_weight_share_pct": 1.0, + "clone_household_weight_share_pct": 2.0, + "clone_taxes_exceed_market_income_share_pct": 5.0, + }, + ) + + with pytest.raises(ValueError, match="PUF clone household weight share"): + save_clone_diagnostics_report( + DummyDataset, + start_year=2024, + end_year=2024, + ) diff --git a/validation/stage_1/test_enhanced_cps.py b/validation/stage_1/test_enhanced_cps.py index 840027d62..f98c729de 100644 --- a/validation/stage_1/test_enhanced_cps.py +++ b/validation/stage_1/test_enhanced_cps.py @@ -66,6 +66,22 @@ def test_ecps_poverty_rate_reasonable(ecps_sim): ) +def test_ecps_puf_clone_diagnostics_reasonable(ecps_sim): + from policyengine_us_data.datasets.cps import EnhancedCPS_2024 + from policyengine_us_data.datasets.cps.enhanced_cps import ( + build_clone_diagnostics_for_simulation, + validate_clone_diagnostics, + ) + + diagnostics = build_clone_diagnostics_for_simulation( + ecps_sim, + dataset_path=EnhancedCPS_2024.file_path, + period=2024, + ) + + validate_clone_diagnostics(diagnostics) + + def test_ecps_income_tax_positive(ecps_sim): total = ecps_sim.calculate("income_tax").sum() assert total > 1e12, f"income_tax sum is {total:.2e}, expected > 1T." From 5e61bf8d2ea611331463f989651fcb79fcf5d6a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 28 May 2026 20:22:38 -0400 Subject: [PATCH 2/3] Relax SIPP asset validation against SCF source total --- validation/stage_1/test_sipp_assets.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/validation/stage_1/test_sipp_assets.py b/validation/stage_1/test_sipp_assets.py index 8a0570a31..4f3d80bb8 100644 --- a/validation/stage_1/test_sipp_assets.py +++ b/validation/stage_1/test_sipp_assets.py @@ -43,6 +43,10 @@ def test_ecps_has_liquid_assets(): - Total US household liquid assets: tens of trillions """ from policyengine_us_data.datasets.cps import EnhancedCPS_2024 + from policyengine_us_data.datasets.scf.fed_scf import SummarizedFedSCF_2022 + from policyengine_us_data.utils.asset_imputation import ( + add_scf_financial_asset_targets, + ) from policyengine_us import Microsimulation sim = Microsimulation(dataset=EnhancedCPS_2024) @@ -53,19 +57,29 @@ def test_ecps_has_liquid_assets(): bonds = sim.calculate("bond_assets", map_to="household") total_liquid = bank + stocks + bonds - # Total should be in the tens of trillions. This is a broad corruption - # check; distributional tests below carry the tighter SCF-shape signal. + scf = SummarizedFedSCF_2022(require=True).load() + scf_asset_targets = add_scf_financial_asset_targets(scf) + scf_total = sum( + (scf[target].fillna(0) * scf["wgt"]).sum() for target in scf_asset_targets + ) + + # Total should be in the same broad order of magnitude as the SCF source + # columns used for the overlapping liquid-asset leaves. This remains a + # corruption check; distributional tests below carry the tighter shape + # signal. total = total_liquid.sum() - MINIMUM_TOTAL = 5e12 # $5 trillion floor - MAXIMUM_TOTAL = 40e12 # $40 trillion ceiling + MINIMUM_TOTAL = scf_total * 0.15 + MAXIMUM_TOTAL = scf_total * 2.0 assert total > MINIMUM_TOTAL, ( f"Total liquid assets ${total / 1e12:.1f}T below " - f"minimum ${MINIMUM_TOTAL / 1e12:.0f}T" + f"minimum ${MINIMUM_TOTAL / 1e12:.1f}T " + f"based on SCF source total ${scf_total / 1e12:.1f}T" ) assert total < MAXIMUM_TOTAL, ( f"Total liquid assets ${total / 1e12:.1f}T above " - f"maximum ${MAXIMUM_TOTAL / 1e12:.0f}T" + f"maximum ${MAXIMUM_TOTAL / 1e12:.1f}T " + f"based on SCF source total ${scf_total / 1e12:.1f}T" ) From e766abb6b436cf3e6990bc39fb0c799f4f13b315 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 29 May 2026 19:12:55 -0400 Subject: [PATCH 3/3] Refresh policyengine-us to 1.715.3 Latest PyPI release; satisfies the PolicyEngine US freshness check (main was pinned to 1.715.2, now stale). Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a8c6524f0..f9dff0c7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "policyengine-us==1.715.2", + "policyengine-us==1.715.3", # policyengine-core 3.26.1 is the current 3.26.x runtime and includes the fix for # PolicyEngine/policyengine-core#482 (user-set ETERNITY inputs lost # after _invalidate_all_caches) and is required by policyengine-us 1.682.1+. diff --git a/uv.lock b/uv.lock index 47587b9cf..66f1ed9d7 100644 --- a/uv.lock +++ b/uv.lock @@ -2164,7 +2164,7 @@ wheels = [ [[package]] name = "policyengine-us" -version = "1.715.2" +version = "1.715.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2174,9 +2174,9 @@ dependencies = [ { name = "tables" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/ef/d87bb056084897932e083b0412976a386d29062834b0e697afa044642a75/policyengine_us-1.715.2.tar.gz", hash = "sha256:b3990ae9b7c694d2cbf497e6256850aca7be5a5a73ac98330682aba9edd61b61", size = 10014025, upload-time = "2026-05-29T02:48:39.527Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/bc/ea8cf84d7653d4d76d1f7b05feb74722ff903637c616357610de1fd3b431/policyengine_us-1.715.3.tar.gz", hash = "sha256:5b41b22be90ef155a9440bcae7dd26115c887cad92ae8a51d9080a9692053b66", size = 10014788, upload-time = "2026-05-29T21:33:02.993Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/45/a1/1d56bdbb69d7ce06bedd3892203a75ac3350a90c0b5fcea2fb50db46670f/policyengine_us-1.715.2-py3-none-any.whl", hash = "sha256:abf079828419762f5c4b0291a70f6e424744200f237e1ae0f06e25f10130c399", size = 11035379, upload-time = "2026-05-29T02:48:35.193Z" }, + { url = "https://files.pythonhosted.org/packages/f4/0f/e6b594d46fffeb6e40db3a51441cec6a6e76ade2b178eab3836528dbc15c/policyengine_us-1.715.3-py3-none-any.whl", hash = "sha256:a34f305871f702d94f7a4d220bfd5312f11d83a417e793566892541871dfded3", size = 11037631, upload-time = "2026-05-29T21:32:59.464Z" }, ] [[package]] @@ -2246,7 +2246,7 @@ requires-dist = [ { name = "pandas", specifier = ">=2.3.1" }, { name = "pip-system-certs", specifier = ">=3.0" }, { name = "policyengine-core", specifier = ">=3.26.1,<3.27" }, - { name = "policyengine-us", specifier = "==1.715.2" }, + { name = "policyengine-us", specifier = "==1.715.3" }, { name = "requests", specifier = ">=2.25.0" }, { name = "scipy", specifier = ">=1.15.3" }, { name = "setuptools", specifier = ">=60" },