From 4cfb90b0405c824e14782de9dd5bbccfd66ef974 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 2 Apr 2025 19:13:46 +0200 Subject: [PATCH 01/15] object-wise leakage with aux quantities --- notebooks/cosmo_val/cat_config.yaml | 13 +++-- requirements.txt | 5 +- sp_validation/cosmo_val.py | 88 +++++++++++++++++++++-------- sp_validation/cosmology.py | 13 ----- sp_validation/utils_cosmo_val.py | 2 +- 5 files changed, 73 insertions(+), 48 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index c7903028..55c2a481 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -614,9 +614,10 @@ SP_v1.4.2: e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_iv + w_col: w_iv R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt + cols: "w_des w_iv snr mag fwhm_PSF" star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA @@ -682,7 +683,7 @@ SP_v1.4.1_noleakage: SP_v1.4.5: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: CFIS/v1.0/ShapePipe/v1.4.x/v1.4.5 ls: dashed colour: blue marker: d @@ -693,13 +694,13 @@ SP_v1.4.5: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: v1.4.x/v1.4.5/unions_shapepipe_cut_struc_2024_v1.4.5.fits + path: unions_shapepipe_cut_struc_2024_v1.4.5.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w_col: w_iv + w_col: w_des R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: @@ -742,7 +743,7 @@ SP_v1.4.5_glass_mock: e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w + w_col: w R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: @@ -785,7 +786,7 @@ SP_test: e2_col: e2 e1_PSF_col: e1_PSF e2_PSF_col: e2_PSF - w: w_iv + w_col: w_iv R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt star: diff --git a/requirements.txt b/requirements.txt index b45ac66b..423a4653 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ importlib_metadata adjustText colorama emcee -cs_util==0.1.2 +cs_util==0.1.3 astropy>=5.0 healpy healsparse @@ -18,7 +18,7 @@ jupytext==1.15.1 #pymaster regions reproject -shear_psf_leakage==0.2.0 +shear_psf_leakage==0.2.1 scipy seaborn setuptools==65.5.1 @@ -27,7 +27,6 @@ statsmodels treecorr tqdm uncertainties -clmm pyccl opencv-python-headless #git+https://github.com/aguinot/cosmo-numba.git@main diff --git a/sp_validation/cosmo_val.py b/sp_validation/cosmo_val.py index bb4cebda..c25dda45 100644 --- a/sp_validation/cosmo_val.py +++ b/sp_validation/cosmo_val.py @@ -1,6 +1,7 @@ # %% import os from contextlib import contextmanager +from functools import partial import colorama import matplotlib.pyplot as plt @@ -160,7 +161,7 @@ def set_params_leakage_scale(self, ver): # Note: for SP these are calibrated shear estimates params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] params_in["e2_col"] = self.cc[ver]["shear"]["e2_col"] - params_in["w_col"] = self.cc[ver]["shear"]["w"] + params_in["w_col"] = self.cc[ver]["shear"]["w_col"] params_in["R11"] = None if ver != "DES" else self.cc[ver]["shear"]["R11"] params_in["R22"] = None if ver != "DES" else self.cc[ver]["shear"]["R22"] @@ -187,7 +188,7 @@ def set_params_leakage_object(self, ver): # Note: for SP these are calibrated shear estimates params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] params_in["e2_col"] = self.cc[ver]["shear"]["e2_col"] - params_in["w_col"] = self.cc[ver]["shear"]["w"] + params_in["w_col"] = self.cc[ver]["shear"]["w_col"] if ( "e1_PSF_col" in self.cc[ver]["shear"] @@ -201,6 +202,9 @@ def set_params_leakage_object(self, ver): + f" shear yaml entry for version {ver}" ) + if "cols" in self.cc[ver]["shear"]: + params_in["cols"] = self.cc[ver]["shear"]["cols"] + params_in["verbose"] = False return params_in @@ -222,20 +226,18 @@ def init_results(self, objectwise=False): results[ver].prepare_output() @contextmanager - def temporarily_load_data(results): + def temporarily_load_data(results_instance): try: - self.print_start(f"Loading catalog for {ver}...", end="") - results.read_data() + self.print_start(f"Loading catalog {results_instance._params['input_path_shear']} ...", end="") + results_instance.read_data() self.print_done(f"done") yield finally: - self.print_done(f"Freeing {ver} from memory") - del results.dat_shear - del results.dat_PSF + self.print_done(f"Freeing {results_instance._params['input_path_shear']} from memory") + del results_instance.dat_shear + del results_instance.dat_PSF - results[ver].temporarily_load_data = lambda: temporarily_load_data( - results[ver] - ) + results[ver].temporarily_load_data = partial(temporarily_load_data, results[ver]) return results @@ -546,7 +548,7 @@ def calculate_scale_dependent_leakage(self): output_path_ab = f"{output_base_path}_a_b.txt" output_path_aa = f"{output_base_path}_a_a.txt" - with self.results[ver].temporarily_load_data(): + with results.temporarily_load_data(): if os.path.exists(output_path_ab) and os.path.exists(output_path_aa): self.print_green( f"Skipping computation, reading {output_path_ab} and {output_path_aa} instead" @@ -829,6 +831,38 @@ def plot_objectwise_leakage(self): cs_plots.savefig(out_path) self.print_done(f"Object-wise leakage coefficients plot saved to {out_path}") + def calculate_objectwise_leakage_aux(self): + + self.print_start("Object-wise leakage auxiliary quantities:") + for ver in self.versions: + self.print_magenta(ver) + + results_obj = self.results_objectwise[ver] + results_obj.check_params() + results_obj.update_params() + results_obj.prepare_output() + + with self.results[ver].temporarily_load_data(): + results_obj._dat = self.results[ver].dat_shear + + #out_base = results_obj.get_out_base(mix, order) + #out_path = f"{out_base}.pkl" + #if os.path.exists(out_path): + #self.print_green( + #f"Skipping object-wise leakage, file {out_path} exists" + #) + #results_obj.par_best_fit = leakage.read_from_file(out_path) + if not "cols" in results_obj._params: + self.print_green("Skipping object-wise leakage (aux quantities), no input columns for regression found") + else: + self.print_cyan("Computing object-wise leakage regression with aux quantities:", results_obj._params["cols"]) + + # Run + results_obj.obs_leakage() + + def plot_objectwise_leakage_aux(self): + self.calculate_objectwise_leakage_aux() + def plot_ellipticity(self, nbins=200): out_path = os.path.abspath(f"{self.cc['paths']['output']}/ell_hist.png") if os.path.exists(out_path): @@ -843,7 +877,8 @@ def plot_ellipticity(self, nbins=200): R = self.cc[ver]["shear"]["R"] e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R - w = self.results[ver].dat_shear["w"] + w_key = self.cc[ver]["shear"]["w_col"] + w = self.results[ver].dat_shear[w_key] axs[0].hist( e1, @@ -897,26 +932,29 @@ def calculate_additive_bias(self): self._c2 = {} for ver in self.versions: self.print_magenta(ver) - R = self.cc[ver]["shear"]["R"] - self._c1[ver] = np.average( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R, - weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ) - self._c2[ver] = np.average( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R, - weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ) - self.print_done("Finished additive bias calculation.") + with self.results[ver].temporarily_load_data(): + R = self.cc[ver]["shear"]["R"] + self._c1[ver] = np.average( + self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R, + weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], + ) + self._c2[ver] = np.average( + self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R, + weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], + ) + self.print_done("Finished additive bias calculation.") @property def c1(self): if not hasattr(self, "_c1"): + print("MKDEBUG calc c1") self.calculate_additive_bias() return self._c1 @property def c2(self): if not hasattr(self, "_c2"): + print("MKDEBUG calc c2") self.calculate_additive_bias() return self._c2 @@ -956,7 +994,7 @@ def calculate_2pcf(self): dec=self.results[ver].dat_shear["Dec"], g1=g1, g2=g2, - w=self.results[ver].dat_shear["w"], + w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], ra_units=self.treecorr_config["ra_units"], dec_units=self.treecorr_config["dec_units"], npatch=self.npatch, @@ -1203,7 +1241,7 @@ def calculate_aperture_mass_dispersion( dec=self.results[ver].dat_shear["Dec"], g1=g1, g2=g2, - w=self.results[ver].dat_shear["w"], + w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], ra_units=self.treecorr_config["ra_units"], dec_units=self.treecorr_config["dec_units"], npatch=npatch, diff --git a/sp_validation/cosmology.py b/sp_validation/cosmology.py index 1260c96e..66986bc7 100644 --- a/sp_validation/cosmology.py +++ b/sp_validation/cosmology.py @@ -26,19 +26,6 @@ from sp_validation.survey import get_footprint -# For theoretical modelling of cluster lensing -try: - import clmm -except Exception: - print('Could not import clmm, continuing...') - -try: - # import clmm.modeling as cm - from clmm import Cosmology -except Exception: - print('Could not import clmm.Cosmology, continuing...') - - # For correlation function calculations import treecorr diff --git a/sp_validation/utils_cosmo_val.py b/sp_validation/utils_cosmo_val.py index ce323f65..5dac7aed 100644 --- a/sp_validation/utils_cosmo_val.py +++ b/sp_validation/utils_cosmo_val.py @@ -93,7 +93,7 @@ def get_params_rho_tau(cat, survey="other"): params["dec_units"] = "deg" - params["w_col"] = cat['shear']["w"] + params["w_col"] = cat['shear']["w_col"] params["e1_col"] = cat['shear']["e1_col"] params["e2_col"] = cat['shear']["e2_col"] params["R11"] = cat['shear'].get("R11") From ef66d677866ef7a98e1fcad7b33340fa47ec6d93 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 3 Apr 2025 16:33:18 +0200 Subject: [PATCH 02/15] leakage aux --- notebooks/cosmo_val/run_cosmo_val.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/notebooks/cosmo_val/run_cosmo_val.py b/notebooks/cosmo_val/run_cosmo_val.py index a8f2f5e9..78449c78 100644 --- a/notebooks/cosmo_val/run_cosmo_val.py +++ b/notebooks/cosmo_val/run_cosmo_val.py @@ -1,6 +1,10 @@ # %% # %load_ext autoreload # %autoreload 2 + +import matplotlib +matplotlib.use("agg") + import matplotlib.pyplot as plt import numpy as np import treecorr @@ -8,8 +12,8 @@ # %% cv = CosmologyValidation( - versions=["SP_v1.4.2"], - data_base_dir="/n17data/mkilbing/astro/data/", + versions=["SP_v1.4.2", "SP_v1.4.5"], + data_base_dir="/n17data/mkilbing/astro/data", npatch=100, ) @@ -30,7 +34,10 @@ cv.plot_scale_dependent_leakage() # %% -#cv.plot_objectwise_leakage() +cv.plot_objectwise_leakage() + +# %% +cv.plot_objectwise_leakage_aux() # %% cv.plot_ellipticity() @@ -38,6 +45,9 @@ # %% cv.plot_separation() +# %% +cv.calculate_additive_bias() + # %% cv.plot_2pcf() From 3f2a3051fe6193083e8171de221986274ab68496 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sun, 6 Apr 2025 13:19:46 +0200 Subject: [PATCH 03/15] cosmo_val print bug fix --- .gitignore | 1 + sp_validation/cosmo_val.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 77cd138b..6630d111 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,4 @@ notebooks/plot_footprints.ipynb notebooks/demo_comprehensive_to_minimal_cat.ipynb notebooks/demo_create_footprint_mask.ipynb notebooks/demo_check_footprint.ipynb +notebooks/demo_comprehensive_to_minimal_cat.ipynb diff --git a/sp_validation/cosmo_val.py b/sp_validation/cosmo_val.py index c25dda45..81161024 100644 --- a/sp_validation/cosmo_val.py +++ b/sp_validation/cosmo_val.py @@ -855,7 +855,7 @@ def calculate_objectwise_leakage_aux(self): if not "cols" in results_obj._params: self.print_green("Skipping object-wise leakage (aux quantities), no input columns for regression found") else: - self.print_cyan("Computing object-wise leakage regression with aux quantities:", results_obj._params["cols"]) + self.print_cyan(f"Computing object-wise leakage regression with aux quantities: {results_obj._params['cols']}") # Run results_obj.obs_leakage() From d7e06f78a70c6f6b253b3a032ab8a38729f00503 Mon Sep 17 00:00:00 2001 From: Martin Kilbinger Date: Mon, 7 Apr 2025 17:48:54 +0200 Subject: [PATCH 04/15] removed commented code --- sp_validation/cosmo_val.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sp_validation/cosmo_val.py b/sp_validation/cosmo_val.py index 81161024..673b41cd 100644 --- a/sp_validation/cosmo_val.py +++ b/sp_validation/cosmo_val.py @@ -845,13 +845,6 @@ def calculate_objectwise_leakage_aux(self): with self.results[ver].temporarily_load_data(): results_obj._dat = self.results[ver].dat_shear - #out_base = results_obj.get_out_base(mix, order) - #out_path = f"{out_base}.pkl" - #if os.path.exists(out_path): - #self.print_green( - #f"Skipping object-wise leakage, file {out_path} exists" - #) - #results_obj.par_best_fit = leakage.read_from_file(out_path) if not "cols" in results_obj._params: self.print_green("Skipping object-wise leakage (aux quantities), no input columns for regression found") else: From f97db747d69191af4a244dd9822ad0d830feed45 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 9 Apr 2025 08:55:41 +0200 Subject: [PATCH 05/15] added columns for obj_leakage --- notebooks/cosmo_val/cat_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 55c2a481..3d917172 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -617,7 +617,7 @@ SP_v1.4.2: w_col: w_iv R: 1.0 redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt - cols: "w_des w_iv snr mag fwhm_PSF" + cols: "w_des w_iv snr mag fwhm_PSF RA Dec FLUX_RADIUS FLUX_AUTO" star: path: unions_shapepipe_star_2024_v1.4.a.fits ra_col: RA From bfdbd73b1995d3af081071d6cd3148833f34b9d4 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 18 Apr 2025 15:14:10 +0200 Subject: [PATCH 06/15] major version as variable --- notebooks/demo_apply_hsp_masks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/notebooks/demo_apply_hsp_masks.py b/notebooks/demo_apply_hsp_masks.py index f1f1c69f..6f20c91f 100644 --- a/notebooks/demo_apply_hsp_masks.py +++ b/notebooks/demo_apply_hsp_masks.py @@ -52,14 +52,15 @@ # + # Set parameters -obj._params["input_path"] = "unions_shapepipe_comprehensive_2024_v1.4.c.hdf5" -obj._params["output_path"] = "unions_shapepipe_comprehensive_struc_2024_v1.4.c.hdf5" -obj._params["mask_dir"] = f"{os.environ['HOME']}/v1.4.x/masks" +ver_maj = "v1.5" +obj._params["input_path"] = f"unions_shapepipe_comprehensive_2024_{ver_maj}.c.hdf5" +obj._params["output_path"] = f"unions_shapepipe_comprehensive_struc_2024_{ver_maj}.c.hdf5" +obj._params["mask_dir"] = f"{os.environ['HOME']}/{ver_maj}.x/masks" obj._params["nside"] = 131072 obj._params["file_base"] = "mask_r_" obj._params["bits"] = bits -obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage.hsp" +obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage_{ver_maj}.x.hsp" obj._params["aux_mask_labels"] = "npoint3" obj._params["verbose"] = True # - From 2eb70b18eb7c3837e59923bcda2fcc39ecf85f00 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 18 Apr 2025 15:14:54 +0200 Subject: [PATCH 07/15] cosmo val config: minor version 'a' for psf and star cats --- notebooks/cosmo_val/cat_config.yaml | 51 ++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 3d917172..587558b8 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -853,9 +853,52 @@ SP_v1.4.5_leak_corr: star_flag: "FLAG_STAR_HSM" square_size: True +SP_v1.5.3: + pipeline: SP + subdir: CFIS/v1.0/ShapePipe/v1.5.x/v1.5.3 + ls: dashed + colour: blue + marker: v + getdist_colour: 0.0, 0.5, 1.0 + cov_th: + A: 2420.2651014497287 #deg^2 + n_e: 7.040818382014773 #arcmin-2 + n_psf: 0.752316232272063 #arcmin-2 + sigma_e: 0.30961528707207325 + shear: + path: unions_shapepipe_cut_struc_2024_v1.5.3.fits + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + e1_col: e1 + e2_col: e2 + e1_PSF_col: e1_PSF + e2_PSF_col: e2_PSF + w_col: w_iv + R: 1.0 + star: + path: unions_shapepipe_star_2024_v1.5.a.fits + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + psf: + path: unions_shapepipe_psf_2024_v1.5.a.fits + hdu: 1 + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e2_PSF_col: E2_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_star_col: E2_STAR_HSM + PSF_size: SIGMA_PSF_HSM + star_size: SIGMA_STAR_HSM + PSF_flag: "FLAG_PSF_HSM" + star_flag: "FLAG_STAR_HSM" + square_size: True + + SP_v1.5.4: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/ + subdir: CFIS/v1.0/ShapePipe/v1.5.x/v1.5.4 ls: dashed colour: green marker: d @@ -866,7 +909,7 @@ SP_v1.5.4: n_psf: 0.752316232272063 #arcmin-2 sigma_e: 0.30961528707207325 shear: - path: v1.5.x/v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits + path: unions_shapepipe_cut_struc_2024_v1.5.4.fits covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt e1_col: e1 e2_col: e2 @@ -875,13 +918,13 @@ SP_v1.5.4: w_col: w_iv R: 1.0 star: - path: unions_shapepipe_star_2024_v1.4.1.fits + path: unions_shapepipe_star_2024_v1.5.a.fits ra_col: RA dec_col: Dec e1_col: e1 e2_col: e2 psf: - path: v1.5.x/unions_shapepipe_psf_2024_v1.5.2.fits + path: unions_shapepipe_psf_2024_v1.5.a.fits hdu: 1 ra_col: RA dec_col: Dec From a428ad796fe047788a8797f3320743a651ceb39c Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 9 May 2025 15:05:30 +0200 Subject: [PATCH 08/15] sp_val cfg paths --- notebooks/cosmo_val/cat_config.yaml | 4 ++-- notebooks/demo_comprehensive_to_minimal_cat.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index c9048c26..c6fc27bd 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -683,7 +683,7 @@ SP_v1.4.1_noleakage: SP_v1.4.5: pipeline: SP - subdir: CFIS/v1.0/ShapePipe/v1.4.x/v1.4.5 + subdir: v1.4.x/v1.4.5 ls: dashed colour: blue marker: d @@ -812,7 +812,7 @@ SP_test: SP_v1.4.5_leak_corr: pipeline: SP - subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5 + subdir: v1.4.x/v1.4.5 ls: dashed colour: midnightblue marker: d diff --git a/notebooks/demo_comprehensive_to_minimal_cat.py b/notebooks/demo_comprehensive_to_minimal_cat.py index 65a4bb1c..f68a43c9 100644 --- a/notebooks/demo_comprehensive_to_minimal_cat.py +++ b/notebooks/demo_comprehensive_to_minimal_cat.py @@ -34,7 +34,7 @@ # Initialize calibration class instance obj = sp_joint.CalibrateCat() -config = obj.read_config_set_params("config_mask.yaml") +config = obj.read_config_set_params("config_mask.P37.yaml") # !pwd @@ -185,7 +185,7 @@ # Write extended data to new HDF5 file obj_appl = sp_joint.ApplyHspMasks() -output_path = obj._params["input_path"].replace("1.X.c", "1.X.m") +output_path = obj._params["input_path"].replace("1.5.c", "1.5.m") obj_appl._params["output_path"] = output_path obj_appl._params["aux_mask_file_list"] = [] From 8b78f89c5341ab1c4d2f2d67b6b022d280c7c634 Mon Sep 17 00:00:00 2001 From: "martin.kilbinger@cea.fr" Date: Tue, 20 Jan 2026 14:17:45 +0100 Subject: [PATCH 09/15] remove conflict marker --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index b38fc2c8..e341250c 100644 --- a/.gitignore +++ b/.gitignore @@ -185,4 +185,3 @@ cosmo_inference/cosmosis_config/*.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.ini !cosmo_inference/cosmosis_config/cosmosis_pipeline_SP_v1.4.6_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1_cell.ini !cosmo_inference/cosmosis_config/priors_mock.ini ->>>>>>> upstream/develop From 64e61da0e67c288d9760300c7541a9a6041f24c8 Mon Sep 17 00:00:00 2001 From: "martin.kilbinger@cea.fr" Date: Tue, 20 Jan 2026 14:19:39 +0100 Subject: [PATCH 10/15] moved cosmo_val to src --- sp_validation/cosmo_val.py | 2245 ---------------------------- src/sp_validation/cosmo_val.py | 2558 ++++++++++---------------------- 2 files changed, 752 insertions(+), 4051 deletions(-) delete mode 100644 sp_validation/cosmo_val.py diff --git a/sp_validation/cosmo_val.py b/sp_validation/cosmo_val.py deleted file mode 100644 index 1a45dca8..00000000 --- a/sp_validation/cosmo_val.py +++ /dev/null @@ -1,2245 +0,0 @@ -# %% -import os -from contextlib import contextmanager -from functools import partial - -import colorama -import matplotlib.pyplot as plt -import numpy as np -import pymaster as nmt -import healpy as hp -import treecorr -from . import utils_cosmo_val -import camb -import re - -import yaml -from astropy.io import fits -from astropy import units as u -from astropy.coordinates import SkyCoord - -import healpy as hp -import healsparse as hsp -import treecorr -import camb -import yaml -from collections import Counter - -import skyproj - -from astropy.io import fits -from astropy.cosmology import Planck18 -from astropy import units as u -from astropy.coordinates import SkyCoord - -try: - from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes -except: - print("Could not import cosmo_numba, continuing without") - -from . import utils_cosmo_val -import pymaster as nmt -from cs_util import plots as cs_plots -from shear_psf_leakage import leakage -from shear_psf_leakage import plots as psfleak_plots -from shear_psf_leakage import run_object, run_scale -from shear_psf_leakage.rho_tau_stat import PSFErrorFit -from uncertainties import ufloat -import matplotlib.scale as mscale - -mscale.register_scale(utils_cosmo_val.SquareRootScale) - - -# %% -class CosmologyValidation: - - def __init__( - self, - versions, - data_base_dir, - catalog_config="./cat_config.yaml", - rho_tau_method="lsq", - cov_estimate_method="th", - compute_cov_rho=True, - n_cov=100, - theta_min=0.1, - theta_max=250, - nbins=20, - var_method="jackknife", - npatch=20, - quantile=0.1587, - theta_min_plot=0.08, - theta_max_plot=250, - ylim_alpha=[-0.005, 0.05], - ylim_xi_sys_ratio=[-0.02, 0.5], - nside=1024, - binning='powspace', - power=1/2, - n_ell_bins=32, - pol_factor=True, - nrandom_cell=10 - ): - - self.versions = versions - self.data_base_dir = data_base_dir - self.rho_tau_method = rho_tau_method - self.cov_estimate_method = cov_estimate_method - self.compute_cov_rho = compute_cov_rho - self.n_cov = n_cov - self.theta_min = theta_min - self.theta_max = theta_max - self.npatch = npatch - self.nbins = nbins - self.quantile = quantile - self.theta_min_plot = theta_min_plot - self.theta_max_plot = theta_max_plot - self.ylim_alpha = ylim_alpha - self.ylim_xi_sys_ratio = ylim_xi_sys_ratio - #For pseudo-Cls - self.nside = nside - self.binning = binning - self.power = power - self.n_ell_bins = n_ell_bins - self.pol_factor = pol_factor - self.nrandom_cell = nrandom_cell - - self.treecorr_config = { - "ra_units": "degrees", - "dec_units": "degrees", - "min_sep": theta_min, - "max_sep": theta_max, - "sep_units": "arcmin", - "nbins": nbins, - "var_method": var_method, - } - - with open(catalog_config, "r") as file: - self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) - - for ver in ["nz", *versions]: - - if ver not in cc: - raise KeyError(f"Version string {ver} not found in config file{catalog_config}") - version_base = f"{data_base_dir}/{cc[ver]['subdir']}" - for key in cc[ver]: - if "path" in cc[ver][key]: - cc[ver][key]["path"] = f"{version_base}/{cc[ver][key]['path']}" - - if not os.path.exists(cc["paths"]["output"]): - os.mkdir(cc["paths"]["output"]) - - def color_reset(self): - print(colorama.Fore.BLACK, end="") - - def print_blue(self, msg, end="\n"): - print(colorama.Fore.BLUE + msg, end=end) - self.color_reset() - - def print_start(self, msg, end="\n"): - print() - self.print_blue(msg, end=end) - - def print_done(self, msg): - self.print_blue(msg) - - def print_magenta(self, msg): - print(colorama.Fore.MAGENTA + msg) - self.color_reset() - - def print_green(self, msg): - print(colorama.Fore.GREEN + msg) - self.color_reset() - - def print_cyan(self, msg): - print(colorama.Fore.CYAN + msg) - self.color_reset() - - def set_params_leakage_scale(self, ver): - params_in = {} - - # Set parameters - params_in["input_path_shear"] = self.cc[ver]["shear"]["path"] - params_in["input_path_PSF"] = self.cc[ver]["star"]["path"] - params_in["dndz_path"] = ( - f"{self.cc['nz']['dndz']['path']}_{self.cc[ver]['pipeline']}_{self.cc['nz']['dndz']['blind']}.txt" - ) - params_in["output_dir"] = f'{self.cc["paths"]["output"]}/leakage_{ver}' - - # Note: for SP these are calibrated shear estimates - params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] - params_in["e2_col"] = self.cc[ver]["shear"]["e2_col"] - params_in["w_col"] = self.cc[ver]["shear"]["w_col"] - params_in["R11"] = None if ver != "DES" else self.cc[ver]["shear"]["R11"] - params_in["R22"] = None if ver != "DES" else self.cc[ver]["shear"]["R22"] - - params_in["ra_star_col"] = self.cc[ver]["star"]["ra_col"] - params_in["dec_star_col"] = self.cc[ver]["star"]["dec_col"] - params_in["e1_PSF_star_col"] = self.cc[ver]["star"]["e1_col"] - params_in["e2_PSF_star_col"] = self.cc[ver]["star"]["e2_col"] - - params_in["theta_min_amin"] = self.theta_min - params_in["theta_max_amin"] = self.theta_max - params_in["n_theta"] = self.nbins - - params_in["verbose"] = False - - return params_in - - def set_params_leakage_object(self, ver): - params_in = {} - - # Set parameters - params_in["input_path_shear"] = self.cc[ver]["shear"]["path"] - params_in["output_dir"] = f'{self.cc["paths"]["output"]}/leakage_{ver}' - - # Note: for SP these are calibrated shear estimates - params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] - params_in["e2_col"] = self.cc[ver]["shear"]["e2_col"] - params_in["w_col"] = self.cc[ver]["shear"]["w_col"] - - if ( - "e1_PSF_col" in self.cc[ver]["shear"] - and "e2_PSF_col" in self.cc[ver]["shear"] - ): - params_in["e1_PSF_col"] = self.cc[ver]["shear"]["e1_PSF_col"] - params_in["e2_PSF_col"] = self.cc[ver]["shear"]["e2_PSF_col"] - else: - raise KeyError( - "Keys 'e1_PSF_col' and 'e2_PSF_col' not found in" - + f" shear yaml entry for version {ver}" - ) - - if "cols" in self.cc[ver]["shear"]: - params_in["cols"] = self.cc[ver]["shear"]["cols"] - - params_in["verbose"] = False - - return params_in - - def init_results(self, objectwise=False): - - results = {} - for ver in self.versions: - - # Set parameters depending on the type of leakage - if objectwise: - results[ver] = run_object.LeakageObject() - results[ver]._params.update(self.set_params_leakage_object(ver)) - else: - results[ver] = run_scale.LeakageScale() - results[ver]._params.update(self.set_params_leakage_scale(ver)) - - results[ver].check_params() - results[ver].prepare_output() - - @contextmanager - def temporarily_load_data(results_instance): - try: - self.print_start(f"Loading catalog {results_instance._params['input_path_shear']} ...", end="") - results_instance.read_data() - self.print_done(f"done") - yield - finally: - self.print_done(f"Freeing {results_instance._params['input_path_shear']} from memory") - del results_instance.dat_shear - del results_instance.dat_PSF - - results[ver].temporarily_load_data = partial(temporarily_load_data, results[ver]) - - return results - - @property - def results(self): - if not hasattr(self, "_results"): - self._results = self.init_results(objectwise=False) - return self._results - - @property - def results_objectwise(self): - if not hasattr(self, "_results_objectwise"): - self._results_objectwise = self.init_results(objectwise=True) - return self._results_objectwise - - def calculate_rho_tau_stats(self): - - out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" - if not os.path.exists(out_dir): - os.mkdir(out_dir) - - self.print_start("Rho stats") - for ver in self.versions: - rho_stat_handler, tau_stat_handler = utils_cosmo_val.get_rho_tau_w_cov( - self.cc, - ver, - self.treecorr_config, - out_dir, - method=self.cov_estimate_method, - cov_rho=self.compute_cov_rho, - ) - self.print_done("Rho stats finished") - - self._rho_stat_handler = rho_stat_handler - self._tau_stat_handler = tau_stat_handler - - @property - def rho_stat_handler(self): - if not hasattr(self, "_rho_stat_handler"): - self.calculate_rho_tau_stats() - return self._rho_stat_handler - - @property - def tau_stat_handler(self): - if not hasattr(self, "_tau_stat_handler"): - self.calculate_rho_tau_stats() - return self._tau_stat_handler - - @property - def colors(self): - return [self.cc[ver]["colour"] for ver in self.versions] - - def plot_rho_stats(self, abs=False): - - filenames = [f"rho_stats_{ver}.fits" for ver in self.versions] - - savefig = "rho_stats.png" - self.rho_stat_handler.plot_rho_stats( - filenames, - self.colors, - self.versions, - savefig=savefig, - legend="outside", - abs=abs, - ) - plt.close() - - self.print_done( - f"Rho stats plot saved to " - + f"{os.path.abspath(self.rho_stat_handler.catalogs._output)}/{savefig}", - ) - - def plot_tau_stats(self, plot_tau_m=False): - filenames = [f"tau_stats_{ver}.fits" for ver in self.versions] - - savefig = "tau_stats.png" - self.tau_stat_handler.plot_tau_stats( - filenames, - self.colors, - self.versions, - savefig=savefig, - legend="outside", - plot_tau_m=plot_tau_m, - ) - plt.close() - - self.print_done( - f"Tau stats plot saved to " - + f"{os.path.abspath(self.tau_stat_handler.catalogs._output)}/{savefig}", - ) - - def set_params_rho_tau(self, params, params_psf, survey="other"): - if survey in ("DES", "SP_axel_v0.0", "SP_axel_v0.0_repr"): - params["patch_number"] = 120 - print("DES, jackknife patch number = 120") - elif survey == "SP_axel_v0.0": - params["patch_number"] = 120 - print("SP_Axel_v0.0, jackknife patch number =120") - elif survey == "SP_v1.4-P3" or survey == "SP_v1.4-P3_LFmask": - params["patch_number"] = 120 - print("SP_v1.4, jackknife patch number =120") - else: - params["patch_number"] = 150 - - params["ra_col"] = params_psf["ra_col"] - params["dec_col"] = params_psf["dec_col"] - params["e1_PSF_col"] = params_psf["e1_PSF_col"] - params["e2_PSF_col"] = params_psf["e2_PSF_col"] - params["e1_star_col"] = params_psf["e1_star_col"] - params["e2_star_col"] = params_psf["e2_star_col"] - params["PSF_size"] = params_psf["PSF_size"] - params["star_size"] = params_psf["star_size"] - if survey != "DES": - params["PSF_flag"] = params_psf["PSF_flag"] - params["star_flag"] = params_psf["star_flag"] - params["ra_units"] = "deg" - params["dec_units"] = "deg" - - return params - - @property - def psf_fitter(self): - if not hasattr(self, "_psf_fitter"): - self._psf_fitter = PSFErrorFit( - self.rho_stat_handler, - self.tau_stat_handler, - self.rho_stat_handler.catalogs._output, - ) - return self._psf_fitter - - def calculate_rho_tau_fits(self): - assert self.rho_tau_method != "none" - - # this initializes the rho_tau_fits attribute - self._rho_tau_fits = {"flat_sample_list": [], "result_list": [], "q_list": []} - quantiles = [1 - self.quantile, self.quantile] - - self._xi_psf_sys = {} - for ver in self.versions: - params = self.set_params_rho_tau( - self.results[ver]._params, - self.cc[ver]["psf"], - survey=ver, - ) - - npatch = {"sim": 300, "jk": params["patch_number"]}.get( - self.cov_estimate_method, None - ) - - flat_samples, result, q = utils_cosmo_val.get_samples( - self.psf_fitter, - ver, - cov_type=self.cov_estimate_method, - apply_debias=npatch, - sampler=self.rho_tau_method, - ) - - self.rho_tau_fits["flat_sample_list"].append(flat_samples) - self.rho_tau_fits["result_list"].append(result) - self.rho_tau_fits["q_list"].append(q) - - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") - nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] - xi_psf_sys_samples = np.array([]).reshape(0, nbins) - - for i in range(len(flat_samples)): - xi_psf_sys = self.psf_fitter.compute_xi_psf_sys(flat_samples[i]) - xi_psf_sys_samples = np.vstack([xi_psf_sys_samples, xi_psf_sys]) - - self._xi_psf_sys[ver] = { - "mean": np.mean(xi_psf_sys_samples, axis=0), - "var": np.var(xi_psf_sys_samples, axis=0), - "quantiles": np.quantile(xi_psf_sys_samples, quantiles, axis=0), - } - - @property - def rho_tau_fits(self): - if not hasattr(self, "_rho_tau_fits"): - self.calculate_rho_tau_fits() - return self._rho_tau_fits - - def plot_rho_tau_fits(self): - out_dir = self.rho_stat_handler.catalogs._output - - savefig = f"{out_dir}/contours_tau_stat.png" - psfleak_plots.plot_contours( - self.rho_tau_fits["flat_sample_list"], - names=["x0", "x1", "x2"], - labels=[r"\alpha", r"\beta", r"\eta"], - savefig=savefig, - legend_labels=self.versions, - legend_loc="upper right", - contour_colors=self.colors, - markers={"x0": 0, "x1": 1, "x2": 1}, - ) - plt.close() - self.print_done(f"Tau contours plot saved to {os.path.abspath(savefig)}") - - plt.figure(figsize=(15, 6)) - for mcmc_result, ver, color, flat_sample in zip( - self.rho_tau_fits["result_list"], - self.versions, - self.colors, - self.rho_tau_fits["flat_sample_list"], - ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") - for i in range(100): - self.psf_fitter.plot_xi_psf_sys( - flat_sample[-i + 1], ver, color, alpha=0.1 - ) - self.psf_fitter.plot_xi_psf_sys(mcmc_result[1], ver, color) - plt.legend() - savefig = os.path.abspath(f"{out_dir}/xi_psf_sys_samples.png") - cs_plots.savefig(savefig) - self.print_done(f"xi_psf_sys samples plot saved to {savefig}") - - plt.figure(figsize=(15, 6)) - for mcmc_result, ver, color, flat_sample in zip( - self.rho_tau_fits["result_list"], - self.versions, - self.colors, - self.rho_tau_fits["flat_sample_list"], - ): - - ls = self.cc[ver]["ls"] - theta = self.psf_fitter.rho_stat_handler.rho_stats["theta"] - xi_psf_sys = self.xi_psf_sys[ver] - plt.plot(theta, xi_psf_sys["mean"], linestyle=ls, color=color) - plt.plot(theta, xi_psf_sys["quantiles"][0], linestyle=ls, color=color) - plt.plot(theta, xi_psf_sys["quantiles"][1], linestyle=ls, color=color) - plt.fill_between( - theta, - xi_psf_sys["quantiles"][0], - xi_psf_sys["quantiles"][1], - color=color, - alpha=0.25, - label=ver, - ) - - plt.xscale("log") - plt.yscale("log") - plt.xlabel(r"$\theta$ [arcmin]") - plt.ylabel(r"$\xi^{\rm PSF}_{\rm sys}$") - plt.title(f"{1 - self.quantile:.1%}, {self.quantile:.1%} quantiles") - plt.legend() - plt.show() - savefig = os.path.abspath(f"{out_dir}/xi_psf_sys_quantiles.png") - cs_plots.savefig(savefig) - self.print_done(f"xi_psf_sys quantiles plot saved to {savefig}") - - for mcmc_result, ver, flat_sample in zip( - self.rho_tau_fits["result_list"], - self.versions, - self.rho_tau_fits["flat_sample_list"], - ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") - for yscale in ("linear", "log"): - out_path = os.path.abspath( - f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" - ) - self.psf_fitter.plot_xi_psf_sys_terms( - ver, mcmc_result[1], out_path, yscale=yscale - ) - self.print_done( - f"{yscale}-scale xi_psf_sys terms plot saved to {out_path}" - ) - - @property - def xi_psf_sys(self): - if not hasattr(self, "_xi_psf_sys"): - self.calculate_rho_tau_fits() - return self._xi_psf_sys - - def plot_footprints(self): - self.print_start("Plotting footprints:") - for ver in self.versions: - self.print_magenta(ver) - results = self.results[ver] - - fp = FootprintPlotter() - - for region in fp._regions: - out_path = os.path.abspath( - f"{self.cc['paths']['output']}/footprint_{ver}_{region}.png" - ) - if os.path.exists(out_path): - self.print_done( - f"Skipping footprint plot, {out_path} exists" - ) - else: - with self.results[ver].temporarily_load_data(): - hsp_map = fp.create_hsp_map( - self.results[ver].dat_shear["RA"], - self.results[ver].dat_shear["Dec"], - ) - fp.plot_region(hsp_map, fp._regions[region], outpath=out_path) - self.print_done("Footprint plot saved to " + out_path) - - def calculate_scale_dependent_leakage(self): - self.print_start("Calculating scale-dependent leakage:") - for ver in self.versions: - self.print_magenta(ver) - results = self.results[ver] - - output_base_path = os.path.abspath( - f'{self.cc["paths"]["output"]}/leakage_{ver}/xi_for_leak_scale' - ) - output_path_ab = f"{output_base_path}_a_b.txt" - output_path_aa = f"{output_base_path}_a_a.txt" - - with self.results[ver].temporarily_load_data(): - if os.path.exists(output_path_ab) and os.path.exists(output_path_aa): - self.print_green( - f"Skipping computation, reading {output_path_ab} and {output_path_aa} instead" - ) - - results.r_corr_gp = treecorr.GGCorrelation(self.treecorr_config) - results.r_corr_gp.read(output_path_ab) - - results.r_corr_pp = treecorr.GGCorrelation(self.treecorr_config) - results.r_corr_pp.read(output_path_aa) - - else: - results.compute_corr_gp_pp_alpha(output_base_path=output_base_path) - - results.do_alpha(fast=True) - results.do_xi_sys() - - self.print_done("Finished scale-dependent leakage calculation.") - - def plot_scale_dependent_leakage(self): - if not hasattr(self.results[self.versions[0]], "r_corr_gp"): - self.calculate_scale_dependent_leakage() - - theta = [] - y = [] - yerr = [] - labels = [] - colors = [] - linestyles = [] - markers = [] - - for ver in self.versions: - if hasattr(self.results[ver], "r_corr_gp"): - theta.append(self.results[ver].r_corr_gp.meanr) - y.append(self.results[ver].alpha_leak) - yerr.append(self.results[ver].sig_alpha_leak) - labels.append(ver) - colors.append(self.cc[ver]["colour"]) - linestyles.append(self.cc[ver]["ls"]) - markers.append(self.cc[ver]["marker"]) - - if len(theta) > 0: - - # Log x - out_path = os.path.abspath( - f"{self.cc['paths']['output']}/alpha_leak_log.png" - ) - - title = r"$\alpha$ leakage" - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\alpha(\theta)$" - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=self.ylim_alpha, - labels=labels, - colors=colors, - linestyles=linestyles, - shift_x=True, - ) - cs_plots.savefig(out_path) - self.print_done(f"Log-scale alpha leakage plot saved to {out_path}") - - # Lin x - out_path = os.path.abspath( - f"{self.cc['paths']['output']}/alpha_leak_lin.png" - ) - - title = r"$\alpha$ leakage" - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\alpha(\theta)$" - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - xlog=False, - xlim=[-10, self.theta_max_plot], - ylim=self.ylim_alpha, - labels=labels, - colors=colors, - linestyles=linestyles, - shift_x=False, - ) - cs_plots.savefig(out_path) - self.print_done(f"Lin-scale alpha leakage plot saved to {out_path}") - - # Plot xi_sys - y = [] - yerr = [] - colors = [] - linestyles = [] - - for ver in self.versions: - if hasattr(self.results[ver], "C_sys_p"): - y.append(self.results[ver].C_sys_p) - yerr.append(self.results[ver].C_sys_std_p) - labels.append(ver) - colors.append(self.cc[ver]["colour"]) - linestyles.append(self.cc[ver]["ls"]) - - if len(y) > 0: - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\xi^{\rm sys}_+(\theta)$" - title = "Cross-correlation leakage" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_p.png") - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - colors=colors, - linestyles=linestyles, - # shift_x=True, - ) - cs_plots.savefig(out_path) - self.print_done(f"xi_sys_plus plot saved to {out_path}") - - y = [] - yerr = [] - for ver in self.versions: - if hasattr(self.results[ver], "C_sys_m"): - y.append(self.results[ver].C_sys_m) - yerr.append(self.results[ver].C_sys_std_m) - - if len(y) > 0: - xlabel = r"$\theta$ [arcmin]" - ylabel = r"$\xi^{\rm sys}_-(\theta)$" - title = "Cross-correlation leakage" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_m.png") - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - cs_plots.plot_data_1d( - theta, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[-1e-7, 1e-6], - colors=colors, - linestyles=linestyles, - # shift_x=True, - ) - cs_plots.savefig(out_path) - self.print_done(f"xi_sys_minus plot saved to {out_path}") - - def calculate_objectwise_leakage(self): - if not hasattr(self.results[self.versions[0]], "alpha_leak_mean"): - self.calculate_scale_dependent_leakage() - - self.print_start("Object-wise leakage:") - mix = True - order = "lin" - for ver in self.versions: - self.print_magenta(ver) - - results_obj = self.results_objectwise[ver] - results_obj.check_params() - results_obj.update_params() - results_obj.prepare_output() - - with self.results[ver].temporarily_load_data(): - results_obj._dat = self.results[ver].dat_shear - - out_base = results_obj.get_out_base(mix, order) - out_path = f"{out_base}.pkl" - if os.path.exists(out_path): - self.print_green( - f"Skipping object-wise leakage, file {out_path} exists" - ) - results_obj.par_best_fit = leakage.read_from_file(out_path) - else: - self.print_cyan("Computing object-wise leakage regression") - - # Run - results_obj.PSF_leakage() - - # Gather coefficients - leakage_coeff = {} - for ver in self.versions: - leakage_coeff[ver] = {} - results = self.results[ver] - results_obj = self.results_objectwise[ver] - # Object-wise leakage - leakage_coeff[ver]["a11"] = ufloat( - results_obj.par_best_fit["a11"].value, - results_obj.par_best_fit["a11"].stderr, - ) - leakage_coeff[ver]["a22"] = ufloat( - results_obj.par_best_fit["a22"].value, - results_obj.par_best_fit["a22"].stderr, - ) - leakage_coeff[ver]["aii_mean"] = 0.5 * ( - leakage_coeff[ver]["a11"] + leakage_coeff[ver]["a22"] - ) - - # Scale-dependent leakage: mean - leakage_coeff[ver]["alpha_mean"] = ufloat( - results.alpha_leak_mean, results.alpha_leak_std - ) - # Scale-dependent leakage: value at smallest scale - leakage_coeff[ver]["alpha_1"] = ufloat( - results.alpha_leak[0], results.sig_alpha_leak[0] - ) - # Scale-dependent leakage: value extrapolated to 0 using affine model - leakage_coeff[ver]["alpha_0"] = ufloat( - results.alpha_affine_best_fit["c"].value, - results.alpha_affine_best_fit["c"].stderr, - ) - - self.leakage_coeff = leakage_coeff - - def plot_objectwise_leakage(self): - if not hasattr(self, "leakage_coeff"): - self.calculate_objectwise_leakage() - - self.print_start("Plotting object-wise leakage:") - fig = cs_plots.figure(figsize=(15, 15)) - - linestyles = ["-", "--", ":"] - fillstyles = ["full", "none", "left", "right", "bottom", "top"] - - for ver in self.versions: - label = ver - for key, ls, fs in zip( - ["alpha_mean", "alpha_1", "alpha_0"], linestyles, fillstyles - ): - x = self.leakage_coeff[ver]["aii_mean"].nominal_value - dx = self.leakage_coeff[ver]["aii_mean"].std_dev - y = self.leakage_coeff[ver][key].nominal_value - dy = self.leakage_coeff[ver][key].std_dev - - eb = plt.errorbar( - x, - y, - xerr=dx, - yerr=dy, - fmt=self.cc[ver]["marker"], - color=self.cc[ver]["colour"], - fillstyle=fs, - label=label, - ) - label = None - eb[-1][0].set_linestyle(ls) - - # y=x line - xlim = 0.02 - x = [-xlim, xlim] - y = x - plt.plot(x, y, "k:", linewidth=0.5) - - plt.legend() - plt.xlabel(r"tr $a$ (object-wise)") - plt.ylabel(r"$\alpha$ (scale-dependent)") - out_path = os.path.abspath( - f"{self.cc['paths']['output']}/leakage_coefficients.png" - ) - cs_plots.savefig(out_path) - self.print_done(f"Object-wise leakage coefficients plot saved to {out_path}") - - def calculate_objectwise_leakage_aux(self): - - self.print_start("Object-wise leakage auxiliary quantities:") - for ver in self.versions: - self.print_magenta(ver) - - results_obj = self.results_objectwise[ver] - results_obj.check_params() - results_obj.update_params() - results_obj.prepare_output() - - with self.results[ver].temporarily_load_data(): - results_obj._dat = self.results[ver].dat_shear - - if not "cols" in results_obj._params: - self.print_green("Skipping object-wise leakage (aux quantities), no input columns for regression found") - else: - self.print_cyan(f"Computing object-wise leakage regression with aux quantities: {results_obj._params['cols']}") - - # Run - results_obj.obs_leakage() - - def plot_objectwise_leakage_aux(self): - self.calculate_objectwise_leakage_aux() - - def plot_ellipticity(self, nbins=200): - out_path = os.path.abspath(f"{self.cc['paths']['output']}/ell_hist.png") - if os.path.exists(out_path): - self.print_done(f"Skipping ellipticity histograms, {out_path} exists") - else: - self.print_start("Computing ellipticity histograms:") - - fig, axs = plt.subplots(1, 2, figsize=(22, 7)) - for ver in self.versions: - with self.results[ver].temporarily_load_data(): - self.print_magenta(ver) - R = self.cc[ver]["shear"]["R"] - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R - w = self.results[ver].dat_shear["w_col"] - - axs[0].hist( - e1, - bins=nbins, - density=False, - histtype="step", - weights=w, - label=ver, - color=self.cc[ver]["colour"], - ) - axs[1].hist( - e2, - bins=nbins, - density=False, - histtype="step", - weights=w, - label=ver, - color=self.cc[ver]["colour"], - ) - - for idx in (0, 1): - axs[idx].set_xlabel(f"$e_{idx}$") - axs[idx].set_ylabel("frequency") - axs[idx].legend() - axs[idx].set_xlim([-1.5, 1.5]) - cs_plots.savefig(out_path) - self.print_done("Ellipticity histograms saved to " + out_path) - - def plot_separation(self, nbins=200): - self.print_start("Separation histograms") - if "SP_matched_MP_v1.0" in self.versions: - fig, axs = plt.subplots(1, 1, figsize=(10, 7)) - sep = self.results["SP_matched_MP_v1.0"].dat_shear["Separation"] - axs.hist( - sep, - bins=nbins, - density=False, - histtype="step", - label="SP_matched_MP_v1.0", - color=self.cc["SP_matched_MP_v1.0"]["colour"], - ) - print("Max separation: %s arcsec" % max(sep)) - axs.set_xlabel(r"Separation $\theta$ [arcsec]") - axs.legend() - else: - self.print_done("SP_matched_MP_v1.0 not in versions") - - def calculate_additive_bias(self): - self.print_start("Calculating additive bias:") - self._c1 = {} - self._c2 = {} - for ver in self.versions: - self.print_magenta(ver) - with self.results[ver].temporarily_load_data(): - R = self.cc[ver]["shear"]["R"] - self._c1[ver] = np.average( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R, - weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ) - self._c2[ver] = np.average( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R, - weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ) - self.print_done("Finished additive bias calculation.") - - @property - def c1(self): - if not hasattr(self, "_c1"): - print("MKDEBUG calc c1") - self.calculate_additive_bias() - return self._c1 - - @property - def c2(self): - if not hasattr(self, "_c2"): - print("MKDEBUG calc c2") - self.calculate_additive_bias() - return self._c2 - - def calculate_2pcf(self): - self.print_start(f"Computing 2PCF") - - self._cat_ggs = {} - for ver in self.versions: - self.print_magenta(ver) - gg = self._cat_ggs[ver] = treecorr.GGCorrelation(self.treecorr_config) - - out_fname = os.path.abspath(f"{self.cc['paths']['output']}/xi_pm_{ver}.txt") - if os.path.exists(out_fname): - self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") - gg.read(out_fname) - else: - # Run TreeCorr - with self.results[ver].temporarily_load_data(): - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - if ver != "DES": - R = self.cc[ver]["shear"]["R"] - g1 = (e1 - self.c1[ver]) / R - g2 = (e2 - self.c2[ver]) / R - else: - R11 = self.cc[ver]["shear"]["R11"] - R22 = self.cc[ver]["shear"]["R22"] - g1 = (e1 - self.c1[ver]) / np.average( - self.results[ver].dat_shear[R11] - ) - g2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - - self.c2[ver] - ) / np.average(self.results[ver].dat_shear[R22]) - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=self.npatch, - ) - gg.process(cat_gal) - gg.write(out_fname) - - # Save xi_p and xi_m results to fits file - lst = np.arange(1, self.treecorr_config["nbins"] + 1) - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column( - name="ANG", format="D", unit="arcmin", array=gg.meanr - ) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - xiplus_hdu.writeto( - f"{self.cc['paths']['output']}/xi_plus_{ver}.fits", overwrite=True - ) - - # append xi_minus header info - ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} - for key in ximinus_dict: - ximinus_hdu.header[key] = ximinus_dict[key] - ximinus_hdu.writeto( - f"{self.cc['paths']['output']}/xi_minus_{ver}.fits", overwrite=True - ) - - self.print_done("Done 2PCF") - - @property - def cat_ggs(self): - if not hasattr(self, "_cat_ggs"): - self.calculate_2pcf() - return self._cat_ggs - - def plot_2pcf(self): - # Plot of n_pairs - fig, ax = plt.subplots(ncols=1, nrows=1) - for ver in self.versions: - plt.plot( - self.cat_ggs[ver].meanr, - self.cat_ggs[ver].npairs, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.ylabel(r"$n_{\rm pair}$") - plt.legend() - out_path = os.path.abspath(f"{self.cc['paths']['output']}/n_pair.png") - cs_plots.savefig(out_path) - self.print_done(f"n_pair plot saved to {out_path}") - - # Plot of xi_+ - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxip), - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\xi_+(\theta)$") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_p.png") - cs_plots.savefig(out_path) - self.print_done(f"xi_plus plot saved to {out_path}") - - # Plot of xi_- - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xim, - yerr=np.sqrt(self.cat_ggs[ver].varxim), - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\xi_-(\theta)$") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_m.png") - cs_plots.savefig(out_path) - self.print_done(f"xi_minus plot saved to {out_path}") - - # Plot of xi_+(theta) * theta - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr, - self.cat_ggs[ver].xip * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxip) * self.cat_ggs[ver].meanr, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\theta \xi_+(\theta)$") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_p_theta.png") - cs_plots.savefig(out_path) - self.print_done(f"xi_plus_theta plot saved to {out_path}") - - # Plot of xi_- * theta - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - for idx, ver in enumerate(self.versions): - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xim * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxim) * self.cat_ggs[ver].meanr, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - ) - plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylabel(r"$\theta \xi_-(\theta)$") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_m_theta.png") - cs_plots.savefig(out_path) - self.print_done(f"xi_minus_theta plot saved to {out_path}") - - # Plot of xi_+ with and without xi_psf_sys - for idx, ver in enumerate(self.versions): - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxim), - label=r"$\xi_+$", - ls="solid", - color="green", - ) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt(self.xi_psf_sys[ver]["var"]), - label=r"$\xi^{\rm psf}_{+, {\rm sys}}$", - ls="dotted", - color="red", - ) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip + self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt(self.cat_ggs[ver].varxip + self.xi_psf_sys[ver]["var"]), - label=r"$\xi_+ + \xi^{\rm psf}_{+, {\rm sys}}$", - ls="dashdot", - color="magenta", - ) - - plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylim(1e-8, 5e-4) - plt.ylabel(r"$\xi_+(\theta)$") - out_path = os.path.abspath( - f"{self.cc['paths']['output']}/xi_p_xi_psf_sys_{ver}.png" - ) - cs_plots.savefig(out_path) - self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") - - def calculate_aperture_mass_dispersion( - self, theta_min=0.3, theta_max=200, nbins=500, nbins_map=15, npatch=25 - ): - self.print_start("Computing aperture-mass dispersion") - - self._map2 = {} - theta_map = np.geomspace(theta_min * 5, theta_max / 2, nbins_map) - self._map2["theta_map"] = theta_map - - treecorr_config = { - **self.treecorr_config, - "min_sep": theta_min, - "max_sep": theta_max, - "nbins": nbins, - } - - for ver in self.versions: - self.print_magenta(ver) - - gg = treecorr.GGCorrelation(treecorr_config) - - out_fname = os.path.abspath( - f"{self.cc['paths']['output']}/xi_for_map2_{ver}.txt" - ) - if os.path.exists(out_fname): - self.print_green(f"Skipping xi for Map2, {out_fname} exists") - gg.read(out_fname) - else: - with self.results[ver].temporarily_load_data(): - R = self.cc[ver]["shear"]["R"] - g1 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - - self.c1[ver] - ) / R - g2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - - self.c2[ver] - ) / R - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=npatch, - ) - - gg.process(cat_gal) - gg.write(out_fname) - del cat_gal - del g1 - del g2 - - mapsq, mapsq_im, mxsq, mxsq_im, varmapsq = gg.calculateMapSq( - R=theta_map, - m2_uform="Schneider", - ) - out_fname_map2 = os.path.abspath( - f"{self.cc['paths']['output']}/map2_{ver}.txt" - ) - if os.path.exists(out_fname_map2): - self.print_green(f"Skipping Map2, {out_fname_map2} exists") - else: - print(f"Writing Map2 to output file {out_fname_map2} ") - gg.writeMapSq(out_fname_map2, R=theta_map, m2_uform="Schneider") - self._map2[ver] = { - "mapsq": mapsq, - "mapsq_im": mapsq_im, - "mxsq": mxsq, - "mxsq_im": mxsq_im, - "varmapsq": varmapsq, - } - - self.print_done("Done aperture-mass dispersion") - - @property - def map2(self): - if not hasattr(self, "_map2"): - self.calculate_aperture_mass_dispersion() - return self._map2 - - def plot_aperture_mass_dispersion(self): - for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: - x = [] - y = [] - yerr = [] - labels = [] - colors = [] - linestyles = [] - for ver in self.versions: - x.append(self.map2["theta_map"]) - y.append(self.map2[ver][mode]) - yerr.append(np.sqrt(self.map2[ver]["varmapsq"])) - labels.append(ver) - colors.append(self.cc[ver]["colour"]) - linestyles.append(self.cc[ver]["ls"]) - - xlabel = r"$\theta$ [arcmin]" - ylabel = "dispersion" - title = f"Aperture-mass dispersion {mode}" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/{mode}.png") - cs_plots.plot_data_1d( - x, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[-1e-6, 2e-5], - colors=colors, - linestyles=linestyles, - shift_x=True, - ) - cs_plots.savefig(out_path) - self.print_done(f"linear-scale {mode} plot saved to {out_path}") - - for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: - x = [] - y = [] - yerr = [] - for ver in self.versions: - x.append(self.map2["theta_map"]) - y.append(np.abs(self.map2[ver][mode])) - yerr.append(np.sqrt(self.map2[ver]["varmapsq"])) - xlabel = r"$\theta$ [arcmin]" - ylabel = "dispersion" - title = f"Aperture-mass dispersion mode {mode}" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/{mode}_log.png") - cs_plots.plot_data_1d( - x, - y, - yerr, - title, - xlabel, - ylabel, - out_path=None, - labels=labels, - xlog=True, - ylog=True, - xlim=[self.theta_min_plot, self.theta_max_plot], - ylim=[1e-9, 3e-5], - colors=colors, - linestyles=linestyles, - shift_x=True, - ) - cs_plots.savefig(out_path) - self.print_done(f"log-scale {mode} plot saved to {out_path}") - - def calculate_pure_eb( - self, - theta_min=0.1, - theta_max=250, - nbins=20, - theta_min_int=0.04, - theta_max_int=500, - nbins_int=1000, - ): - self.print_start("Computing pure E/B") - - treecorr_config = { - **self.treecorr_config, - "min_sep": theta_min, - "max_sep": theta_max, - "nbins": nbins, - } - - # nbins_int = (nbins - 1) * integration_oversample + 1 - treecorr_config_int = { - **treecorr_config, - "min_sep": theta_min_int, - "max_sep": theta_max_int, - "nbins": nbins_int, - } - - print("Integration correlation function parameters:") - print(treecorr_config_int) - - print("Output correlation function parameters:") - print(treecorr_config) - - for ver in self.versions: - self.print_magenta(ver) - - gg_int = treecorr.GGCorrelation(treecorr_config_int) - gg = treecorr.GGCorrelation(treecorr_config) - - out_fname = os.path.abspath( - # f"{self.cc['paths']['output']}/xi_for_pure_eb_{ver}.txt" - f"{self.cc['paths']['output']}/xi_for_pure_eb_thetamin={theta_min}_thetamax={theta_max}_nbins={nbins}_npatch={self.npatch}_{ver}.txt" - ) - out_fname_int = os.path.abspath( - f"{self.cc['paths']['output']}/xi_for_pure_eb_int_thetaminint={theta_min_int}_thetamaxint_{theta_max_int}_nbinsint={nbins_int}_npatchint={self.npatch}_{ver}.txt" - # f"{self.cc['paths']['output']}/xi_for_pure_eb_{ver}_int.txt" - ) - if os.path.exists(out_fname) and os.path.exists(out_fname_int): - self.print_green( - f"Skipping xi for COSEBIs:\n{out_fname}\n{out_fname_int}\nexist." - ) - gg.read(out_fname) - gg_int.read(out_fname_int) - else: - with self.results[ver].temporarily_load_data(): - R = self.cc[ver]["shear"]["R"] - g1 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - - self.c1[ver] - ) / R - g2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - - self.c2[ver] - ) / R - - cat = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=self.npatch, - ) - del g1, g2, R - - self.print_cyan("Integration binning") - gg_int.process(cat) - gg_int.write(out_fname_int, write_patch_results=True, write_cov=True) - - self.print_cyan("Output binning") - gg.process(cat) - gg.write(out_fname, write_patch_results=True, write_cov=True) - - def pure_EB(corrs): - gg, gg_int = corrs - return get_pure_EB_modes( - theta=gg.meanr, - xip=gg.xip, - xim=gg.xim, - theta_int=gg_int.meanr, - xip_int=gg_int.xip, - xim_int=gg_int.xim, - tmin=theta_min, - tmax=theta_max, - ) - - xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb = pure_EB([gg, gg_int]) - cov = treecorr.estimate_multi_cov( - [gg, gg_int], "jackknife", func=lambda x: np.hstack(pure_EB(x)) - ) - - results = { - "xip_E": xip_E, - "xim_E": xim_E, - "xip_B": xip_B, - "xim_B": xim_B, - "xip_amb": xip_amb, - "xim_amb": xim_amb, - "cov": cov, - } - - return results - -class FootprintPlotter: - """Class to create footprint plots. - - Parameters - ----------- - nside_coverage: int, optional - basic resolution of map; default is 32 - nside_map: - fine resolution for plotting; default is 2048 - - """ - - # Dictionary storing region parameters - _regions = { - "NGC": {"ra_0": 180, "extend": [120, 270, 20, 70], "vmax": 60}, - "SGC": {"ra_0": 15, "extend": [-20, 45, 20, 45], "vmax": 60}, - "fullsky": {"ra_0": 150, "extend": [0, 360, -90, 90], "vmax": 60}, - } - - def __init__(self, nside_coverage=32, nside_map=2048): - - self._nside_coverage = nside_coverage - self._nside_map = nside_map - - def create_hsp_map(self, ra, dec): - """Create Hsp Map. - - Create healsparse map. - - Parameters - ---------- - ra : numpy.ndarray - right ascension values - dec : numpy.ndarray - declination values - - Returns - ------- - hsp.HealSparseMap - map - - """ - # Create empty map - hsp_map = hsp.HealSparseMap.make_empty( - self._nside_coverage, - self._nside_map, - dtype=np.float32, - sentinel=np.nan - ) - - # Get pixel list corresponding to coordinates - hpix = hp.ang2pix(self._nside_map, ra, dec, nest=True, lonlat=True) - - # Get count of objects per pixel - pixel_counts = Counter(hpix) - - # List of unique pixels - unique_hpix = np.array(list(pixel_counts.keys())) - - # Number of objects - values = np.array(list(pixel_counts.values()), dtype=np.float32) - - # Create maps with numbers per pixel - hsp_map[unique_hpix] = values - - return hsp_map - - def plot_area( - self, - hsp_map, - ra_0=0, - extend=[120, 270, 29, 70], - vmax=60, - projection=None, - outpath=None, - title=None, - ): - """Plot Area. - - Plot catalogue in an area on the sky. - - Parameters - ---------- - hsp_map : hsp_HealSparseMap - input map - ra_0 : float, optional - anchor point in R.A.; default is 0 - extend : list, optional - sky region, extend=[ra_low, ra_high, dec_low, dec_high]; - default is [120, 270, 29, 70] - vmax : float, optional - maximum pixel value to plot with color; default is 60 - projection : skyproj.McBrydeSkyproj - if ``None`` (default), a new plot is created - outpath : str, optional - output path, default is ``None`` - title : str, optional - print title if not ``None`` (default) - - Returns - -------- - skyproj.McBrydeSkyproj - projection instance - plt.axes.Axes - axes instance - - Raises - ------ - ValueError - if no object found in region - - """ - if not projection: - - # Create new figure and axes - fig, ax = plt.subplots(figsize=(10, 10)) - - # Create new projection - projection = skyproj.McBrydeSkyproj( - ax=ax, - lon_0=ra_0, - extent=extend, - autorescale=True, - vmax=vmax - ) - else: - ax = None - - try: - _ = projection.draw_hspmap( - hsp_map, lon_range=extend[0:2], - lat_range=extend[2:] - ) - except ValueError: - msg = "No object found in region to draw" - print(f"{msg}, continuing...") - #raise ValueError(msg) - - projection.draw_milky_way(width=25, linewidth=1.5, color='black', linestyle='-') - - if title: - plt.title(title, pad=5) - - if outpath: - plt.savefig(outpath) - - return projection, ax - - def plot_region(self, hsp_map, region, projection=None, outpath=None, title=None): - - return self.plot_area( - hsp_map, - region["ra_0"], - region["extend"], - region["vmax"], - projection=projection, - outpath=outpath, - title=title, - ) - - def plot_all_regions(self, hsp_map, outbase=None): - - for region in self._regions: - if outbase: - outpath = f"{outbase}_{region}.png" - else: - outpath = None - self.plot_region(hsp_map, self._regions[region], outpath=outpath) - - - @classmethod - def hp_pixel_centers(cls, nside, nest=False): - - # Get number of pixels for given nside - npix = hp.nside2npix(nside) - - # Get pixel indices - pix_indices = np.arange(npix) - - # Get coordinates of pixel centers - ra, dec = hp.pix2ang(nside, pix_indices, nest=nest, lonlat=True) - - return ra, dec, npix - - @classmethod - def plot_footprint_as_hp(cls, hsp_map, nside, outpath=None, title=None): - - ra, dec, npix = cls.hp_pixel_centers(nside) - - # Create an empty HEALPix map - m = np.full(npix, np.nan) - - fig, ax = plt.subplots(figsize=(10, 10)) - - # Plot the HEALPix grid - hp.mollview(m, title=title, coord="C", notext=True, rot=(180, 0, 0)) - - # Define the Galactic Plane: l = [0, 360], b = 0° - for l0, ls in zip((-5, 0, 5), (":", "-", ":")): - l_values = np.linspace(0, 360, 500) # 500 points along the plane - b_values = np.zeros_like(l_values) # Galactic latitude is 0 (the plane) - - # Convert (l, b) to (λ, β) - Ecliptic coordinates - coords = SkyCoord(l=l_values*u.degree, b=b_values*u.degree, frame='galactic') - ecl_coords = coords.transform_to('barycentrictrueecliptic') # Ecliptic frame - - # Extract Ecliptic longitude (λ) and latitude (β) - lambda_ecl = ecl_coords.lon.deg # Ecliptic longitude - beta_ecl = ecl_coords.lat.deg # Ecliptic latitude - - # Convert to HEALPix projection coordinates (colatitude, longitude) - theta = np.radians(90 - beta_ecl) # HEALPix uses colatitude - phi = np.radians(lambda_ecl) # HEALPix uses longitude - - # Create a healpy Mollweide projection in Ecliptic coordinates - hp.projplot(theta, phi, linestyle=ls, color='black', linewidth=1) # Plot the outline - - # Apply mask - mask_values = hsp_map.get_values_pos(ra, dec, valid_mask=True, lonlat=True) - - ok = np.where(mask_values == False)[0] - #nok = np.where(mask_values == False)[0] - - hp.projscatter(ra[ok], dec[ok], lonlat=True, color="green", s=1, marker=".") - #hp.projscatter(ra[nok], dec[nok], lonlat=True, color="red", s=1, marker=".") - - plt.tight_layout() - - if outpath: - plt.savefig(outpath) - - plt.show() - -def hsp_map_logical_or(maps, verbose=False): - """ - Hsp Map Logical Or. - - Logical AND of HealSparseMaps. - - """ - if verbose: - print("Combine all maps...") - - # Ensure consistency in coverage and data type - nside_coverage = maps[0].nside_coverage - nside_sparse = maps[0].nside_sparse - dtype = maps[0].dtype - - for m in maps: - # MKDEBUG TODO: Change nside if possible - if m.nside_coverage != nside_coverage: - raise ValueError( - f"Coverage nside={m.nside_coverage} does not match {nside_coverage}" - ) - if m.dtype != dtype: - raise ValueError( - f"Data type {m.dtype} does not match {dtype}" - ) - - # Create an empty HealSparse map - map_comb = hsp.HealSparseMap.make_empty(nside_coverage, nside_sparse, dtype=dtype) - for idx, m in enumerate(maps): - map_comb |= m - - if verbose: - valid_pixels = map_comb.valid_pixels - n_tot = np.sum(valid_pixels) - n_true = np.count_nonzero(valid_pixels) - n_false = n_tot - n_true - print(f"after map {idx}: frac_true={n_true / n_tot:g}, frac_false={n_false / n_tot:g}") - - return map_comb - - def calculate_pseudo_cl_eb_cov(self): - """ - Compute a theoretical Gaussian covariance of the Pseudo-Cl for EE, EB and BB. - """ - self.print_start("Computing Pseudo-Cl covariance") - - nside = self.nside - - try: - self._pseudo_cls - except AttributeError: - self._pseudo_cls = {} - for ver in self.versions: - self.print_magenta(ver) - - if not ver in self._pseudo_cls.keys(): - self._pseudo_cls[ver] = {} - - out_path = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_{ver}.fits") - if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl covariance calculation, {out_path} exists") - self._pseudo_cls[ver]['cov'] = fits.open(out_path) - else: - - params = utils_cosmo_val.get_params_rho_tau(self.cc[ver], survey=ver) - - self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") - - lmax = 2*self.nside - path_redshift_distr = self.data_base_dir + self.cc[ver]["shear"]["redshift_distr"] - pw = hp.pixwin(nside, lmax=lmax) - fiducial_cl = self.get_fiducial(lmax, path_redshift_distr)*pw**2 - - self.print_cyan("Getting a sample of the fiducial Cl's with noise") - - lmin = 8 - lmax = 2*self.nside - b_lmax = lmax - 1 - - if self.binning == 'linear': - step = 10 - b = nmt.NmtBin.from_nside_linear(self.nside, step) - elif self.binning == 'powspace': - ells = np.arange(lmin, lmax+1) - - start = np.power(lmin, self.power) - end = np.power(lmax, self.power) - bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) - - #Get bandpowers - bpws = np.digitize(ells.astype(float), bins_ell) - 1 - bpws[0] = 0 - bpws[-1] = self.n_ell_bins-1 - - b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - - #Load data and create shear and noise maps - cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - - n_gal, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) - mask = n_gal != 0 - - cl_noise, f, wsp = self.get_sample(params, self.nside, b_lmax, b, cat_gal, n_gal, mask, unique_pix, idx_rep) - - fiducial_cl = np.array([fiducial_cl, 0.*fiducial_cl, 0.*fiducial_cl, 0.*fiducial_cl])+ np.mean(cl_noise, axis=1, keepdims=True) - - self.print_cyan("Computing the Pseudo-Cl covariance") - - cw = nmt.NmtCovarianceWorkspace.from_fields(f, f, f, f) - - covar_22_22 = nmt.gaussian_covariance(cw, 2, 2, 2, 2, - fiducial_cl, - fiducial_cl, - fiducial_cl, - fiducial_cl, - wsp, wb=wsp).reshape([self.n_ell_bins, 4, self.n_ell_bins, 4]) - - covar_EE_EE = covar_22_22[:, 0, :, 0] - covar_EE_EB = covar_22_22[:, 0, :, 1] - covar_EE_BE = covar_22_22[:, 0, :, 2] - covar_EE_BB = covar_22_22[:, 0, :, 3] - covar_EB_EE = covar_22_22[:, 1, :, 0] - covar_EB_EB = covar_22_22[:, 1, :, 1] - covar_EB_BE = covar_22_22[:, 1, :, 2] - covar_EB_BB = covar_22_22[:, 1, :, 3] - covar_BE_EE = covar_22_22[:, 2, :, 0] - covar_BE_EB = covar_22_22[:, 2, :, 1] - covar_BE_BE = covar_22_22[:, 2, :, 2] - covar_BE_BB = covar_22_22[:, 2, :, 3] - covar_BB_EE = covar_22_22[:, 3, :, 0] - covar_BB_EB = covar_22_22[:, 3, :, 1] - covar_BB_BE = covar_22_22[:, 3, :, 2] - covar_BB_BB = covar_22_22[:, 3, :, 3] - - self.print_cyan("Saving Pseudo-Cl covariance") - - hdu = fits.HDUList() - - hdu.append(fits.ImageHDU(covar_EE_EE, name="COVAR_EE_EE")) - hdu.append(fits.ImageHDU(covar_EE_EB, name="COVAR_EE_EB")) - hdu.append(fits.ImageHDU(covar_EE_BE, name="COVAR_EE_BE")) - hdu.append(fits.ImageHDU(covar_EE_BB, name="COVAR_EE_BB")) - hdu.append(fits.ImageHDU(covar_EB_EE, name="COVAR_EB_EE")) - hdu.append(fits.ImageHDU(covar_EB_EB, name="COVAR_EB_EB")) - hdu.append(fits.ImageHDU(covar_EB_BE, name="COVAR_EB_BE")) - hdu.append(fits.ImageHDU(covar_EB_BB, name="COVAR_EB_BB")) - hdu.append(fits.ImageHDU(covar_BE_EE, name="COVAR_BE_EE")) - hdu.append(fits.ImageHDU(covar_BE_EB, name="COVAR_BE_EB")) - hdu.append(fits.ImageHDU(covar_BE_BE, name="COVAR_BE_BE")) - hdu.append(fits.ImageHDU(covar_BE_BB, name="COVAR_BE_BB")) - hdu.append(fits.ImageHDU(covar_BB_EE, name="COVAR_BB_EE")) - hdu.append(fits.ImageHDU(covar_BB_EB, name="COVAR_BB_EB")) - hdu.append(fits.ImageHDU(covar_BB_BE, name="COVAR_BB_BE")) - hdu.append(fits.ImageHDU(covar_BB_BB, name="COVAR_BB_BB")) - - hdu.writeto(out_path, overwrite=True) - - self._pseudo_cls[ver]['cov'] = hdu - - self.print_done("Done Pseudo-Cl covariance") - - def calculate_pseudo_cl(self): - """ - Compute the pseudo-Cl of given catalogs. - """ - self.print_start("Computing pseudo-Cl's") - - nside = self.nside - - try: - self._pseudo_cls - except AttributeError: - self._pseudo_cls = {} - for ver in self.versions: - self.print_magenta(ver) - - self._pseudo_cls[ver] = {} - - out_path = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_{ver}.fits") - if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear - else: - params = utils_cosmo_val.get_params_rho_tau(self.cc[ver], survey=ver) - - #Load data and create shear and noise maps - cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - - w = cat_gal[params['w_col']] - self.print_cyan("Creating maps and computing Cl's...") - n_gal_map, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) - mask = n_gal_map != 0 - - shear_map_e1 = np.zeros(hp.nside2npix(nside)) - shear_map_e2 = np.zeros(hp.nside2npix(nside)) - - e1 = cat_gal[params['e1_col']] - e2 = cat_gal[params['e2_col']] - - del cat_gal - - shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1*w) - shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2*w) - shear_map_e1[mask] /= n_gal_map[mask] - shear_map_e2[mask] /= n_gal_map[mask] - - shear_map = shear_map_e1 + 1j*shear_map_e2 - - del shear_map_e1, shear_map_e2 - - ell_eff, cl_shear, wsp = self.get_pseudo_cls(shear_map) - - cl_noise = np.zeros((4, self.n_ell_bins)) - - for i in range(self.nrandom_cell): - - noise_map_e1 = np.zeros(hp.nside2npix(nside)) - noise_map_e2 = np.zeros(hp.nside2npix(nside)) - - e1_rot, e2_rot = self.apply_random_rotation(e1, e2) - - - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) - - noise_map_e1[mask] /= n_gal_map[mask] - noise_map_e2[mask] /= n_gal_map[mask] - - noise_map = noise_map_e1 + 1j*noise_map_e2 - del noise_map_e1, noise_map_e2 - - _, cl_noise_, _ = self.get_pseudo_cls(noise_map, wsp) - cl_noise += cl_noise_ - - cl_noise /= self.nrandom_cell - del e1, e2, e1_rot, e2_rot, w - del n_gal_map - - #This is a problem because the measurement depends on the seed. To be fixed. - #cl_shear = cl_shear - np.mean(cl_noise, axis=1, keepdims=True) - cl_shear = cl_shear - cl_noise - - self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) - - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear - - self.print_done("Done pseudo-Cl's") - - def get_fiducial(self, lmax, redshift_distr): - """ - Get the power spectrum at Planck18 cosmology using CAMB. - """ - planck = Planck18 - - h = planck.H0.value/100 - Om = planck.Om0 - Ob = planck.Ob0 - Oc = Om - Ob - ns = 0.965 - As = 2.1e-9 - m_nu = 0.06 - w = -1 - - pars = camb.set_params(H0=100*h, omch2=Oc*h**2, ombh2=Ob*h**2, ns=ns, mnu=m_nu, w=w, As=As, WantTransfer=True, NonLinear=camb.model.NonLinear_both) - Onu = pars.omeganu - Oc = Om - Ob - Onu - pars = camb.set_params(H0=100*h, omch2=Oc*h**2, ombh2=Ob*h**2, ns=ns, mnu=m_nu, w=w, As=As, WantTransfer=True, NonLinear=camb.model.NonLinear_both) - - z, dndz = np.loadtxt(redshift_distr, unpack=True) - - #getthe expected cl's from CAMB - pars.min_l = 1 - pars.set_for_lmax(lmax) - pars.SourceWindows = [ - camb.sources.SplinedSourceWindow(z=z, W=dndz, source_type='lensing') - ] - theory_cls = camb.get_results(pars).get_source_cls_dict(lmax=lmax, raw_cl=True) - return theory_cls['W1xW1'] - - def get_n_gal_map(self, params, nside, cat_gal): - """ - Compute the galaxy number density map. - """ - ra = cat_gal[params['ra_col']] - dec = cat_gal[params['dec_col']] - w = cat_gal[params['w_col']] - - theta = (90. - dec) * np.pi / 180. - phi = ra * np.pi / 180. - pix = hp.ang2pix(nside, theta, phi) - - unique_pix, idx, idx_rep = np.unique(pix, return_index=True, return_inverse=True) - n_gal = np.zeros(hp.nside2npix(nside)) - n_gal[unique_pix] = np.bincount(idx_rep, weights=w) - return n_gal, unique_pix, idx, idx_rep - - def get_gaussian_real(self, params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep): - - e1_rot, e2_rot = self.apply_random_rotation(cat_gal[params['e1_col']], cat_gal[params['e2_col']]) - noise_map_e1 = np.zeros(hp.nside2npix(nside)) - noise_map_e2 = np.zeros(hp.nside2npix(nside)) - - w = cat_gal[params['w_col']] - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) - noise_map_e1[mask] /= n_gal[mask] - noise_map_e2[mask] /= n_gal[mask] - - return noise_map_e1 + 1j*noise_map_e2 - - def get_sample(self, params, nside, lmax, b, cat_gal, n_gal, mask, unique_pix, idx_rep): - noise_map = self.get_gaussian_real(params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep) - - f = nmt.NmtField(mask=mask, maps=[noise_map.real, noise_map.imag], lmax=lmax) - - wsp = nmt.NmtWorkspace.from_fields(f, f, b) - - cl_noise = nmt.compute_coupled_cell(f, f) - cl_noise = wsp.decouple_cell(cl_noise) - - return cl_noise, f, wsp - - def get_pseudo_cls(self, map, wsp=None): - """ - Compute the pseudo-cl for a given map. - """ - - lmin = 8 - lmax = 2*self.nside - b_lmax = lmax - 1 - - if self.binning == 'linear': - step = 10 - b = nmt.NmtBin.from_nside_linear(self.nside, step) - elif self.binning == 'powspace': - ells = np.arange(lmin, lmax+1) - - start = np.power(lmin, self.power) - end = np.power(lmax, self.power) - bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) - - #Get bandpowers - bpws = np.digitize(ells.astype(float), bins_ell) - 1 - bpws[0] = 0 - bpws[-1] = self.n_ell_bins-1 - - b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - - ell_eff = b.get_effective_ells() - - factor = -1 if self.pol_factor else 1 - - f_all = nmt.NmtField(mask=(map!=0), maps=[map.real, factor*map.imag], lmax=b_lmax) - - if wsp is None: - wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) - - cl_coupled = nmt.compute_coupled_cell(f_all, f_all) - cl_all = wsp.decouple_cell(cl_coupled) - - return ell_eff, cl_all, wsp - - def apply_random_rotation(self, e1, e2): - """ - Apply a random rotation to the ellipticity components e1 and e2. - - Parameters - ---------- - e1 : np.array - First component of the ellipticity. - e2 : np.array - Second component of the ellipticity. - - Returns - ------- - np.array - First component of the rotated ellipticity. - np.array - Second component of the rotated ellipticity. - """ - np.random.seed() - rot_angle = np.random.rand(len(e1))*2*np.pi - e1_out = e1*np.cos(rot_angle) + e2*np.sin(rot_angle) - e2_out = -e1*np.sin(rot_angle) + e2*np.cos(rot_angle) - return e1_out, e2_out - - def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): - """ - Save pseudo-Cl's to a FITS file. - - Parameters - ---------- - pseudo_cl : np.array - Pseudo-Cl's to save. - out_path : str - Path to save the pseudo-Cl's to. - """ - #Create columns of the fits file - col1 = fits.Column(name="ELL", format="D", array=ell_eff) - col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) - col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) - col4 = fits.Column(name="BB", format="D", array=pseudo_cl[3]) - coldefs = fits.ColDefs([col1, col2, col3, col4]) - cell_hdu = fits.BinTableHDU.from_columns(coldefs, name="PSEUDO_CELL") - - cell_hdu.writeto(out_path, overwrite=True) - - @property - def pseudo_cls(self): - if not hasattr(self, "_pseudo_cls"): - self.calculate_pseudo_cl() - self.calculate_pseudo_cl_eb_cov() - return self._pseudo_cls - - def plot_pseudo_cl(self): - """ - Plot pseudo-Cl's for given catalogs. - """ - self.print_cyan("Plotting pseudo-Cl's") - - #Plotting EE - out_path = os.path.abspath(f"{self.cc['paths']['output']}/cell_ee.png") - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data - ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"], capsize=2) - - ax[0].set_ylabel('$\ell C_\ell$') - - ax[0].set_xlim(ell.min()-10, ell.max()+100) - ax[0].set_xscale('squareroot') - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data - ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"]) - - ax[1].set_xlabel('$\ell$') - ax[1].set_ylabel('$C_\ell$') - - ax[1].set_xlim(ell.min()-10, ell.max()+100) - ax[1].set_xscale('squareroot') - ax[1].set_yscale('log') - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle('Pseudo-Cl EE (Gaussian covariance)') - plt.legend() - plt.savefig(out_path) - - #Plotting EB - out_path = os.path.abspath(f"{self.cc['paths']['output']}/cell_eb.png") - - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data - ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"], capsize=2) - - ax[0].axhline(0, color='black', linestyle='--') - ax[0].set_ylabel('$\ell C_\ell$') - - ax[0].set_xlim(ell.min()-10, ell.max()+100) - ax[0].set_xscale('squareroot') - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data - ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"]) - - ax[1].set_xlabel('$\ell$') - ax[1].set_ylabel('$C_\ell$') - - ax[1].set_xlim(ell.min()-10, ell.max()+100) - ax[1].set_xscale('squareroot') - ax[1].set_yscale('log') - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle('Pseudo-Cl EB (Gaussian covariance)') - plt.legend() - plt.savefig(out_path) - - #Plotting BB - out_path = os.path.abspath(f"{self.cc['paths']['output']}/cell_bb.png") - - fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) - - for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data - ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"], capsize=2) - - ax[0].axhline(0, color='black', linestyle='--') - ax[0].set_ylabel('$\ell C_\ell$') - - ax[0].set_xlim(ell.min()-10, ell.max()+100) - ax[0].set_xscale('squareroot') - ax[0].set_xticks(np.array([100, 400, 900, 1600])) - ax[0].minorticks_on() - ax[0].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] - ax[0].xaxis.set_ticks(minor_ticks, minor=True) - - for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data - ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"]) - - ax[1].set_xlabel('$\ell$') - ax[1].set_ylabel('$C_\ell$') - - ax[1].set_xlim(ell.min()-10, ell.max()+100) - ax[1].set_xscale('squareroot') - ax[1].set_yscale('log') - ax[1].set_xticks(np.array([100, 400, 900, 1600])) - ax[1].minorticks_on() - ax[1].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] - ax[1].xaxis.set_ticks(minor_ticks, minor=True) - - plt.suptitle('Pseudo-Cl BB (Gaussian covariance)') - plt.legend() - plt.savefig(out_path) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index d1f276b7..1a45dca8 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,193 +1,63 @@ # %% -import copy import os -from pathlib import Path -import configparser +from contextlib import contextmanager +from functools import partial import colorama -import healpy as hp import matplotlib.pyplot as plt -import matplotlib.ticker as mticker import numpy as np import pymaster as nmt +import healpy as hp +import treecorr +from . import utils_cosmo_val +import camb +import re + +import yaml +from astropy.io import fits +from astropy import units as u +from astropy.coordinates import SkyCoord + +import healpy as hp +import healsparse as hsp import treecorr +import camb import yaml +from collections import Counter + +import skyproj + from astropy.io import fits +from astropy.cosmology import Planck18 +from astropy import units as u +from astropy.coordinates import SkyCoord + +try: + from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes +except: + print("Could not import cosmo_numba, continuing without") + +from . import utils_cosmo_val +import pymaster as nmt from cs_util import plots as cs_plots -from shear_psf_leakage import leakage, run_object, run_scale +from shear_psf_leakage import leakage from shear_psf_leakage import plots as psfleak_plots +from shear_psf_leakage import run_object, run_scale from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat +import matplotlib.scale as mscale -from .cosmology import get_cosmo, get_theo_c_ell -from .plots import FootprintPlotter -from .rho_tau import ( - get_params_rho_tau, - get_rho_tau_w_cov, - get_samples, -) +mscale.register_scale(utils_cosmo_val.SquareRootScale) # %% class CosmologyValidation: - """Framework for cosmic shear validation and systematics analysis. - - Handles two-point correlation function measurements, PSF systematics (rho/tau), - pseudo-C_ell analysis, and covariance estimation for weak lensing surveys. - Supports multiple catalog versions with automatic leakage-corrected variants. - - Parameters - ---------- - versions : list of str - Catalog version identifiers to analyze. Appending '_leak_corr' to a base - version creates a virtual catalog using leakage-corrected ellipticity columns - (e1_col_corrected/e2_col_corrected) from the base version configuration. - catalog_config : str, default './cat_config.yaml' - Path to catalog configuration YAML defining survey metadata, file paths, - and analysis settings for each version. - output_dir : str, optional - Override for output directory. If None, uses catalog config's paths.output. - rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' - Fitting method for PSF leakage systematics parameters. - cov_estimate_method : {'th', 'jk'}, default 'th' - Covariance estimation: 'th' for semi-analytic theory, 'jk' for jackknife. - compute_cov_rho : bool, default True - Whether to compute covariance for rho statistics during PSF analysis. - n_cov : int, default 100 - Number of realizations for covariance estimation when using theory method. - theta_min : float, default 0.1 - Minimum angular separation in arcminutes for correlation function binning. - theta_max : float, default 250 - Maximum angular separation in arcminutes for correlation function binning. - nbins : int, default 20 - Number of angular bins for TreeCorr real-space correlation functions. - var_method : {'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'}, default 'jackknife' - TreeCorr variance estimation method. - npatch : int, default 20 - Number of spatial patches for jackknife variance estimation. - quantile : float, default 0.1587 - Quantile for uncertainty bands in plots (default: 1-sigma ≈ 0.159). - theta_min_plot : float, default 0.08 - Minimum angular scale for plotting (may differ from analysis cut). - theta_max_plot : float, default 250 - Maximum angular scale for plotting. - ylim_alpha : list of float, default [-0.005, 0.05] - Y-axis limits for alpha systematic parameter plots. - ylim_xi_sys_ratio : list of float, default [-0.02, 0.5] - Y-axis limits for xi systematics ratio plots. - nside : int, default 1024 - HEALPix resolution for pseudo-C_ell analysis and area computation. - binning : {'powspace', 'linspace', 'logspace'}, default 'powspace' - Ell binning scheme for pseudo-C_ell (powspace = ell^power spacing). - power : float, default 0.5 - Exponent for power-law binning when binning='powspace'. - n_ell_bins : int, default 32 - Number of ell bins for pseudo-C_ell analysis (used with binning='powspace'). - ell_step : int, default 10 - Bin width in ell for linear binning (used with binning='linear'). - pol_factor : bool, default True - Apply polarization correction factor in pseudo-C_ell calculations. - nrandom_cell : int, default 10 - Number of random realizations for C_ell error estimation. - cosmo_params : dict, optional - Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. - - Attributes - ---------- - versions : list of str - Validated catalog versions after processing _leak_corr variants. - cc : dict - Loaded catalog configuration with resolved absolute paths. - catalog_config_path : Path - Resolved path to the catalog configuration file. - treecorr_config : dict - Configuration dictionary passed to TreeCorr correlation objects. - cosmo : pyccl.Cosmology - Cosmology object for theory predictions. - - Notes - ----- - - Path resolution: Relative paths in catalog config are resolved using each - version's 'subdir' field as the base directory. - - Virtual _leak_corr versions: These create deep copies of the base version - config, swapping e1_col/e2_col with e1_col_corrected/e2_col_corrected. - - TreeCorr cross_patch_weight: Automatically set to 'match' for jackknife, - 'simple' otherwise, following TreeCorr best practices. - """ - - @staticmethod - def _split_seed_variant(version): - """Return the base version and seed label if version encodes a seed.""" - if "_seed" not in version: - return None, None - base, seed_label = version.rsplit("_seed", 1) - if not base or not seed_label.isdigit(): - return None, None - return base, seed_label - - @staticmethod - def _materialize_seed_path( - base_cfg, seed_label, version, base_version, catalog_config - ): - """Render the seed-specific shear path using Python string formatting.""" - shear_cfg = base_cfg["shear"] - template = shear_cfg.get("path_template") - - try: - seed_value = int(seed_label) - except ValueError as error: - raise ValueError( - f"Seed suffix for '{version}' is not numeric; cannot materialize path." - ) from error - - format_context = {"seed": seed_value, "seed_label": seed_label} - - if template: - try: - return template.format(**format_context) - except KeyError as error: - raise KeyError( - f"Missing placeholder '{error.args[0]}' in path_template for " - f"'{base_version}' while materializing '{version}'. Update " - f"{catalog_config}." - ) from error - except ValueError as error: - raise ValueError( - f"Invalid format specification in path_template for '{base_version}' " - f"while materializing '{version}'." - ) from error - - path = shear_cfg.get("path", "") - token_start = path.rfind("seed") - if token_start == -1: - raise ValueError( - f"Cannot materialize '{version}': '{base_version}' lacks a shear " - f"path_template and its shear path '{path}' does not contain a 'seed' " - f"token. Update {catalog_config}." - ) - cursor = token_start + 4 # len("seed") - if cursor < len(path) and not path[cursor].isdigit(): - cursor += 1 - digit_start = cursor - while cursor < len(path) and path[cursor].isdigit(): - cursor += 1 - digit_end = cursor - digits = path[digit_start:digit_end] - if not digits: - raise ValueError( - f"Cannot materialize '{version}': shear path '{path}' for base version " - f"'{base_version}' lacks digits after the seed token. Update " - f"{catalog_config}." - ) - - template = f"{path[:digit_start]}{{seed_label}}{path[digit_end:]}" - return template.format(**format_context) def __init__( self, versions, + data_base_dir, catalog_config="./cat_config.yaml", - output_dir=None, rho_tau_method="lsq", cov_estimate_method="th", compute_cov_rho=True, @@ -203,17 +73,15 @@ def __init__( ylim_alpha=[-0.005, 0.05], ylim_xi_sys_ratio=[-0.02, 0.5], nside=1024, - nside_mask = 2**12, - binning="powspace", - power=1 / 2, + binning='powspace', + power=1/2, n_ell_bins=32, - ell_step=10, pol_factor=True, - cell_method='map', - nrandom_cell=10, - path_onecovariance=None, - cosmo_params=None, + nrandom_cell=10 ): + + self.versions = versions + self.data_base_dir = data_base_dir self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method self.compute_cov_rho = compute_cov_rho @@ -227,28 +95,13 @@ def __init__( self.theta_max_plot = theta_max_plot self.ylim_alpha = ylim_alpha self.ylim_xi_sys_ratio = ylim_xi_sys_ratio - # For pseudo-Cls + #For pseudo-Cls self.nside = nside self.binning = binning self.power = power self.n_ell_bins = n_ell_bins - self.ell_step = ell_step self.pol_factor = pol_factor self.nrandom_cell = nrandom_cell - self.cell_method = cell_method - self.nside_mask = nside_mask - self.path_onecovariance = path_onecovariance - - assert self.cell_method in ["map", "catalog"], "cell_method must be 'map' or 'catalog'" - - # For theory calculations: - # Create cosmology object using new functionality - if cosmo_params is not None: - self.cosmo = get_cosmo(**cosmo_params) - else: - # Use Planck 2018 defaults - self.cosmo = get_cosmo() - self.treecorr_config = { "ra_units": "degrees", @@ -258,234 +111,22 @@ def __init__( "sep_units": "arcmin", "nbins": nbins, "var_method": var_method, - "cross_patch_weight": "match" if var_method == "jackknife" else "simple", } - self.catalog_config_path = Path(catalog_config) - with self.catalog_config_path.open("r") as file: + with open(catalog_config, "r") as file: self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) - def resolve_paths_for_version(ver): - """Resolve relative paths for a version using its subdir.""" - subdir = Path(cc[ver]["subdir"]) + for ver in ["nz", *versions]: + + if ver not in cc: + raise KeyError(f"Version string {ver} not found in config file{catalog_config}") + version_base = f"{data_base_dir}/{cc[ver]['subdir']}" for key in cc[ver]: if "path" in cc[ver][key]: - path = Path(cc[ver][key]["path"]) - cc[ver][key]["path"] = ( - str(path) if path.is_absolute() else str(subdir / path) - ) - - resolve_paths_for_version("nz") - processed = {"nz"} - final_versions = [] - leak_suffix = "_leak_corr" - - def ensure_version_exists(ver): - if ver in processed: - return - - if ver in cc: - resolve_paths_for_version(ver) - processed.add(ver) - return - - seed_base, seed_label = self._split_seed_variant(ver) - - if ver.endswith(leak_suffix): - base_ver = ver[: -len(leak_suffix)] - ensure_version_exists(base_ver) - shear_cfg = cc[base_ver]["shear"] - if "e1_col_corrected" not in shear_cfg or "e2_col_corrected" not in shear_cfg: - raise ValueError( - f"{base_ver} does not have e1_col_corrected/e2_col_corrected " - f"fields; cannot create {ver}" - ) - if ver not in cc: - cc[ver] = copy.deepcopy(cc[base_ver]) - cc[ver]["shear"]["e1_col"] = shear_cfg["e1_col_corrected"] - cc[ver]["shear"]["e2_col"] = shear_cfg["e2_col_corrected"] - resolve_paths_for_version(ver) - processed.add(ver) - return - - if seed_base is not None: - ensure_version_exists(seed_base) - if ver not in cc: - cc[ver] = copy.deepcopy(cc[seed_base]) - seed_path = self._materialize_seed_path( - cc[seed_base], - seed_label, - ver, - seed_base, - catalog_config, - ) - cc[ver]["shear"]["path"] = seed_path - resolve_paths_for_version(ver) - processed.add(ver) - return - - raise KeyError( - f"Version string {ver} not found in config file {catalog_config}" - ) - - for ver in versions: - ensure_version_exists(ver) - final_versions.append(ver) - - self.versions = final_versions - - # Override output directory if provided - if output_dir is not None: - cc["paths"]["output"] = output_dir - - os.makedirs(cc["paths"]["output"], exist_ok=True) - - def get_redshift(self, version): - """Load redshift distribution for a catalog version. - - Parameters - ---------- - version : str - Catalog version identifier - - Returns - ------- - z : ndarray - Redshift values - nz : ndarray - n(z) probability density - """ - redshift_path = self.cc[version]["shear"]["redshift_path"] - return np.loadtxt(redshift_path, unpack=True) - - def compute_survey_stats( - self, - ver, - weights_key_override=None, - mask_path=None, - nside=None, - overwrite_config=False, - ): - """Compute effective survey statistics for a catalog version. - - Parameters - ---------- - ver : str - Version string registered in the catalog config. - weights_key_override : str, optional - Override the weight column key (defaults to the configured `w_col`). - mask_path : str, optional - Explicit mask path to use when measuring survey area. - nside : int, optional - If provided, compute survey area from the catalog using this NSIDE when no - mask path is available. - overwrite_config : bool, optional - If True, persist the derived statistics back to the catalog configuration. - - Returns - ------- - dict - Dictionary containing: - - area_deg2: Survey area in square degrees. - - n_eff: Effective number density per arcmin^2. - - sigma_e: Per-component shape noise. - - sum_w: Sum of weights. - - sum_w2: Sum of squared weights. - - catalog_size: Number of galaxies processed. - """ - if ver not in self.cc: - raise KeyError(f"Version {ver} not found in catalog configuration") - - shear_cfg = self.cc[ver]["shear"] - cov_th = self.cc[ver].get("cov_th", {}) - - if "path" not in shear_cfg: - raise KeyError(f"No shear catalog path defined for version {ver}") + cc[ver][key]["path"] = f"{version_base}/{cc[ver][key]['path']}" - catalog_path = shear_cfg["path"] - if not os.path.exists(catalog_path): - raise FileNotFoundError(f"Shear catalog not found: {catalog_path}") - - data = fits.getdata(catalog_path, memmap=True) - n_rows = len(data) - - e1 = np.asarray(data[shear_cfg["e1_col"]], dtype=float) - e2 = np.asarray(data[shear_cfg["e2_col"]], dtype=float) - - weight_column = weights_key_override or shear_cfg["w_col"] - if weight_column not in data.columns.names: - raise KeyError(f"Weight column '{weight_column}' missing in {catalog_path}") - - w = np.asarray(data[weight_column], dtype=float) - - sum_w = float(np.sum(w)) - sum_w2 = float(np.sum(w**2)) - sum_w2_e2 = float(np.sum((w**2) * (e1**2 + e2**2))) - - if mask_path is not None: - if not os.path.exists(mask_path): - raise FileNotFoundError(f"Mask path not found: {mask_path}") - mask_candidate = mask_path - else: - mask_candidate = self.cc[ver].get("mask") - if isinstance(mask_candidate, str) and not os.path.isabs(mask_candidate): - mask_candidate = str(Path(self.cc[ver]["subdir"]) / mask_candidate) - if mask_candidate is not None and not os.path.exists(mask_candidate): - mask_candidate = None - - area_deg2 = None - if mask_candidate is not None and os.path.exists(mask_candidate): - area_deg2 = self._area_from_mask(mask_candidate) - elif cov_th.get("A") is not None: - area_deg2 = float(cov_th["A"]) - elif nside is not None: - area_deg2 = self._area_from_catalog(catalog_path, nside) - else: - raise ValueError( - f"Unable to determine survey area for {ver}. Provide mask_path or nside." - ) - - area_arcmin2 = area_deg2 * 3600.0 - - n_eff = (sum_w**2) / (area_arcmin2 * sum_w2) if sum_w2 > 0 else 0.0 - sigma_e = np.sqrt(sum_w2_e2 / sum_w2) if sum_w2 > 0 else 0.0 - - results = { - "area_deg2": area_deg2, - "n_eff": n_eff, - "sigma_e": sigma_e, - "sum_w": sum_w, - "sum_w2": sum_w2, - "catalog_size": n_rows, - } - - if overwrite_config: - if "cov_th" not in self.cc[ver]: - self.cc[ver]["cov_th"] = {} - self.cc[ver]["cov_th"]["A"] = float(area_deg2) - self.cc[ver]["cov_th"]["n_e"] = float(n_eff) - self.cc[ver]["cov_th"]["sigma_e"] = float(sigma_e) - self._write_catalog_config() - - return results - - def _area_from_catalog(self, catalog_path, nside): - data = fits.getdata(catalog_path, memmap=True) - ra = np.asarray(data["RA"], dtype=float) - dec = np.asarray(data["Dec"], dtype=float) - theta = np.radians(90.0 - dec) - phi = np.radians(ra) - pix = hp.ang2pix(nside, theta, phi, lonlat=False) - unique_pix = np.unique(pix) - return float(unique_pix.size * hp.nside2pixarea(nside, degrees=True)) - - def _area_from_mask(self, mask_map_path): - mask = hp.read_map(mask_map_path, dtype=np.float64) - return float(mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True)) - - def _write_catalog_config(self): - with self.catalog_config_path.open("w") as file: - yaml.dump(self.cc, file, sort_keys=False) + if not os.path.exists(cc["paths"]["output"]): + os.mkdir(cc["paths"]["output"]) def color_reset(self): print(colorama.Fore.BLACK, end="") @@ -522,7 +163,7 @@ def set_params_leakage_scale(self, ver): params_in["dndz_path"] = ( f"{self.cc['nz']['dndz']['path']}_{self.cc[ver]['pipeline']}_{self.cc['nz']['dndz']['blind']}.txt" ) - params_in["output_dir"] = f"{self.cc['paths']['output']}/leakage_{ver}" + params_in["output_dir"] = f'{self.cc["paths"]["output"]}/leakage_{ver}' # Note: for SP these are calibrated shear estimates params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] @@ -549,7 +190,7 @@ def set_params_leakage_object(self, ver): # Set parameters params_in["input_path_shear"] = self.cc[ver]["shear"]["path"] - params_in["output_dir"] = f"{self.cc['paths']['output']}/leakage_{ver}" + params_in["output_dir"] = f'{self.cc["paths"]["output"]}/leakage_{ver}' # Note: for SP these are calibrated shear estimates params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] @@ -568,13 +209,18 @@ def set_params_leakage_object(self, ver): + f" shear yaml entry for version {ver}" ) + if "cols" in self.cc[ver]["shear"]: + params_in["cols"] = self.cc[ver]["shear"]["cols"] + params_in["verbose"] = False return params_in def init_results(self, objectwise=False): + results = {} for ver in self.versions: + # Set parameters depending on the type of leakage if objectwise: results[ver] = run_object.LeakageObject() @@ -586,6 +232,20 @@ def init_results(self, objectwise=False): results[ver].check_params() results[ver].prepare_output() + @contextmanager + def temporarily_load_data(results_instance): + try: + self.print_start(f"Loading catalog {results_instance._params['input_path_shear']} ...", end="") + results_instance.read_data() + self.print_done(f"done") + yield + finally: + self.print_done(f"Freeing {results_instance._params['input_path_shear']} from memory") + del results_instance.dat_shear + del results_instance.dat_PSF + + results[ver].temporarily_load_data = partial(temporarily_load_data, results[ver]) + return results @property @@ -600,34 +260,21 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise - def basename(self, version, treecorr_config=None, npatch=None): - cfg = treecorr_config or self.treecorr_config - patches = npatch or self.npatch - return ( - f"{version}_minsep={cfg['min_sep']}" - f"_maxsep={cfg['max_sep']}" - f"_nbins={cfg['nbins']}" - f"_npatch={patches}" - ) - - def calculate_rho_tau_stats(self): + out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) self.print_start("Rho stats") for ver in self.versions: - base = self.basename(ver) - rho_stat_handler, tau_stat_handler = get_rho_tau_w_cov( + rho_stat_handler, tau_stat_handler = utils_cosmo_val.get_rho_tau_w_cov( self.cc, ver, self.treecorr_config, out_dir, - base, method=self.cov_estimate_method, cov_rho=self.compute_cov_rho, - npatch=self.npatch, ) self.print_done("Rho stats finished") @@ -649,107 +296,10 @@ def tau_stat_handler(self): @property def colors(self): return [self.cc[ver]["colour"] for ver in self.versions] - - @property - def area(self): - if not hasattr(self, "_area"): - self.calculate_area() - return self._area - - @property - def n_eff_gal(self): - if not hasattr(self, "_n_eff_gal"): - self.calculate_n_eff_gal() - return self._n_eff_gal - - @property - def ellipticity_dispersion(self): - if not hasattr(self, "_ellipticity_dispersion"): - self.calculate_ellipticity_dispersion() - return self._ellipticity_dispersion - - @property - def pseudo_cls(self): - if not hasattr(self, "_pseudo_cls"): - self.calculate_pseudo_cl() - self.calculate_pseudo_cl_eb_cov() - return self._pseudo_cls - - @property - def pseudo_cls_onecov(self): - if not hasattr(self, "_pseudo_cls_onecov"): - self.calculate_pseudo_cl_onecovariance() - return self._pseudo_cls_onecov - - def calculate_area(self): - self.print_start("Calculating area") - area = {} - for ver in self.versions: - self.print_magenta(ver) - - if not hasattr(self.cc[ver]['shear'], 'mask'): - print("Mask not found in config file, calculating area from binned catalog") - area[ver] = self.calculate_area_from_binned_catalog(ver) - else: - mask = hp.read_map(self.cc[ver]['shear']['mask'], verbose=False) - nside_mask = hp.get_nside(mask) - print(f"nside_mask = {nside_mask}") - area[ver] = np.sum(mask) * hp.nside2pixarea(nside_mask, degrees=True) - print(f"Area = {area[ver]:.2f} deg^2") - - self._area = area - self.print_done("Area calculation finished") - - def calculate_area_from_binned_catalog(self, ver): - print(f"nside_mask = {self.nside_mask}") - with self.results[ver].temporarily_read_data(): - ra = self.results[ver].dat_shear["RA"] - dec = self.results[ver].dat_shear["Dec"] - hsp_map = hp.ang2pix( - self.nside_mask, - np.radians(90 - dec), - np.radians(ra), - lonlat=False, - ) - mask = np.bincount(hsp_map, minlength=hp.nside2npix(self.nside_mask)) > 0 - - area = np.sum(mask) * hp.nside2pixarea(self.nside_mask, degrees=True) - print(f"Area = {area:.2f} deg^2") - - return area - - def calculate_n_eff_gal(self): - self.print_start("Calculating effective number of galaxy") - n_eff_gal = {} - for ver in self.versions: - self.print_magenta(ver) - with self.results[ver].temporarily_read_data(): - w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] - n_eff_gal[ver] = 1/(self.area[ver]*60*60)* np.sum(w)**2/np.sum(w**2) - print(f"n_eff_gal = {n_eff_gal[ver]:.2f} gal./arcmin^-2") - - self._n_eff_gal = n_eff_gal - self.print_done("Effective number of galaxy calculation finished") - - def calculate_ellipticity_dispersion(self): - self.print_start("Calculating ellipticity dispersion") - ellipticity_dispersion = {} - for ver in self.versions: - self.print_magenta(ver) - with self.results[ver].temporarily_read_data(): - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] - ellipticity_dispersion[ver] = np.sqrt( - 0.5*(np.average(e1**2, weights=w**2) + np.average(e2**2, weights=w**2)) - ) - print(f"Ellipticity dispersion = {ellipticity_dispersion[ver]:.4f}") - self._ellipticity_dispersion = ellipticity_dispersion def plot_rho_stats(self, abs=False): - filenames = [ - f"rho_stats_{self.basename(ver)}.fits" for ver in self.versions - ] + + filenames = [f"rho_stats_{ver}.fits" for ver in self.versions] savefig = "rho_stats.png" self.rho_stat_handler.plot_rho_stats( @@ -759,19 +309,16 @@ def plot_rho_stats(self, abs=False): savefig=savefig, legend="outside", abs=abs, - show=True, - close=True, ) + plt.close() self.print_done( - "Rho stats plot saved to " + f"Rho stats plot saved to " + f"{os.path.abspath(self.rho_stat_handler.catalogs._output)}/{savefig}", ) def plot_tau_stats(self, plot_tau_m=False): - filenames = [ - f"tau_stats_{self.basename(ver)}.fits" for ver in self.versions - ] + filenames = [f"tau_stats_{ver}.fits" for ver in self.versions] savefig = "tau_stats.png" self.tau_stat_handler.plot_tau_stats( @@ -781,17 +328,15 @@ def plot_tau_stats(self, plot_tau_m=False): savefig=savefig, legend="outside", plot_tau_m=plot_tau_m, - show=True, - close=True, ) + plt.close() self.print_done( - "Tau stats plot saved to " + f"Tau stats plot saved to " + f"{os.path.abspath(self.tau_stat_handler.catalogs._output)}/{savefig}", ) def set_params_rho_tau(self, params, params_psf, survey="other"): - params = {**params} if survey in ("DES", "SP_axel_v0.0", "SP_axel_v0.0_repr"): params["patch_number"] = 120 print("DES, jackknife patch number = 120") @@ -804,8 +349,8 @@ def set_params_rho_tau(self, params, params_psf, survey="other"): else: params["patch_number"] = 150 - params["ra_PSF_col"] = params_psf["ra_col"] - params["dec_PSF_col"] = params_psf["dec_col"] + params["ra_col"] = params_psf["ra_col"] + params["dec_col"] = params_psf["dec_col"] params["e1_PSF_col"] = params_psf["e1_PSF_col"] params["e2_PSF_col"] = params_psf["e2_PSF_col"] params["e1_star_col"] = params_psf["e1_star_col"] @@ -818,8 +363,6 @@ def set_params_rho_tau(self, params, params_psf, survey="other"): params["ra_units"] = "deg" params["dec_units"] = "deg" - params["w_col"] = self.cc[survey]["shear"]["w_col"] - return params @property @@ -851,12 +394,9 @@ def calculate_rho_tau_fits(self): self.cov_estimate_method, None ) - base = self.basename(ver) - - flat_samples, result, q = get_samples( + flat_samples, result, q = utils_cosmo_val.get_samples( self.psf_fitter, ver, - base, cov_type=self.cov_estimate_method, apply_debias=npatch, sampler=self.rho_tau_method, @@ -866,7 +406,7 @@ def calculate_rho_tau_fits(self): self.rho_tau_fits["result_list"].append(result) self.rho_tau_fits["q_list"].append(q) - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") + self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] xi_psf_sys_samples = np.array([]).reshape(0, nbins) @@ -899,9 +439,8 @@ def plot_rho_tau_fits(self): legend_loc="upper right", contour_colors=self.colors, markers={"x0": 0, "x1": 1, "x2": 1}, - show=True, - close=True, ) + plt.close() self.print_done(f"Tau contours plot saved to {os.path.abspath(savefig)}") plt.figure(figsize=(15, 6)) @@ -911,17 +450,16 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") + self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") for i in range(100): self.psf_fitter.plot_xi_psf_sys( flat_sample[-i + 1], ver, color, alpha=0.1 ) self.psf_fitter.plot_xi_psf_sys(mcmc_result[1], ver, color) plt.legend() - out_path = os.path.abspath(f"{out_dir}/xi_psf_sys_samples.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_psf_sys samples plot saved to {out_path}") + savefig = os.path.abspath(f"{out_dir}/xi_psf_sys_samples.png") + cs_plots.savefig(savefig) + self.print_done(f"xi_psf_sys samples plot saved to {savefig}") plt.figure(figsize=(15, 6)) for mcmc_result, ver, color, flat_sample in zip( @@ -930,6 +468,7 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): + ls = self.cc[ver]["ls"] theta = self.psf_fitter.rho_stat_handler.rho_stats["theta"] xi_psf_sys = self.xi_psf_sys[ver] @@ -951,23 +490,23 @@ def plot_rho_tau_fits(self): plt.ylabel(r"$\xi^{\rm PSF}_{\rm sys}$") plt.title(f"{1 - self.quantile:.1%}, {self.quantile:.1%} quantiles") plt.legend() - out_path = os.path.abspath(f"{out_dir}/xi_psf_sys_quantiles.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_psf_sys quantiles plot saved to {out_path}") + plt.show() + savefig = os.path.abspath(f"{out_dir}/xi_psf_sys_quantiles.png") + cs_plots.savefig(savefig) + self.print_done(f"xi_psf_sys quantiles plot saved to {savefig}") for mcmc_result, ver, flat_sample in zip( self.rho_tau_fits["result_list"], self.versions, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") + self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") for yscale in ("linear", "log"): out_path = os.path.abspath( f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" ) self.psf_fitter.plot_xi_psf_sys_terms( - ver, mcmc_result[1], out_path, yscale=yscale, show=True + ver, mcmc_result[1], out_path, yscale=yscale ) self.print_done( f"{yscale}-scale xi_psf_sys terms plot saved to {out_path}" @@ -983,17 +522,20 @@ def plot_footprints(self): self.print_start("Plotting footprints:") for ver in self.versions: self.print_magenta(ver) - + results = self.results[ver] + fp = FootprintPlotter() - - for region in fp._regions: + + for region in fp._regions: out_path = os.path.abspath( f"{self.cc['paths']['output']}/footprint_{ver}_{region}.png" ) if os.path.exists(out_path): - self.print_done(f"Skipping footprint plot, {out_path} exists") + self.print_done( + f"Skipping footprint plot, {out_path} exists" + ) else: - with self.results[ver].temporarily_read_data(): + with self.results[ver].temporarily_load_data(): hsp_map = fp.create_hsp_map( self.results[ver].dat_shear["RA"], self.results[ver].dat_shear["Dec"], @@ -1008,15 +550,15 @@ def calculate_scale_dependent_leakage(self): results = self.results[ver] output_base_path = os.path.abspath( - f"{self.cc['paths']['output']}/leakage_{ver}/xi_for_leak_scale" + f'{self.cc["paths"]["output"]}/leakage_{ver}/xi_for_leak_scale' ) output_path_ab = f"{output_base_path}_a_b.txt" output_path_aa = f"{output_base_path}_a_a.txt" - with self.results[ver].temporarily_read_data(): + + with self.results[ver].temporarily_load_data(): if os.path.exists(output_path_ab) and os.path.exists(output_path_aa): self.print_green( - f"Skipping computation, reading {output_path_ab} and " - f"{output_path_aa} instead" + f"Skipping computation, reading {output_path_ab} and {output_path_aa} instead" ) results.r_corr_gp = treecorr.GGCorrelation(self.treecorr_config) @@ -1056,6 +598,7 @@ def plot_scale_dependent_leakage(self): markers.append(self.cc[ver]["marker"]) if len(theta) > 0: + # Log x out_path = os.path.abspath( f"{self.cc['paths']['output']}/alpha_leak_log.png" @@ -1080,8 +623,7 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, shift_x=True, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"Log-scale alpha leakage plot saved to {out_path}") # Lin x @@ -1108,8 +650,7 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, shift_x=False, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"Lin-scale alpha leakage plot saved to {out_path}") # Plot xi_sys @@ -1131,6 +672,7 @@ def plot_scale_dependent_leakage(self): ylabel = r"$\xi^{\rm sys}_+(\theta)$" title = "Cross-correlation leakage" out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_p.png") + fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) cs_plots.plot_data_1d( theta, y, @@ -1146,8 +688,7 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, # shift_x=True, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"xi_sys_plus plot saved to {out_path}") y = [] @@ -1162,6 +703,7 @@ def plot_scale_dependent_leakage(self): ylabel = r"$\xi^{\rm sys}_-(\theta)$" title = "Cross-correlation leakage" out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_m.png") + fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) cs_plots.plot_data_1d( theta, y, @@ -1178,8 +720,7 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, # shift_x=True, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"xi_sys_minus plot saved to {out_path}") def calculate_objectwise_leakage(self): @@ -1197,31 +738,25 @@ def calculate_objectwise_leakage(self): results_obj.update_params() results_obj.prepare_output() - # Skip read_data() and copy catalogue from scale leakage instance instead - # results_obj._dat = self.results[ver].dat_shear + with self.results[ver].temporarily_load_data(): + results_obj._dat = self.results[ver].dat_shear - out_base = results_obj.get_out_base(mix, order) - out_path = f"{out_base}.pkl" - if os.path.exists(out_path): - self.print_green( - f"Skipping object-wise leakage, file {out_path} exists" - ) - results_obj.par_best_fit = leakage.read_from_file(out_path) - else: - self.print_cyan("Computing object-wise leakage regression") + out_base = results_obj.get_out_base(mix, order) + out_path = f"{out_base}.pkl" + if os.path.exists(out_path): + self.print_green( + f"Skipping object-wise leakage, file {out_path} exists" + ) + results_obj.par_best_fit = leakage.read_from_file(out_path) + else: + self.print_cyan("Computing object-wise leakage regression") - # Run - with results_obj.temporarily_read_data(): - try: + # Run results_obj.PSF_leakage() - except KeyError as e: - print(f"{e}\nExpected key is missing from catalog.") - # remove the results object for this version - self.results_objectwise.pop(ver) # Gather coefficients leakage_coeff = {} - for ver in self.results_objectwise: + for ver in self.versions: leakage_coeff[ver] = {} results = self.results[ver] results_obj = self.results_objectwise[ver] @@ -1259,12 +794,12 @@ def plot_objectwise_leakage(self): self.calculate_objectwise_leakage() self.print_start("Plotting object-wise leakage:") - cs_plots.figure(figsize=(15, 15)) + fig = cs_plots.figure(figsize=(15, 15)) linestyles = ["-", "--", ":"] fillstyles = ["full", "none", "left", "right", "bottom", "top"] - for ver in self.results_objectwise: + for ver in self.versions: label = ver for key, ls, fs in zip( ["alpha_mean", "alpha_1", "alpha_0"], linestyles, fillstyles @@ -1299,10 +834,34 @@ def plot_objectwise_leakage(self): out_path = os.path.abspath( f"{self.cc['paths']['output']}/leakage_coefficients.png" ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"Object-wise leakage coefficients plot saved to {out_path}") + def calculate_objectwise_leakage_aux(self): + + self.print_start("Object-wise leakage auxiliary quantities:") + for ver in self.versions: + self.print_magenta(ver) + + results_obj = self.results_objectwise[ver] + results_obj.check_params() + results_obj.update_params() + results_obj.prepare_output() + + with self.results[ver].temporarily_load_data(): + results_obj._dat = self.results[ver].dat_shear + + if not "cols" in results_obj._params: + self.print_green("Skipping object-wise leakage (aux quantities), no input columns for regression found") + else: + self.print_cyan(f"Computing object-wise leakage regression with aux quantities: {results_obj._params['cols']}") + + # Run + results_obj.obs_leakage() + + def plot_objectwise_leakage_aux(self): + self.calculate_objectwise_leakage_aux() + def plot_ellipticity(self, nbins=200): out_path = os.path.abspath(f"{self.cc['paths']['output']}/ell_hist.png") if os.path.exists(out_path): @@ -1311,22 +870,17 @@ def plot_ellipticity(self, nbins=200): self.print_start("Computing ellipticity histograms:") fig, axs = plt.subplots(1, 2, figsize=(22, 7)) - bins = np.linspace(-1.5, 1.5, nbins + 1) for ver in self.versions: - self.print_magenta(ver) - R = self.cc[ver]["shear"]["R"] - with self.results[ver].temporarily_read_data(): - e1 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R - ) - e2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R - ) - w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] + with self.results[ver].temporarily_load_data(): + self.print_magenta(ver) + R = self.cc[ver]["shear"]["R"] + e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R + e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R + w = self.results[ver].dat_shear["w_col"] axs[0].hist( e1, - bins=bins, + bins=nbins, density=False, histtype="step", weights=w, @@ -1335,38 +889,6 @@ def plot_ellipticity(self, nbins=200): ) axs[1].hist( e2, - bins=bins, - density=False, - histtype="step", - weights=w, - label=ver, - color=self.cc[ver]["colour"], - ) - - for idx in (0, 1): - axs[idx].set_xlabel(f"$e_{idx}$") - axs[idx].set_ylabel("frequency") - axs[idx].legend() - axs[idx].set_xlim([-1.5, 1.5]) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done("Ellipticity histograms saved to " + out_path) - - def plot_weights(self, nbins=200): - out_path = os.path.abspath(f"{self.cc['paths']['output']}/weight_hist.png") - if os.path.exists(out_path): - self.print_done(f"Skipping weight histograms, {out_path} exists") - else: - self.print_start("Computing weight histograms:") - - fig, ax = plt.subplots(1, 1, figsize=(10, 7)) - for ver in self.versions: - self.print_magenta(ver) - with self.results[ver].temporarily_read_data(): - w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] - - plt.hist( - w, bins=nbins, density=False, histtype="step", @@ -1375,21 +897,19 @@ def plot_weights(self, nbins=200): color=self.cc[ver]["colour"], ) - plt.xlabel("$w$") - plt.ylabel("frequency") - plt.yscale("log") - plt.legend() - # plt.xlim([-0.01, 1.2]) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done("Weight histograms saved to " + out_path) + for idx in (0, 1): + axs[idx].set_xlabel(f"$e_{idx}$") + axs[idx].set_ylabel("frequency") + axs[idx].legend() + axs[idx].set_xlim([-1.5, 1.5]) + cs_plots.savefig(out_path) + self.print_done("Ellipticity histograms saved to " + out_path) def plot_separation(self, nbins=200): self.print_start("Separation histograms") if "SP_matched_MP_v1.0" in self.versions: fig, axs = plt.subplots(1, 1, figsize=(10, 7)) - with self.results["SP_matched_MP_v1.0"].temporarily_read_data(): - sep = self.results["SP_matched_MP_v1.0"].dat_shear["Separation"] + sep = self.results["SP_matched_MP_v1.0"].dat_shear["Separation"] axs.hist( sep, bins=nbins, @@ -1410,218 +930,128 @@ def calculate_additive_bias(self): self._c2 = {} for ver in self.versions: self.print_magenta(ver) - R = self.cc[ver]["shear"]["R"] - e1_col, e2_col, w_col = [ - self.cc[ver]["shear"][k] for k in ["e1_col", "e2_col", "w_col"] - ] - with self.results[ver].temporarily_read_data(): + with self.results[ver].temporarily_load_data(): + R = self.cc[ver]["shear"]["R"] self._c1[ver] = np.average( - self.results[ver].dat_shear[e1_col] / R, - weights=self.results[ver].dat_shear[w_col], + self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R, + weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], ) self._c2[ver] = np.average( - self.results[ver].dat_shear[e2_col] / R, - weights=self.results[ver].dat_shear[w_col], + self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R, + weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], ) - self.print_done("Finished additive bias calculation.") + self.print_done("Finished additive bias calculation.") @property def c1(self): if not hasattr(self, "_c1"): + print("MKDEBUG calc c1") self.calculate_additive_bias() return self._c1 @property def c2(self): if not hasattr(self, "_c2"): + print("MKDEBUG calc c2") self.calculate_additive_bias() return self._c2 - def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): - """ - Calculate the two-point correlation function (2PCF) ξ± for a given catalog - version with TreeCorr. - - By default the class instance's `npatch` and `treecorr_config` entries are - used to - initialize the TreeCorr Catalog and GGCorrelation objects, but may be - overridden - by passing keyword arguments. - - Parameters: - ver (str): The catalog version to process. - - npatch (int, optional): The number of patches to use for the calculation. - Defaults to the instance's `npatch` attribute. - - save_fits (bool, optional): Whether to save the ξ± results to FITS files. - Defaults to False. - - **treecorr_config: Additional TreeCorr configuration parameters that will - override the instance's default `treecorr_config`. For example, `min_sep=1`. - - Returns: - treecorr.GGCorrelation: The TreeCorr GGCorrelation object containing the - computed 2PCF results. - - Notes: - - If the output file for the given configuration already exists, the - calculation is skipped, and the results are loaded from the file. - - If a patch file for the given configuration does not exist, it is - created during the process. - - FITS files for ξ+ and ξ− are saved with additional metadata in their - headers if `save_fits` is True. - """ - - self.print_magenta(f"Computing {ver} ξ±") - - npatch = npatch or self.npatch - treecorr_config = { - **self.treecorr_config, - **treecorr_config, - "var_method": "jackknife" if int(npatch) > 1 else "shot", - } + def calculate_2pcf(self): + self.print_start(f"Computing 2PCF") - gg = treecorr.GGCorrelation(treecorr_config) + self._cat_ggs = {} + for ver in self.versions: + self.print_magenta(ver) + gg = self._cat_ggs[ver] = treecorr.GGCorrelation(self.treecorr_config) - # If the output file already exists, skip the calculation - out_fname = os.path.abspath( - f"{self.cc['paths']['output']}/{ver}_xi_minsep={treecorr_config['min_sep']}_maxsep={treecorr_config['max_sep']}_nbins={treecorr_config['nbins']}_npatch={npatch}.txt" - ) + out_fname = os.path.abspath(f"{self.cc['paths']['output']}/xi_pm_{ver}.txt") + if os.path.exists(out_fname): + self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") + gg.read(out_fname) + else: + # Run TreeCorr + with self.results[ver].temporarily_load_data(): + e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] + e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + if ver != "DES": + R = self.cc[ver]["shear"]["R"] + g1 = (e1 - self.c1[ver]) / R + g2 = (e2 - self.c2[ver]) / R + else: + R11 = self.cc[ver]["shear"]["R11"] + R22 = self.cc[ver]["shear"]["R22"] + g1 = (e1 - self.c1[ver]) / np.average( + self.results[ver].dat_shear[R11] + ) + g2 = ( + self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + - self.c2[ver] + ) / np.average(self.results[ver].dat_shear[R22]) + cat_gal = treecorr.Catalog( + ra=self.results[ver].dat_shear["RA"], + dec=self.results[ver].dat_shear["Dec"], + g1=g1, + g2=g2, + w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], + ra_units=self.treecorr_config["ra_units"], + dec_units=self.treecorr_config["dec_units"], + npatch=self.npatch, + ) + gg.process(cat_gal) + gg.write(out_fname) - if os.path.exists(out_fname): - self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") - gg.read(out_fname) + # Save xi_p and xi_m results to fits file + lst = np.arange(1, self.treecorr_config["nbins"] + 1) - else: - # Load data and create a catalog - with self.results[ver].temporarily_read_data(): - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] - if ver != "DES": - R = self.cc[ver]["shear"]["R"] - g1 = (e1 - self.c1[ver]) / R - g2 = (e2 - self.c2[ver]) / R - else: - R11 = self.cc[ver]["shear"]["R11"] - R22 = self.cc[ver]["shear"]["R22"] - g1 = (e1 - self.c1[ver]) / np.average( - self.results[ver].dat_shear[R11] + col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) + col3 = fits.Column(name="ANGBIN", format="K", array=lst) + col4 = fits.Column(name="VALUE", format="D", array=gg.xip) + col5 = fits.Column( + name="ANG", format="D", unit="arcmin", array=gg.meanr ) - g2 = (e2 - self.c2[ver]) / np.average( - self.results[ver].dat_shear[R22] + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") + + col4 = fits.Column(name="VALUE", format="D", array=gg.xim) + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") + + # append xi_plus header info + xiplus_dict = { + "2PTDATA": "T", + "QUANT1": "G+R", + "QUANT2": "G+R", + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + } + for key in xiplus_dict: + xiplus_hdu.header[key] = xiplus_dict[key] + xiplus_hdu.writeto( + f"{self.cc['paths']['output']}/xi_plus_{ver}.fits", overwrite=True ) - # Use patch file if it exists - patch_file = os.path.abspath( - f"{self.cc['paths']['output']}/{ver}_patches_npatch={npatch}.dat" - ) - - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=w, - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=npatch, - patch_centers=patch_file if os.path.exists(patch_file) else None, - ) - - # If no patch file exists, save the current patches - if not os.path.exists(patch_file): - cat_gal.write_patch_centers(patch_file) - - # Process the catalog & write the correlation functions - gg.process(cat_gal) - gg.write(out_fname, write_patch_results=True, write_cov=True) - - # Save xi_p and xi_m results to fits file - # (moved outside so it runs even if txt exists) - if save_fits: - lst = np.arange(1, treecorr_config["nbins"] + 1) - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column( - name="ANG", format="D", unit="arcmin", array=gg.rnom - ) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - # Use same naming format as txt output - fits_base = out_fname.replace(".txt", "").replace("_xi_", "_") - xiplus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_plus_{ver}')}.fits", - overwrite=True, - ) - - # append xi_minus header info - ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} - for key in ximinus_dict: - ximinus_hdu.header[key] = ximinus_dict[key] - ximinus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_minus_{ver}')}.fits", - overwrite=True, - ) - - # Add correlation object to class - if not hasattr(self, "cat_ggs"): - self.cat_ggs = {} - self.cat_ggs[ver] = gg + # append xi_minus header info + ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} + for key in ximinus_dict: + ximinus_hdu.header[key] = ximinus_dict[key] + ximinus_hdu.writeto( + f"{self.cc['paths']['output']}/xi_minus_{ver}.fits", overwrite=True + ) - self.print_done("Done 2PCF") + self.print_done("Done 2PCF") - return gg + @property + def cat_ggs(self): + if not hasattr(self, "_cat_ggs"): + self.calculate_2pcf() + return self._cat_ggs def plot_2pcf(self): # Plot of n_pairs fig, ax = plt.subplots(ncols=1, nrows=1) for ver in self.versions: - self.calculate_2pcf(ver) plt.plot( self.cat_ggs[ver].meanr, self.cat_ggs[ver].npairs, @@ -1633,8 +1063,7 @@ def plot_2pcf(self): plt.ylabel(r"$n_{\rm pair}$") plt.legend() out_path = os.path.abspath(f"{self.cc['paths']['output']}/n_pair.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"n_pair plot saved to {out_path}") # Plot of xi_+ @@ -1650,14 +1079,13 @@ def plot_2pcf(self): ) plt.xscale("log") plt.yscale("log") - plt.legend() + plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\xi_+(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_p.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"xi_plus plot saved to {out_path}") # Plot of xi_- @@ -1673,14 +1101,13 @@ def plot_2pcf(self): ) plt.xscale("log") plt.yscale("log") - plt.legend() + plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\xi_-(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_m.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"xi_minus plot saved to {out_path}") # Plot of xi_+(theta) * theta @@ -1695,14 +1122,14 @@ def plot_2pcf(self): color=self.cc[ver]["colour"], ) plt.xscale("log") - plt.legend() + plt.yscale("log") + plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\theta \xi_+(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_p_theta.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"xi_plus_theta plot saved to {out_path}") # Plot of xi_- * theta @@ -1717,121 +1144,57 @@ def plot_2pcf(self): color=self.cc[ver]["colour"], ) plt.xscale("log") - plt.legend() + plt.yscale("log") + plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\theta \xi_-(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_m_theta.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"xi_minus_theta plot saved to {out_path}") # Plot of xi_+ with and without xi_psf_sys - # but skip if xi_psf_sys is not calculated since that takes forever - if hasattr(self, "_xi_psf_sys"): - for idx, ver in enumerate(self.versions): - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxim), - label=r"$\xi_+$", - ls="solid", - color="green", - ) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt(self.xi_psf_sys[ver]["var"]), - label=r"$\xi^{\rm psf}_{+, {\rm sys}}$", - ls="dotted", - color="red", - ) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip + self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt( - self.cat_ggs[ver].varxip + self.xi_psf_sys[ver]["var"] - ), - label=r"$\xi_+ + \xi^{\rm psf}_{+, {\rm sys}}$", - ls="dashdot", - color="magenta", - ) - - plt.xscale("log") - plt.yscale("log") - plt.legend() - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylim(1e-8, 5e-4) - plt.ylabel(r"$\xi_+(\theta)$") - out_path = os.path.abspath( - f"{self.cc['paths']['output']}/xi_p_xi_psf_sys_{ver}.png" - ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") - - def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): - - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(10, 7)) - - for idx, ver in enumerate(self.versions): - xi_psf_sys = self.xi_psf_sys[ver] - gg = self.cat_ggs[ver] - - ratio = xi_psf_sys["mean"] / gg.xip - ratio_err = np.sqrt( - (np.sqrt(xi_psf_sys["var"]) / gg.xip) ** 2 - + (xi_psf_sys["mean"] * np.sqrt(gg.varxip) / gg.xip**2) ** 2 + fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) + plt.errorbar( + self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), + self.cat_ggs[ver].xip, + yerr=np.sqrt(self.cat_ggs[ver].varxim), + label=r"$\xi_+$", + ls="solid", + color="green", ) - - theta = gg.meanr - jittered_theta = theta * (1+idx*offset) - plt.errorbar( - jittered_theta, - ratio, - yerr=ratio_err, - label=ver, - ls=self.cc[ver]["ls"], - color=self.cc[ver]["colour"], - fmt=self.cc[ver].get("marker", None), - capsize=5 + self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), + self.xi_psf_sys[ver]["mean"], + yerr=np.sqrt(self.xi_psf_sys[ver]["var"]), + label=r"$\xi^{\rm psf}_{+, {\rm sys}}$", + ls="dotted", + color="red", + ) + plt.errorbar( + self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), + self.cat_ggs[ver].xip + self.xi_psf_sys[ver]["mean"], + yerr=np.sqrt(self.cat_ggs[ver].varxip + self.xi_psf_sys[ver]["var"]), + label=r"$\xi_+ + \xi^{\rm psf}_{+, {\rm sys}}$", + ls="dashdot", + color="magenta", ) - plt.fill_between( - [self.theta_min_plot, self.theta_max_plot], - - threshold, - + threshold, - color="black", - alpha=0.1, - label=f"{threshold:.0%} threshold", - ) - plt.plot( - [self.theta_min_plot, self.theta_max_plot], - [threshold, threshold], - ls="dashed", - color="black") - plt.plot( - [self.theta_min_plot, self.theta_max_plot], - [-threshold, -threshold], - ls="dashed", - color="black") - plt.xscale("log") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.ylabel(r"$\xi^{\rm psf}_{+, {\rm sys}} / \xi_+$") - plt.gca().yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1)) - plt.legend() - plt.title("Ratio of PSF systematics to cosmic shear signal") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/ratio_xi_sys_xi.png") - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() - print(f"Ratio of xi_psf_sys to xi plot saved to {out_path}") - + plt.xscale("log") + plt.yscale("log") + plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") + plt.ticklabel_format(axis="y") + plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") + plt.xlim([self.theta_min_plot, self.theta_max_plot]) + plt.ylim(1e-8, 5e-4) + plt.ylabel(r"$\xi_+(\theta)$") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/xi_p_xi_psf_sys_{ver}.png" + ) + cs_plots.savefig(out_path) + self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") def calculate_aperture_mass_dispersion( self, theta_min=0.3, theta_max=200, nbins=500, nbins_map=15, npatch=25 @@ -1861,7 +1224,7 @@ def calculate_aperture_mass_dispersion( self.print_green(f"Skipping xi for Map2, {out_fname} exists") gg.read(out_fname) else: - with self.results[ver].temporarily_read_data(): + with self.results[ver].temporarily_load_data(): R = self.cc[ver]["shear"]["R"] g1 = ( self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] @@ -1952,8 +1315,7 @@ def plot_aperture_mass_dispersion(self): linestyles=linestyles, shift_x=True, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"linear-scale {mode} plot saved to {out_path}") for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: @@ -1985,601 +1347,395 @@ def plot_aperture_mass_dispersion(self): linestyles=linestyles, shift_x=True, ) - cs_plots.savefig(out_path, close_fig=False) - cs_plots.show() + cs_plots.savefig(out_path) self.print_done(f"log-scale {mode} plot saved to {out_path}") def calculate_pure_eb( self, - version, - min_sep=None, - max_sep=None, - nbins=None, - min_sep_int=0.08, - max_sep_int=300, - nbins_int=100, - npatch=256, - var_method="jackknife", - cov_path_int=None, - cosmo_cov=None, - n_samples=1000, + theta_min=0.1, + theta_max=250, + nbins=20, + theta_min_int=0.04, + theta_max_int=500, + nbins_int=1000, ): - """ - Calculate the pure E/B modes for the given catalog version. - The class instance's treecorr_config will be used for the "reporting" binning - by default, but any kwargs passed to this function will overwrite the defaults. - - Parameters - ---------- - version : str - The catalog version to compute the pure E/B modes for. - min_sep : float, optional - Minimum separation for the reporting binning. Defaults to the value in - self.treecorr_config if not provided. - max_sep : float, optional - Maximum separation for the reporting binning. Defaults to the value in - self.treecorr_config if not provided. - nbins : int, optional - Number of bins for the reporting binning. Defaults to the value in - self.treecorr_config if not provided. - min_sep_int : float, optional - Minimum separation for the integration binning. Defaults to 0.08. - max_sep_int : float, optional - Maximum separation for the integration binning. Defaults to 300. - nbins_int : int, optional - Number of bins for the integration binning. Defaults to 100. - npatch : int, optional - Number of patches for the jackknife or bootstrap resampling. Defaults to - the value in self.npatch if not provided. - var_method : str, optional - Variance estimation method. Defaults to "jackknife". - cov_path_int : str, optional - Path to the covariance matrix for the reporting binning. Replaces the - treecorr covariance matrix if provided, meaning that var_method has no - effect on the results although it is still passed to - CosmologyValidation.calculate_2pcf. - cosmo_cov : pyccl.Cosmology, optional - Cosmology object to use for theoretical xi+/xi- predictions in the - semi-analytical covariance calculation. Defaults to self.cosmo if not - provided. - n_samples : int, optional - Number of Monte Carlo samples for semi-analytical covariance propagation. - Defaults to 1000. - - Returns - ------- - dict - A dictionary containing the following keys: - - "xip_E": Pure E-mode correlation function for xi+. - - "xim_E": Pure E-mode correlation function for xi-. - - "xip_B": Pure B-mode correlation function for xi+. - - "xim_B": Pure B-mode correlation function for xi-. - - "xip_amb": Ambiguity mode for xi+. - - "xim_amb": Ambiguity mode for xi-. - - "cov": Covariance matrix for the pure E/B modes. - - "gg": The two-point correlation function object for the reporting binning. - - "gg_int": The two-point correlation function object for the - integration binning. - - "eb_samples": (only when using semi-analytical covariance) Semi-analytic - EB samples used for covariance calculation. Shape: (n_samples, 6*nbins) - - Notes - ----- - - A shared patch file is used for the reporting and integration binning, - and is created if it does not exist. - """ - from .b_modes import calculate_pure_eb_correlation - - self.print_start(f"Computing {version} pure E/B") + self.print_start("Computing pure E/B") - # Set up parameters with defaults - npatch = npatch or self.npatch - min_sep = min_sep or self.treecorr_config["min_sep"] - max_sep = max_sep or self.treecorr_config["max_sep"] - nbins = nbins or self.treecorr_config["nbins"] - - # Create TreeCorr configurations treecorr_config = { **self.treecorr_config, - "min_sep": min_sep, - "max_sep": max_sep, + "min_sep": theta_min, + "max_sep": theta_max, "nbins": nbins, } + # nbins_int = (nbins - 1) * integration_oversample + 1 treecorr_config_int = { **treecorr_config, - "min_sep": min_sep_int, - "max_sep": max_sep_int, + "min_sep": theta_min_int, + "max_sep": theta_max_int, "nbins": nbins_int, } - # Calculate correlation functions - gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) - gg_int = self.calculate_2pcf(version, npatch=npatch, **treecorr_config_int) - - # Get redshift distribution if using analytic covariance - if cov_path_int is not None: - z, nz = self.get_redshift(version) - z_dist = np.column_stack([z, nz]) - else: - z_dist = None - - # Delegate to b_modes module - results = calculate_pure_eb_correlation( - gg=gg, - gg_int=gg_int, - var_method=var_method, - cov_path_int=cov_path_int, - cosmo_cov=cosmo_cov, - n_samples=n_samples, - z_dist=z_dist - ) - - return results - - def plot_pure_eb( - self, - versions=None, - output_dir=None, - fiducial_xip_scale_cut=None, - fiducial_xim_scale_cut=None, - min_sep=None, - max_sep=None, - nbins=None, - min_sep_int=0.08, - max_sep_int=300, - nbins_int=100, - npatch=None, - var_method="jackknife", - cov_path_int=None, - cosmo_cov=None, - n_samples=1000, - results=None, - **kwargs - ): - """ - Generate comprehensive pure E/B mode analysis plots. - - Creates four types of plots for each version: - 1. Integration vs Reporting comparison - 2. E/B/Ambiguous correlation functions - 3. 2D PTE heatmaps - 4. Covariance matrix visualization + print("Integration correlation function parameters:") + print(treecorr_config_int) - Parameters - ---------- - versions : list, optional - List of catalog versions to process. Uses self.versions if None. - output_dir : str, optional - Output directory for plots. Uses configured output path if None. - fiducial_xip_scale_cut : tuple, optional - (min_scale, max_scale) for xi+ fiducial analysis, shown as gray regions - fiducial_xim_scale_cut : tuple, optional - (min_scale, max_scale) for xi- fiducial analysis, shown as gray regions - min_sep, max_sep, nbins : float, float, int, optional - Binning parameters for reporting scale. Uses treecorr_config if None. - min_sep_int, max_sep_int, nbins_int : float, float, int - Binning parameters for integration scale - (default: 0.08-300 arcmin, 100 bins) - npatch : int, optional - Number of patches for jackknife covariance. Uses self.npatch if None. - var_method : str - Variance method ("jackknife" or "semi-analytic"). - Automatically set to "semi-analytic" when cov_path_int is provided. - cov_path_int : str, optional - Path to integration covariance matrix for semi-analytical calculation - cosmo_cov : pyccl.Cosmology, optional - Cosmology for theoretical predictions in semi-analytical covariance - n_samples : int - Number of Monte Carlo samples for semi-analytical covariance (default: 1000) - results : dict or list, optional - Precalculated results to avoid recomputation. Can be a single results dict - for one version, or a list of results dicts for multiple versions. - If None (default), results will be calculated using calculate_pure_eb. - **kwargs : dict - Additional arguments passed to calculate_eb_statistics - - Notes - ----- - This function orchestrates the full E/B mode analysis workflow: - - Uses instance configuration as defaults for unspecified parameters - - Automatically switches to analytical variance when theoretical - covariance provided - - Generates standardized output file naming based on all analysis - parameters - - Delegates individual plot generation to specialized functions in - b_modes module - """ - from .b_modes import ( - calculate_eb_statistics, - plot_eb_covariance_matrix, - plot_integration_vs_reporting, - plot_pte_2d_heatmaps, - plot_pure_eb_correlations, - ) + print("Output correlation function parameters:") + print(treecorr_config) - # Use instance defaults for unspecified parameters - versions = versions or self.versions - output_dir = output_dir or self.cc['paths']['output'] - npatch = npatch or self.npatch - - # Override var_method to analytic when cov_path_int is provided - if cov_path_int is not None: - var_method = "semi-analytic" - - # Use treecorr_config defaults for reporting scale binning - min_sep = min_sep or self.treecorr_config["min_sep"] - max_sep = max_sep or self.treecorr_config["max_sep"] - nbins = nbins or self.treecorr_config["nbins"] - - # Handle results parameter - convert to list format for consistent processing - if results is not None: - if isinstance(results, dict): - # Single results dict provided - should match single version - if len(versions) != 1: - raise ValueError( - "Single results dict provided but multiple versions specified. " - "Provide results list matching versions length." - ) - results_list = [results] - elif isinstance(results, list): - # List of results provided - if len(results) != len(versions): - raise ValueError( - f"Results list length ({len(results)}) does not match versions " - f"length ({len(versions)})" - ) - results_list = results - else: - raise TypeError("Results must be dict, list, or None") - else: - results_list = [None] * len(versions) - - for idx, version in enumerate(versions): - # Generate standardized output filename stub - out_stub = ( - f"{output_dir}/{version}_eb_minsep={min_sep}_" - f"maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_" - f"maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_" - f"varmethod={var_method}" - ) + for ver in self.versions: + self.print_magenta(ver) - # Get or calculate results for this version - version_results = results_list[idx] or self.calculate_pure_eb( - version, - min_sep=min_sep, - max_sep=max_sep, - nbins=nbins, - min_sep_int=min_sep_int, - max_sep_int=max_sep_int, - nbins_int=nbins_int, - npatch=npatch, - var_method=var_method, - cov_path_int=cov_path_int, - cosmo_cov=cosmo_cov, - n_samples=n_samples, - ) + gg_int = treecorr.GGCorrelation(treecorr_config_int) + gg = treecorr.GGCorrelation(treecorr_config) - # Calculate E/B statistics for all bin combinations (only if not provided) - version_results = calculate_eb_statistics( - version_results, - cov_path_int=cov_path_int, - n_samples=n_samples, - **kwargs + out_fname = os.path.abspath( + # f"{self.cc['paths']['output']}/xi_for_pure_eb_{ver}.txt" + f"{self.cc['paths']['output']}/xi_for_pure_eb_thetamin={theta_min}_thetamax={theta_max}_nbins={nbins}_npatch={self.npatch}_{ver}.txt" ) - - # Generate all plots using specialized plotting functions - gg, gg_int = version_results["gg"], version_results["gg_int"] - - # Integration vs Reporting comparison plot - plot_integration_vs_reporting( - gg, gg_int, - out_stub + "_integration_vs_reporting.png", - version + out_fname_int = os.path.abspath( + f"{self.cc['paths']['output']}/xi_for_pure_eb_int_thetaminint={theta_min_int}_thetamaxint_{theta_max_int}_nbinsint={nbins_int}_npatchint={self.npatch}_{ver}.txt" + # f"{self.cc['paths']['output']}/xi_for_pure_eb_{ver}_int.txt" ) + if os.path.exists(out_fname) and os.path.exists(out_fname_int): + self.print_green( + f"Skipping xi for COSEBIs:\n{out_fname}\n{out_fname_int}\nexist." + ) + gg.read(out_fname) + gg_int.read(out_fname_int) + else: + with self.results[ver].temporarily_load_data(): + R = self.cc[ver]["shear"]["R"] + g1 = ( + self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] + - self.c1[ver] + ) / R + g2 = ( + self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + - self.c2[ver] + ) / R - # E/B/Ambiguous correlation functions plot - plot_pure_eb_correlations( - version_results, - out_stub + "_xis.png", - version, - fiducial_xip_scale_cut=fiducial_xip_scale_cut, - fiducial_xim_scale_cut=fiducial_xim_scale_cut - ) + cat = treecorr.Catalog( + ra=self.results[ver].dat_shear["RA"], + dec=self.results[ver].dat_shear["Dec"], + g1=g1, + g2=g2, + w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], + ra_units=self.treecorr_config["ra_units"], + dec_units=self.treecorr_config["dec_units"], + npatch=self.npatch, + ) + del g1, g2, R + + self.print_cyan("Integration binning") + gg_int.process(cat) + gg_int.write(out_fname_int, write_patch_results=True, write_cov=True) + + self.print_cyan("Output binning") + gg.process(cat) + gg.write(out_fname, write_patch_results=True, write_cov=True) + + def pure_EB(corrs): + gg, gg_int = corrs + return get_pure_EB_modes( + theta=gg.meanr, + xip=gg.xip, + xim=gg.xim, + theta_int=gg_int.meanr, + xip_int=gg_int.xip, + xim_int=gg_int.xim, + tmin=theta_min, + tmax=theta_max, + ) - # 2D PTE heatmaps plot - plot_pte_2d_heatmaps( - version_results, - version, - out_stub + "_ptes.png", - fiducial_xip_scale_cut=fiducial_xip_scale_cut, - fiducial_xim_scale_cut=fiducial_xim_scale_cut + xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb = pure_EB([gg, gg_int]) + cov = treecorr.estimate_multi_cov( + [gg, gg_int], "jackknife", func=lambda x: np.hstack(pure_EB(x)) ) - # Covariance matrix plot - plot_eb_covariance_matrix( - version_results["cov"], - var_method, - out_stub + "_covariance.png", - version - ) + results = { + "xip_E": xip_E, + "xim_E": xim_E, + "xip_B": xip_B, + "xim_B": xim_B, + "xip_amb": xip_amb, + "xim_amb": xim_amb, + "cov": cov, + } - def calculate_cosebis( - self, - version, - min_sep_int=0.5, - max_sep_int=500, - nbins_int=1000, - npatch=None, - nmodes=10, - cov_path=None, - evaluate_all_scale_cuts=False, - min_sep=None, - max_sep=None, - nbins=None, - ): - """ - Calculate COSEBIs from a finely-binned correlation function. + return results - COSEBIs fundamentally require fine binning for accurate transformations. - This function computes a single, finely-binned correlation function using - integration binning parameters and can evaluate either a single scale cut - (full range) or multiple scale cuts systematically. +class FootprintPlotter: + """Class to create footprint plots. + + Parameters + ----------- + nside_coverage: int, optional + basic resolution of map; default is 32 + nside_map: + fine resolution for plotting; default is 2048 + """ + + # Dictionary storing region parameters + _regions = { + "NGC": {"ra_0": 180, "extend": [120, 270, 20, 70], "vmax": 60}, + "SGC": {"ra_0": 15, "extend": [-20, 45, 20, 45], "vmax": 60}, + "fullsky": {"ra_0": 150, "extend": [0, 360, -90, 90], "vmax": 60}, + } + + def __init__(self, nside_coverage=32, nside_map=2048): + + self._nside_coverage = nside_coverage + self._nside_map = nside_map + + def create_hsp_map(self, ra, dec): + """Create Hsp Map. + + Create healsparse map. + Parameters ---------- - version : str - The catalog version to compute the COSEBIs for. - min_sep_int : float, optional - Minimum separation for integration binning (fine binning for COSEBIs). - Defaults to 0.5 arcmin. - max_sep_int : float, optional - Maximum separation for integration binning (fine binning for COSEBIs). - Defaults to 500 arcmin. - nbins_int : int, optional - Number of bins for integration binning (fine binning for COSEBIs). - Defaults to 1000. - npatch : int, optional - Number of patches for the jackknife resampling. Defaults to self.npatch. - nmodes : int, optional - Number of COSEBIs modes to compute. Defaults to 10. - cov_path : str, optional - Path to theoretical covariance matrix. When provided, enables analytic - covariance calculation. - evaluate_all_scale_cuts : bool, optional - If True, evaluates COSEBIs for all possible scale cut combinations - using the reporting binning parameters. If False, uses the full - integration range as a single scale cut. Defaults to False. - min_sep : float, optional - Minimum separation for reporting binning (only used when - evaluate_all_scale_cuts=True). Defaults to self.treecorr_config["min_sep"]. - max_sep : float, optional - Maximum separation for reporting binning (only used when - evaluate_all_scale_cuts=True). Defaults to self.treecorr_config["max_sep"]. - nbins : int, optional - Number of bins for reporting binning (only used when - evaluate_all_scale_cuts=True). Defaults to self.treecorr_config["nbins"]. - + ra : numpy.ndarray + right ascension values + dec : numpy.ndarray + declination values + Returns ------- - dict - When evaluate_all_scale_cuts=False: Dictionary containing COSEBIs results - with E/B modes, covariances, and statistics for the full range. - When evaluate_all_scale_cuts=True: Dictionary with scale cut tuples as - keys and results dictionaries as values, containing results for all - possible scale cut combinations. - - Notes - ----- + hsp.HealSparseMap + map + """ - from .b_modes import calculate_cosebis + # Create empty map + hsp_map = hsp.HealSparseMap.make_empty( + self._nside_coverage, + self._nside_map, + dtype=np.float32, + sentinel=np.nan + ) - self.print_start(f"Computing {version} COSEBIs") + # Get pixel list corresponding to coordinates + hpix = hp.ang2pix(self._nside_map, ra, dec, nest=True, lonlat=True) - # Set up parameters with defaults - npatch = npatch or self.npatch + # Get count of objects per pixel + pixel_counts = Counter(hpix) - # Always use integration binning for COSEBIs calculation (fine binning) - treecorr_config = { - **self.treecorr_config, - "min_sep": min_sep_int, - "max_sep": max_sep_int, - "nbins": nbins_int, - } - - # Calculate single fine-binned correlation function for COSEBIs - print( - f"Computing fine-binned 2PCF with {nbins_int} bins from {min_sep_int} to " - f"{max_sep_int} arcmin" - ) - gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) - - if evaluate_all_scale_cuts: - # Use reporting binning parameters or inherit from class config - min_sep = min_sep or self.treecorr_config["min_sep"] - max_sep = max_sep or self.treecorr_config["max_sep"] - nbins = nbins or self.treecorr_config["nbins"] - - # Generate scale cuts using np.geomspace (no TreeCorr needed) - bin_edges = np.geomspace(min_sep, max_sep, nbins + 1) - scale_cuts = [ - (bin_edges[start], bin_edges[stop]) - for start in range(nbins) - for stop in range(start+1, nbins+1) - ] - - print(f"Evaluating {len(scale_cuts)} scale cut combinations") - - # Call b_modes function with scale cuts list - results = calculate_cosebis( - gg=gg, nmodes=nmodes, scale_cuts=scale_cuts, cov_path=cov_path - ) - else: - # Single scale cut behavior: use full range - results = calculate_cosebis( - gg=gg, nmodes=nmodes, scale_cuts=None, cov_path=cov_path - ) - # Extract single results dict from scale_cuts dictionary - results = list(results.values())[0] + # List of unique pixels + unique_hpix = np.array(list(pixel_counts.keys())) - return results + # Number of objects + values = np.array(list(pixel_counts.values()), dtype=np.float32) - def plot_cosebis( + # Create maps with numbers per pixel + hsp_map[unique_hpix] = values + + return hsp_map + + def plot_area( self, - version=None, - output_dir=None, - min_sep_int=0.5, max_sep_int=500, nbins_int=1000, # Integration binning - npatch=None, nmodes=10, cov_path=None, - evaluate_all_scale_cuts=False, # New parameter - min_sep=None, max_sep=None, nbins=None, # Reporting binning - fiducial_scale_cut=None, # For plotting reference - results=None, + hsp_map, + ra_0=0, + extend=[120, 270, 29, 70], + vmax=60, + projection=None, + outpath=None, + title=None, ): - """ - Generate comprehensive COSEBIs analysis plots for a single version. - - Creates two types of plots: - 1. COSEBIs E/B mode correlation functions - 2. COSEBIs covariance matrix - + """Plot Area. + + Plot catalogue in an area on the sky. + Parameters ---------- - version : str, optional - Version string to process. Defaults to first version in self.versions. - output_dir : str, optional - Output directory for plots. Defaults to self.cc['paths']['output']. - min_sep_int, max_sep_int, nbins_int : float, float, int - Integration binning parameters for correlation function - (default: 0.5, 500, 1000) - npatch : int, optional - Number of patches for jackknife covariance. Defaults to instance value. - nmodes : int - Number of COSEBIs modes to compute (default: 10) - cov_path : str, optional - Path to theoretical covariance matrix. When provided, analytic - covariance is used. - evaluate_all_scale_cuts : bool - Whether to evaluate all scale cuts (default: False) - min_sep, max_sep, nbins : float, float, int, optional - Reporting binning parameters. Only used when evaluate_all_scale_cuts=True. - fiducial_scale_cut : tuple, optional - (min_scale, max_scale) reference scale cut for plotting when - evaluate_all_scale_cuts=True - results : dict, optional - Precalculated results to avoid recomputation. If None (default), - results will be calculated using calculate_cosebis. - - Notes - ----- - This function orchestrates the full COSEBIs analysis workflow: - - Uses instance configuration as defaults for unspecified parameters - - Calculates COSEBIs for the version using the updated parameter interface - - Generates mode plots and covariance visualization - - Output files are named with analysis parameters for reproducibility - - When evaluate_all_scale_cuts=True, results contain multiple scale cuts; - fiducial_scale_cut determines which one is used for plotting + hsp_map : hsp_HealSparseMap + input map + ra_0 : float, optional + anchor point in R.A.; default is 0 + extend : list, optional + sky region, extend=[ra_low, ra_high, dec_low, dec_high]; + default is [120, 270, 29, 70] + vmax : float, optional + maximum pixel value to plot with color; default is 60 + projection : skyproj.McBrydeSkyproj + if ``None`` (default), a new plot is created + outpath : str, optional + output path, default is ``None`` + title : str, optional + print title if not ``None`` (default) + + Returns + -------- + skyproj.McBrydeSkyproj + projection instance + plt.axes.Axes + axes instance + + Raises + ------ + ValueError + if no object found in region + """ - from .b_modes import ( - find_conservative_scale_cut_key, - plot_cosebis_covariance_matrix, - plot_cosebis_modes, - plot_cosebis_scale_cut_heatmap, - ) - - # Use instance defaults if not specified - version = version or self.versions[0] - output_dir = output_dir or self.cc['paths']['output'] - npatch = npatch or self.treecorr_config.get('npatch', 256) + if not projection: + + # Create new figure and axes + fig, ax = plt.subplots(figsize=(10, 10)) + + # Create new projection + projection = skyproj.McBrydeSkyproj( + ax=ax, + lon_0=ra_0, + extent=extend, + autorescale=True, + vmax=vmax + ) + else: + ax = None - # Determine variance method based on whether theoretical covariance is used - var_method = "analytic" if cov_path is not None else "jackknife" + try: + _ = projection.draw_hspmap( + hsp_map, lon_range=extend[0:2], + lat_range=extend[2:] + ) + except ValueError: + msg = "No object found in region to draw" + print(f"{msg}, continuing...") + #raise ValueError(msg) + + projection.draw_milky_way(width=25, linewidth=1.5, color='black', linestyle='-') + + if title: + plt.title(title, pad=5) - # Create output filename with integration parameters to match Snakemake - out_stub = ( - f"{output_dir}/{version}_cosebis_minsep={min_sep_int}_" - f"maxsep={max_sep_int}_nbins={nbins_int}_npatch={npatch}_" - f"varmethod={var_method}_nmodes={nmodes}" + if outpath: + plt.savefig(outpath) + + return projection, ax + + def plot_region(self, hsp_map, region, projection=None, outpath=None, title=None): + + return self.plot_area( + hsp_map, + region["ra_0"], + region["extend"], + region["vmax"], + projection=projection, + outpath=outpath, + title=title, ) - # Add scale cut info if provided - if fiducial_scale_cut is not None: - out_stub += f"_scalecut={fiducial_scale_cut[0]}-{fiducial_scale_cut[1]}" - - # if evaluate_all_scale_cuts: - # out_stub += f"_allcuts_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}" - - # Get or calculate results for this version - if results is None: - # Calculate COSEBIs using instance method - results = self.calculate_cosebis( - version, - min_sep_int=min_sep_int, - max_sep_int=max_sep_int, - nbins_int=nbins_int, - npatch=npatch, - nmodes=nmodes, - cov_path=cov_path, - evaluate_all_scale_cuts=evaluate_all_scale_cuts, - min_sep=min_sep, - max_sep=max_sep, - nbins=nbins, - ) + def plot_all_regions(self, hsp_map, outbase=None): - # Generate plots using specialized plotting functions - # Extract single result for plotting if multiple scale cuts were evaluated - if (isinstance(results, dict) and - all(isinstance(k, tuple) for k in results.keys())): - # Multiple scale cuts: use fiducial_scale_cut if provided, otherwise use - # full range - if fiducial_scale_cut is not None: - plot_results = results[ - find_conservative_scale_cut_key(results, fiducial_scale_cut) - ] + for region in self._regions: + if outbase: + outpath = f"{outbase}_{region}.png" else: - # Use full range result (largest scale cut) - max_range_key = max(results.keys(), key=lambda x: x[1] - x[0]) - plot_results = results[max_range_key] - else: - # Single result - plot_results = results - - plot_cosebis_modes( - plot_results, - version, - out_stub + "_cosebis.png", - fiducial_scale_cut=fiducial_scale_cut - ) + outpath = None + self.plot_region(hsp_map, self._regions[region], outpath=outpath) + - plot_cosebis_covariance_matrix( - plot_results, - version, - var_method, - out_stub + "_covariance.png" - ) + @classmethod + def hp_pixel_centers(cls, nside, nest=False): + + # Get number of pixels for given nside + npix = hp.nside2npix(nside) + + # Get pixel indices + pix_indices = np.arange(npix) + + # Get coordinates of pixel centers + ra, dec = hp.pix2ang(nside, pix_indices, nest=nest, lonlat=True) - # Generate scale cut heatmap if we have multiple scale cuts - if (isinstance(results, dict) and - all(isinstance(k, tuple) for k in results.keys()) and - len(results) > 1): - # Create temporary gg object with correct binning for mapping - treecorr_config_temp = { - **self.treecorr_config, - "min_sep": min_sep or self.treecorr_config["min_sep"], - "max_sep": max_sep or self.treecorr_config["max_sep"], - "nbins": nbins or self.treecorr_config["nbins"], - } - gg_temp = self.calculate_2pcf( - version, npatch=npatch, **treecorr_config_temp - ) + return ra, dec, npix - plot_cosebis_scale_cut_heatmap( - results, - gg_temp, - version, - out_stub + "_scalecut_ptes.png", - fiducial_scale_cut=fiducial_scale_cut + @classmethod + def plot_footprint_as_hp(cls, hsp_map, nside, outpath=None, title=None): + + ra, dec, npix = cls.hp_pixel_centers(nside) + + # Create an empty HEALPix map + m = np.full(npix, np.nan) + + fig, ax = plt.subplots(figsize=(10, 10)) + + # Plot the HEALPix grid + hp.mollview(m, title=title, coord="C", notext=True, rot=(180, 0, 0)) + + # Define the Galactic Plane: l = [0, 360], b = 0° + for l0, ls in zip((-5, 0, 5), (":", "-", ":")): + l_values = np.linspace(0, 360, 500) # 500 points along the plane + b_values = np.zeros_like(l_values) # Galactic latitude is 0 (the plane) + + # Convert (l, b) to (λ, β) - Ecliptic coordinates + coords = SkyCoord(l=l_values*u.degree, b=b_values*u.degree, frame='galactic') + ecl_coords = coords.transform_to('barycentrictrueecliptic') # Ecliptic frame + + # Extract Ecliptic longitude (λ) and latitude (β) + lambda_ecl = ecl_coords.lon.deg # Ecliptic longitude + beta_ecl = ecl_coords.lat.deg # Ecliptic latitude + + # Convert to HEALPix projection coordinates (colatitude, longitude) + theta = np.radians(90 - beta_ecl) # HEALPix uses colatitude + phi = np.radians(lambda_ecl) # HEALPix uses longitude + + # Create a healpy Mollweide projection in Ecliptic coordinates + hp.projplot(theta, phi, linestyle=ls, color='black', linewidth=1) # Plot the outline + + # Apply mask + mask_values = hsp_map.get_values_pos(ra, dec, valid_mask=True, lonlat=True) + + ok = np.where(mask_values == False)[0] + #nok = np.where(mask_values == False)[0] + + hp.projscatter(ra[ok], dec[ok], lonlat=True, color="green", s=1, marker=".") + #hp.projscatter(ra[nok], dec[nok], lonlat=True, color="red", s=1, marker=".") + + plt.tight_layout() + + if outpath: + plt.savefig(outpath) + + plt.show() + +def hsp_map_logical_or(maps, verbose=False): + """ + Hsp Map Logical Or. + + Logical AND of HealSparseMaps. + + """ + if verbose: + print("Combine all maps...") + + # Ensure consistency in coverage and data type + nside_coverage = maps[0].nside_coverage + nside_sparse = maps[0].nside_sparse + dtype = maps[0].dtype + + for m in maps: + # MKDEBUG TODO: Change nside if possible + if m.nside_coverage != nside_coverage: + raise ValueError( + f"Coverage nside={m.nside_coverage} does not match {nside_coverage}" ) + if m.dtype != dtype: + raise ValueError( + f"Data type {m.dtype} does not match {dtype}" + ) + + # Create an empty HealSparse map + map_comb = hsp.HealSparseMap.make_empty(nside_coverage, nside_sparse, dtype=dtype) + for idx, m in enumerate(maps): + map_comb |= m + + if verbose: + valid_pixels = map_comb.valid_pixels + n_tot = np.sum(valid_pixels) + n_true = np.count_nonzero(valid_pixels) + n_false = n_tot - n_true + print(f"after map {idx}: frac_true={n_true / n_tot:g}, frac_false={n_false / n_tot:g}") + return map_comb def calculate_pseudo_cl_eb_cov(self): """ @@ -2596,7 +1752,7 @@ def calculate_pseudo_cl_eb_cov(self): for ver in self.versions: self.print_magenta(ver) - if ver not in self._pseudo_cls.keys(): + if not ver in self._pseudo_cls.keys(): self._pseudo_cls[ver] = {} out_path = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_{ver}.fits") @@ -2604,32 +1760,15 @@ def calculate_pseudo_cl_eb_cov(self): self.print_done(f"Skipping Pseudo-Cl covariance calculation, {out_path} exists") self._pseudo_cls[ver]['cov'] = fits.open(out_path) else: - params = get_params_rho_tau(self.cc[ver], survey=ver) + + params = utils_cosmo_val.get_params_rho_tau(self.cc[ver], survey=ver) self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") - lmax = 2 * self.nside - z, dndz = self.get_redshift(ver) - ell = np.arange(1, lmax + 1) + lmax = 2*self.nside + path_redshift_distr = self.data_base_dir + self.cc[ver]["shear"]["redshift_distr"] pw = hp.pixwin(nside, lmax=lmax) - if pw.shape[0] != len(ell) + 1: - raise ValueError( - "Unexpected pixwin length for lmax=" - f"{lmax}: got {pw.shape[0]}, expected {len(ell)+1}" - ) - pw = pw[1:len(ell)+1] - - # Load redshift distribution and calculate theory C_ell - fiducial_cl = ( - get_theo_c_ell( - ell=ell, - z=z, - nz=dndz, - backend="camb", - cosmo=self.cosmo, - ) - * pw**2 - ) + fiducial_cl = self.get_fiducial(lmax, path_redshift_distr)*pw**2 self.print_cyan("Getting a sample of the fiducial Cl's with noise") @@ -2637,14 +1776,12 @@ def calculate_pseudo_cl_eb_cov(self): lmax = 2*self.nside b_lmax = lmax - 1 - ells = np.arange(lmin, lmax+1) - if self.binning == 'linear': - # Linear bands of width ell_step, respecting actual lmax - bpws = (ells - lmin) // self.ell_step - bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all - b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) + step = 10 + b = nmt.NmtBin.from_nside_linear(self.nside, step) elif self.binning == 'powspace': + ells = np.arange(lmin, lmax+1) + start = np.power(lmin, self.power) end = np.power(lmax, self.power) bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) @@ -2721,150 +1858,6 @@ def calculate_pseudo_cl_eb_cov(self): self.print_done("Done Pseudo-Cl covariance") - def calculate_pseudo_cl_onecovariance(self): - """ - Compute the pseudo-Cl covariance using OneCovariance. - """ - self.print_start("Computing Pseudo-Cl covariance with OneCovariance") - - if self.path_onecovariance is None: - raise ValueError("path_onecovariance must be provided to use OneCovariance") - - if not os.path.exists(self.path_onecovariance): - raise ValueError(f"OneCovariance path {self.path_onecovariance} does not exist") - - template_config = os.path.join(self.path_onecovariance, "config_files", "config_3x2pt_pure_Cell_UNIONS.ini") - if not os.path.exists(template_config): - raise ValueError(f"Template config file {template_config} does not exist") - - self._pseudo_cls_onecov = {} - for ver in self.versions: - self.print_magenta(ver) - - out_dir = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_onecov_{ver}/") - os.makedirs(out_dir, exist_ok=True) - - if os.path.exists(os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat")): - self.print_done(f"Skipping OneCovariance calculation, {out_dir} exists") - self._load_onecovariance_cov(out_dir, ver) - else: - - mask_path = self.cc[ver]['shear']['mask'] - redshift_distr_path = os.path.join(self.data_base_dir, self.cc[ver]['shear']['redshift_distr']) - - config_path = os.path.join(out_dir, f"config_onecov_{ver}.ini") - - self.print_cyan(f"Modifying OneCovariance config file and saving it to {config_path}") - self._modify_onecov_config(template_config, config_path, out_dir, mask_path, redshift_distr_path, ver) - - self.print_cyan("Running OneCovariance...") - cmd = f"python {os.path.join(self.path_onecovariance, 'covariance.py')} {config_path}" - self.print_cyan(f"Command: {cmd}") - ret = os.system(cmd) - if ret != 0: - raise RuntimeError(f"OneCovariance command failed with return code {ret}") - self.print_cyan("OneCovariance completed successfully.") - self._load_onecovariance_cov(out_dir, ver) - - self.print_done("Done Pseudo-Cl covariance with OneCovariance") - - def _modify_onecov_config(self, template_config, config_path, out_dir, mask_path, redshift_distr_path, ver): - """ - Modify OneCovariance configuration file with correct mask, redshift distribution, - and ellipticity dispersion parameters. - - Parameters - ---------- - template_config : str - Path to the template configuration file - config_path : str - Path where the modified configuration will be saved - mask_path : str - Path to the mask file - redshift_distr_path : str - Path to the redshift distribution file - """ - config = configparser.ConfigParser() - # Load the template configuration - config.read(template_config) - - # Update mask path - mask_base = os.path.basename(os.path.abspath(mask_path)) - mask_folder = os.path.dirname(os.path.abspath(mask_path)) - config['survey specs']['mask_directory'] = mask_folder - config['survey specs']['mask_file_lensing'] = mask_base - config['survey specs']['survey_area_lensing_in_deg2'] = str(self.area[ver]) - config['survey specs']['ellipticity_dispersion'] = str(self.ellipticity_dispersion[ver]) - config['survey specs']['n_eff_lensing'] = str(self.n_eff_gal[ver]) - - # Update redshift distribution path - redshift_distr_base = os.path.basename(os.path.abspath(redshift_distr_path)) - redshift_distr_folder = os.path.dirname(os.path.abspath(redshift_distr_path)) - config['redshift']['z_directory'] = redshift_distr_folder - config['redshift']['zlens_file'] = redshift_distr_base - - #Update output directory - config['output settings']['directory'] = out_dir - - # Save the modified configuration - with open(config_path, 'w') as f: - config.write(f) - - def get_cov_from_onecov(self, cov_one_cov, gaussian=True): - n_bins = np.sqrt(cov_one_cov.shape[0]).astype(int) - cov = np.zeros((n_bins, n_bins)) - - index_value = 10 if gaussian else 9 - for i in range(n_bins): - for j in range(n_bins): - cov[i, j] = cov_one_cov[i * n_bins + j, index_value] - - return cov - - def _load_onecovariance_cov(self, out_dir, ver): - self.print_cyan(f"Loading OneCovariance results from {out_dir}") - cov_one_cov = np.genfromtxt(os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat")) - gaussian_one_cov = self.get_cov_from_onecov(cov_one_cov, gaussian=True) - all_one_cov = self.get_cov_from_onecov(cov_one_cov, gaussian=False) - - self._pseudo_cls_onecov[ver] = { - 'gaussian_cov': gaussian_one_cov, - 'all_cov': all_one_cov - } - - def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): - assert gaussian_part in ["iNKA", "OneCovariance"], "gaussian_part must be 'iNKA' or 'OneCovariance'" - self.print_start(f"Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part") - - self._pseudo_cls_cov_g_ng = {} - - for ver in self.versions: - self.print_magenta(ver) - out_file = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_g_ng_{gaussian_part}_{ver}.fits") - if os.path.exists(out_file): - self.print_done(f"Skipping Gaussian and Non-Gaussian covariance calculation, {out_file} exists") - cov_hdu = fits.open(out_file) - self._pseudo_cls_cov_g_ng[ver] = cov_hdu - continue - if gaussian_part == "iNKA": - gaussian_cov = self.pseudo_cls[ver]['cov']['COVAR_EE_EE'].data - non_gaussian_cov = self.pseudo_cls_onecov[ver]['all_cov'] - self.pseudo_cls_onecov[ver]['gaussian_cov'] - full_cov = gaussian_cov + non_gaussian_cov - elif gaussian_part == "OneCovariance": - gaussian_cov = self.pseudo_cls_onecov[ver]['gaussian_cov'] - non_gaussian_cov = self.pseudo_cls_onecov[ver]['all_cov'] - self.pseudo_cls_onecov[ver]['gaussian_cov'] - full_cov = self.pseudo_cls_onecov[ver]['all_cov'] - else: - raise ValueError(f"Unknown gaussian_part: {gaussian_part}") - self.print_cyan("Saving Gaussian and Non-Gaussian covariance...") - hdu = fits.HDUList() - hdu.append(fits.ImageHDU(gaussian_cov, name="COVAR_GAUSSIAN")) - hdu.append(fits.ImageHDU(non_gaussian_cov, name="COVAR_NON_GAUSSIAN")) - hdu.append(fits.ImageHDU(full_cov, name="COVAR_FULL")) - hdu.writeto(out_file, overwrite=True) - self._pseudo_cls_cov_g_ng[ver] = hdu - self.print_done(f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part") - def calculate_pseudo_cl(self): """ Compute the pseudo-Cl of given catalogs. @@ -2887,98 +1880,104 @@ def calculate_pseudo_cl(self): self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") cl_shear = fits.getdata(out_path) self._pseudo_cls[ver]['pseudo_cl'] = cl_shear - elif self.cell_method == 'map': - self.calculate_pseudo_cl_map(ver, nside, out_path) - elif self.cell_method == 'catalog': - self.calculate_pseudo_cl_catalog(ver, out_path) else: - raise ValueError(f"Unknown cell method: {self.cell_method}") - - self.print_done("Done pseudo-Cl's") - - def calculate_pseudo_cl_map(self, ver, nside, out_path): - params = get_params_rho_tau(self.cc[ver], survey=ver) - - #Load data and create shear and noise maps - cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - - w = cat_gal[params['w_col']] - self.print_cyan("Creating maps and computing Cl's...") - n_gal_map, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) - mask = n_gal_map != 0 - - shear_map_e1 = np.zeros(hp.nside2npix(nside)) - shear_map_e2 = np.zeros(hp.nside2npix(nside)) + params = utils_cosmo_val.get_params_rho_tau(self.cc[ver], survey=ver) - e1 = cat_gal[params['e1_col']] - e2 = cat_gal[params['e2_col']] + #Load data and create shear and noise maps + cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - del cat_gal - - shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1*w) - shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2*w) - shear_map_e1[mask] /= n_gal_map[mask] - shear_map_e2[mask] /= n_gal_map[mask] + w = cat_gal[params['w_col']] + self.print_cyan("Creating maps and computing Cl's...") + n_gal_map, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) + mask = n_gal_map != 0 - shear_map = shear_map_e1 + 1j*shear_map_e2 + shear_map_e1 = np.zeros(hp.nside2npix(nside)) + shear_map_e2 = np.zeros(hp.nside2npix(nside)) - del shear_map_e1, shear_map_e2 + e1 = cat_gal[params['e1_col']] + e2 = cat_gal[params['e2_col']] - ell_eff, cl_shear, wsp = self.get_pseudo_cls_map(shear_map) + del cat_gal + + shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1*w) + shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2*w) + shear_map_e1[mask] /= n_gal_map[mask] + shear_map_e2[mask] /= n_gal_map[mask] - cl_noise = np.zeros_like(cl_shear) - - for i in range(self.nrandom_cell): + shear_map = shear_map_e1 + 1j*shear_map_e2 - noise_map_e1 = np.zeros(hp.nside2npix(nside)) - noise_map_e2 = np.zeros(hp.nside2npix(nside)) + del shear_map_e1, shear_map_e2 - e1_rot, e2_rot = self.apply_random_rotation(e1, e2) + ell_eff, cl_shear, wsp = self.get_pseudo_cls(shear_map) - - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) + cl_noise = np.zeros((4, self.n_ell_bins)) + + for i in range(self.nrandom_cell): - noise_map_e1[mask] /= n_gal_map[mask] - noise_map_e2[mask] /= n_gal_map[mask] + noise_map_e1 = np.zeros(hp.nside2npix(nside)) + noise_map_e2 = np.zeros(hp.nside2npix(nside)) - noise_map = noise_map_e1 + 1j*noise_map_e2 - del noise_map_e1, noise_map_e2 + e1_rot, e2_rot = self.apply_random_rotation(e1, e2) - _, cl_noise_, _ = self.get_pseudo_cls_map(noise_map, wsp) - cl_noise += cl_noise_ - - cl_noise /= self.nrandom_cell - del e1, e2, w - try: - del e1_rot, e2_rot - except NameError: #Continue if the random generation has been skipped. - pass - del n_gal_map + + noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) + noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) - #This is a problem because the measurement depends on the seed. To be fixed. - #cl_shear = cl_shear - np.mean(cl_noise, axis=1, keepdims=True) - cl_shear = cl_shear - cl_noise + noise_map_e1[mask] /= n_gal_map[mask] + noise_map_e2[mask] /= n_gal_map[mask] - self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + noise_map = noise_map_e1 + 1j*noise_map_e2 + del noise_map_e1, noise_map_e2 - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear + _, cl_noise_, _ = self.get_pseudo_cls(noise_map, wsp) + cl_noise += cl_noise_ + + cl_noise /= self.nrandom_cell + del e1, e2, e1_rot, e2_rot, w + del n_gal_map - def calculate_pseudo_cl_catalog(self, ver, out_path): - params = get_params_rho_tau(self.cc[ver], survey=ver) + #This is a problem because the measurement depends on the seed. To be fixed. + #cl_shear = cl_shear - np.mean(cl_noise, axis=1, keepdims=True) + cl_shear = cl_shear - cl_noise - #Load data and create shear and noise maps - cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) + self.print_cyan("Saving pseudo-Cl's...") + self.save_pseudo_cl(ell_eff, cl_shear, out_path) - ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog(catalog=cat_gal, params=params) + cl_shear = fits.getdata(out_path) + self._pseudo_cls[ver]['pseudo_cl'] = cl_shear - self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.print_done("Done pseudo-Cl's") - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear + def get_fiducial(self, lmax, redshift_distr): + """ + Get the power spectrum at Planck18 cosmology using CAMB. + """ + planck = Planck18 + + h = planck.H0.value/100 + Om = planck.Om0 + Ob = planck.Ob0 + Oc = Om - Ob + ns = 0.965 + As = 2.1e-9 + m_nu = 0.06 + w = -1 + + pars = camb.set_params(H0=100*h, omch2=Oc*h**2, ombh2=Ob*h**2, ns=ns, mnu=m_nu, w=w, As=As, WantTransfer=True, NonLinear=camb.model.NonLinear_both) + Onu = pars.omeganu + Oc = Om - Ob - Onu + pars = camb.set_params(H0=100*h, omch2=Oc*h**2, ombh2=Ob*h**2, ns=ns, mnu=m_nu, w=w, As=As, WantTransfer=True, NonLinear=camb.model.NonLinear_both) + + z, dndz = np.loadtxt(redshift_distr, unpack=True) + + #getthe expected cl's from CAMB + pars.min_l = 1 + pars.set_for_lmax(lmax) + pars.SourceWindows = [ + camb.sources.SplinedSourceWindow(z=z, W=dndz, source_type='lensing') + ] + theory_cls = camb.get_results(pars).get_source_cls_dict(lmax=lmax, raw_cl=True) + return theory_cls['W1xW1'] def get_n_gal_map(self, params, nside, cat_gal): """ @@ -2997,12 +1996,9 @@ def get_n_gal_map(self, params, nside, cat_gal): n_gal[unique_pix] = np.bincount(idx_rep, weights=w) return n_gal, unique_pix, idx, idx_rep - def get_gaussian_real( - self, params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep - ): - e1_rot, e2_rot = self.apply_random_rotation( - cat_gal[params["e1_col"]], cat_gal[params["e2_col"]] - ) + def get_gaussian_real(self, params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep): + + e1_rot, e2_rot = self.apply_random_rotation(cat_gal[params['e1_col']], cat_gal[params['e2_col']]) noise_map_e1 = np.zeros(hp.nside2npix(nside)) noise_map_e2 = np.zeros(hp.nside2npix(nside)) @@ -3026,7 +2022,7 @@ def get_sample(self, params, nside, lmax, b, cat_gal, n_gal, mask, unique_pix, i return cl_noise, f, wsp - def get_pseudo_cls_map(self, map, wsp=None): + def get_pseudo_cls(self, map, wsp=None): """ Compute the pseudo-cl for a given map. """ @@ -3035,56 +2031,12 @@ def get_pseudo_cls_map(self, map, wsp=None): lmax = 2*self.nside b_lmax = lmax - 1 - ells = np.arange(lmin, lmax+1) - if self.binning == 'linear': - # Linear bands of width ell_step, respecting actual lmax - bpws = (ells - lmin) // self.ell_step - bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all - b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) + step = 10 + b = nmt.NmtBin.from_nside_linear(self.nside, step) elif self.binning == 'powspace': - start = np.power(lmin, self.power) - end = np.power(lmax, self.power) - bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) + ells = np.arange(lmin, lmax+1) - #Get bandpowers - bpws = np.digitize(ells.astype(float), bins_ell) - 1 - bpws[0] = 0 - bpws[-1] = self.n_ell_bins-1 - - b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - - ell_eff = b.get_effective_ells() - - factor = -1 if self.pol_factor else 1 - - f_all = nmt.NmtField(mask=(map!=0), maps=[map.real, factor*map.imag], lmax=b_lmax) - - if wsp is None: - wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) - - cl_coupled = nmt.compute_coupled_cell(f_all, f_all) - cl_all = wsp.decouple_cell(cl_coupled) - - return ell_eff, cl_all, wsp - - def get_pseudo_cls_catalog(self, catalog, params, wsp=None): - """ - Compute the pseudo-cl for a given catalog. - """ - - lmin = 8 - lmax = 2*self.nside - b_lmax = lmax - 1 - - ells = np.arange(lmin, lmax+1) - - if self.binning == 'linear': - # Linear bands of width ell_step, respecting actual lmax - bpws = (ells - lmin) // self.ell_step - bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all - b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - elif self.binning == 'powspace': start = np.power(lmin, self.power) end = np.power(lmax, self.power) bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) @@ -3100,13 +2052,7 @@ def get_pseudo_cls_catalog(self, catalog, params, wsp=None): factor = -1 if self.pol_factor else 1 - f_all = nmt.NmtFieldCatalog(positions=[catalog[params['ra_col']], catalog[params['dec_col']]], - weights=catalog[params['w_col']], - field=[catalog[params['e1_col']], factor*catalog[params['e2_col']]], - lmax=b_lmax, - lmax_mask=b_lmax, - spin=2, - lonlat=True) + f_all = nmt.NmtField(mask=(map!=0), maps=[map.real, factor*map.imag], lmax=b_lmax) if wsp is None: wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) @@ -3183,7 +2129,7 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"], capsize=2) - ax[0].set_ylabel(r"$\ell C_\ell$") + ax[0].set_ylabel('$\ell C_\ell$') ax[0].set_xlim(ell.min()-10, ell.max()+100) ax[0].set_xscale('squareroot') @@ -3198,8 +2144,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"]) - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") + ax[1].set_xlabel('$\ell$') + ax[1].set_ylabel('$C_\ell$') ax[1].set_xlim(ell.min()-10, ell.max()+100) ax[1].set_xscale('squareroot') @@ -3224,8 +2170,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"], capsize=2) - ax[0].axhline(0, color="black", linestyle="--") - ax[0].set_ylabel(r"$\ell C_\ell$") + ax[0].axhline(0, color='black', linestyle='--') + ax[0].set_ylabel('$\ell C_\ell$') ax[0].set_xlim(ell.min()-10, ell.max()+100) ax[0].set_xscale('squareroot') @@ -3240,8 +2186,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"]) - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") + ax[1].set_xlabel('$\ell$') + ax[1].set_ylabel('$C_\ell$') ax[1].set_xlim(ell.min()-10, ell.max()+100) ax[1].set_xscale('squareroot') @@ -3266,8 +2212,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"], capsize=2) - ax[0].axhline(0, color="black", linestyle="--") - ax[0].set_ylabel(r"$\ell C_\ell$") + ax[0].axhline(0, color='black', linestyle='--') + ax[0].set_ylabel('$\ell C_\ell$') ax[0].set_xlim(ell.min()-10, ell.max()+100) ax[0].set_xscale('squareroot') @@ -3282,8 +2228,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"]) - ax[1].set_xlabel(r"$\ell$") - ax[1].set_ylabel(r"$C_\ell$") + ax[1].set_xlabel('$\ell$') + ax[1].set_ylabel('$C_\ell$') ax[1].set_xlim(ell.min()-10, ell.max()+100) ax[1].set_xscale('squareroot') From 0b4562ffcf9f5e4812982c587221033d9a04ff66 Mon Sep 17 00:00:00 2001 From: "martin.kilbinger@cea.fr" Date: Tue, 20 Jan 2026 14:32:04 +0100 Subject: [PATCH 11/15] added _aux functions (back) --- src/sp_validation/cosmo_val.py | 2535 +++++++++++++++++++++++--------- 1 file changed, 1807 insertions(+), 728 deletions(-) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 1a45dca8..edc9a255 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1,63 +1,193 @@ # %% +import copy import os -from contextlib import contextmanager -from functools import partial +from pathlib import Path +import configparser import colorama +import healpy as hp import matplotlib.pyplot as plt +import matplotlib.ticker as mticker import numpy as np import pymaster as nmt -import healpy as hp -import treecorr -from . import utils_cosmo_val -import camb -import re - -import yaml -from astropy.io import fits -from astropy import units as u -from astropy.coordinates import SkyCoord - -import healpy as hp -import healsparse as hsp import treecorr -import camb import yaml -from collections import Counter - -import skyproj - from astropy.io import fits -from astropy.cosmology import Planck18 -from astropy import units as u -from astropy.coordinates import SkyCoord - -try: - from cosmo_numba.B_modes.schneider2022 import get_pure_EB_modes -except: - print("Could not import cosmo_numba, continuing without") - -from . import utils_cosmo_val -import pymaster as nmt from cs_util import plots as cs_plots -from shear_psf_leakage import leakage +from shear_psf_leakage import leakage, run_object, run_scale from shear_psf_leakage import plots as psfleak_plots -from shear_psf_leakage import run_object, run_scale from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat -import matplotlib.scale as mscale -mscale.register_scale(utils_cosmo_val.SquareRootScale) +from .cosmology import get_cosmo, get_theo_c_ell +from .plots import FootprintPlotter +from .rho_tau import ( + get_params_rho_tau, + get_rho_tau_w_cov, + get_samples, +) # %% class CosmologyValidation: + """Framework for cosmic shear validation and systematics analysis. + + Handles two-point correlation function measurements, PSF systematics (rho/tau), + pseudo-C_ell analysis, and covariance estimation for weak lensing surveys. + Supports multiple catalog versions with automatic leakage-corrected variants. + + Parameters + ---------- + versions : list of str + Catalog version identifiers to analyze. Appending '_leak_corr' to a base + version creates a virtual catalog using leakage-corrected ellipticity columns + (e1_col_corrected/e2_col_corrected) from the base version configuration. + catalog_config : str, default './cat_config.yaml' + Path to catalog configuration YAML defining survey metadata, file paths, + and analysis settings for each version. + output_dir : str, optional + Override for output directory. If None, uses catalog config's paths.output. + rho_tau_method : {'lsq', 'mcmc'}, default 'lsq' + Fitting method for PSF leakage systematics parameters. + cov_estimate_method : {'th', 'jk'}, default 'th' + Covariance estimation: 'th' for semi-analytic theory, 'jk' for jackknife. + compute_cov_rho : bool, default True + Whether to compute covariance for rho statistics during PSF analysis. + n_cov : int, default 100 + Number of realizations for covariance estimation when using theory method. + theta_min : float, default 0.1 + Minimum angular separation in arcminutes for correlation function binning. + theta_max : float, default 250 + Maximum angular separation in arcminutes for correlation function binning. + nbins : int, default 20 + Number of angular bins for TreeCorr real-space correlation functions. + var_method : {'jackknife', 'sample', 'bootstrap', 'marked_bootstrap'}, default 'jackknife' + TreeCorr variance estimation method. + npatch : int, default 20 + Number of spatial patches for jackknife variance estimation. + quantile : float, default 0.1587 + Quantile for uncertainty bands in plots (default: 1-sigma ≈ 0.159). + theta_min_plot : float, default 0.08 + Minimum angular scale for plotting (may differ from analysis cut). + theta_max_plot : float, default 250 + Maximum angular scale for plotting. + ylim_alpha : list of float, default [-0.005, 0.05] + Y-axis limits for alpha systematic parameter plots. + ylim_xi_sys_ratio : list of float, default [-0.02, 0.5] + Y-axis limits for xi systematics ratio plots. + nside : int, default 1024 + HEALPix resolution for pseudo-C_ell analysis and area computation. + binning : {'powspace', 'linspace', 'logspace'}, default 'powspace' + Ell binning scheme for pseudo-C_ell (powspace = ell^power spacing). + power : float, default 0.5 + Exponent for power-law binning when binning='powspace'. + n_ell_bins : int, default 32 + Number of ell bins for pseudo-C_ell analysis (used with binning='powspace'). + ell_step : int, default 10 + Bin width in ell for linear binning (used with binning='linear'). + pol_factor : bool, default True + Apply polarization correction factor in pseudo-C_ell calculations. + nrandom_cell : int, default 10 + Number of random realizations for C_ell error estimation. + cosmo_params : dict, optional + Cosmological parameters to pass to get_cosmo(). If None, uses Planck 2018. + + Attributes + ---------- + versions : list of str + Validated catalog versions after processing _leak_corr variants. + cc : dict + Loaded catalog configuration with resolved absolute paths. + catalog_config_path : Path + Resolved path to the catalog configuration file. + treecorr_config : dict + Configuration dictionary passed to TreeCorr correlation objects. + cosmo : pyccl.Cosmology + Cosmology object for theory predictions. + + Notes + ----- + - Path resolution: Relative paths in catalog config are resolved using each + version's 'subdir' field as the base directory. + - Virtual _leak_corr versions: These create deep copies of the base version + config, swapping e1_col/e2_col with e1_col_corrected/e2_col_corrected. + - TreeCorr cross_patch_weight: Automatically set to 'match' for jackknife, + 'simple' otherwise, following TreeCorr best practices. + """ + + @staticmethod + def _split_seed_variant(version): + """Return the base version and seed label if version encodes a seed.""" + if "_seed" not in version: + return None, None + base, seed_label = version.rsplit("_seed", 1) + if not base or not seed_label.isdigit(): + return None, None + return base, seed_label + + @staticmethod + def _materialize_seed_path( + base_cfg, seed_label, version, base_version, catalog_config + ): + """Render the seed-specific shear path using Python string formatting.""" + shear_cfg = base_cfg["shear"] + template = shear_cfg.get("path_template") + + try: + seed_value = int(seed_label) + except ValueError as error: + raise ValueError( + f"Seed suffix for '{version}' is not numeric; cannot materialize path." + ) from error + + format_context = {"seed": seed_value, "seed_label": seed_label} + + if template: + try: + return template.format(**format_context) + except KeyError as error: + raise KeyError( + f"Missing placeholder '{error.args[0]}' in path_template for " + f"'{base_version}' while materializing '{version}'. Update " + f"{catalog_config}." + ) from error + except ValueError as error: + raise ValueError( + f"Invalid format specification in path_template for '{base_version}' " + f"while materializing '{version}'." + ) from error + + path = shear_cfg.get("path", "") + token_start = path.rfind("seed") + if token_start == -1: + raise ValueError( + f"Cannot materialize '{version}': '{base_version}' lacks a shear " + f"path_template and its shear path '{path}' does not contain a 'seed' " + f"token. Update {catalog_config}." + ) + cursor = token_start + 4 # len("seed") + if cursor < len(path) and not path[cursor].isdigit(): + cursor += 1 + digit_start = cursor + while cursor < len(path) and path[cursor].isdigit(): + cursor += 1 + digit_end = cursor + digits = path[digit_start:digit_end] + if not digits: + raise ValueError( + f"Cannot materialize '{version}': shear path '{path}' for base version " + f"'{base_version}' lacks digits after the seed token. Update " + f"{catalog_config}." + ) + + template = f"{path[:digit_start]}{{seed_label}}{path[digit_end:]}" + return template.format(**format_context) def __init__( self, versions, - data_base_dir, catalog_config="./cat_config.yaml", + output_dir=None, rho_tau_method="lsq", cov_estimate_method="th", compute_cov_rho=True, @@ -73,15 +203,17 @@ def __init__( ylim_alpha=[-0.005, 0.05], ylim_xi_sys_ratio=[-0.02, 0.5], nside=1024, - binning='powspace', - power=1/2, + nside_mask = 2**12, + binning="powspace", + power=1 / 2, n_ell_bins=32, + ell_step=10, pol_factor=True, - nrandom_cell=10 + cell_method='map', + nrandom_cell=10, + path_onecovariance=None, + cosmo_params=None, ): - - self.versions = versions - self.data_base_dir = data_base_dir self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method self.compute_cov_rho = compute_cov_rho @@ -95,13 +227,28 @@ def __init__( self.theta_max_plot = theta_max_plot self.ylim_alpha = ylim_alpha self.ylim_xi_sys_ratio = ylim_xi_sys_ratio - #For pseudo-Cls + # For pseudo-Cls self.nside = nside self.binning = binning self.power = power self.n_ell_bins = n_ell_bins + self.ell_step = ell_step self.pol_factor = pol_factor self.nrandom_cell = nrandom_cell + self.cell_method = cell_method + self.nside_mask = nside_mask + self.path_onecovariance = path_onecovariance + + assert self.cell_method in ["map", "catalog"], "cell_method must be 'map' or 'catalog'" + + # For theory calculations: + # Create cosmology object using new functionality + if cosmo_params is not None: + self.cosmo = get_cosmo(**cosmo_params) + else: + # Use Planck 2018 defaults + self.cosmo = get_cosmo() + self.treecorr_config = { "ra_units": "degrees", @@ -111,22 +258,234 @@ def __init__( "sep_units": "arcmin", "nbins": nbins, "var_method": var_method, + "cross_patch_weight": "match" if var_method == "jackknife" else "simple", } - with open(catalog_config, "r") as file: + self.catalog_config_path = Path(catalog_config) + with self.catalog_config_path.open("r") as file: self.cc = cc = yaml.load(file.read(), Loader=yaml.FullLoader) - for ver in ["nz", *versions]: - - if ver not in cc: - raise KeyError(f"Version string {ver} not found in config file{catalog_config}") - version_base = f"{data_base_dir}/{cc[ver]['subdir']}" + def resolve_paths_for_version(ver): + """Resolve relative paths for a version using its subdir.""" + subdir = Path(cc[ver]["subdir"]) for key in cc[ver]: if "path" in cc[ver][key]: - cc[ver][key]["path"] = f"{version_base}/{cc[ver][key]['path']}" + path = Path(cc[ver][key]["path"]) + cc[ver][key]["path"] = ( + str(path) if path.is_absolute() else str(subdir / path) + ) + + resolve_paths_for_version("nz") + processed = {"nz"} + final_versions = [] + leak_suffix = "_leak_corr" + + def ensure_version_exists(ver): + if ver in processed: + return + + if ver in cc: + resolve_paths_for_version(ver) + processed.add(ver) + return + + seed_base, seed_label = self._split_seed_variant(ver) + + if ver.endswith(leak_suffix): + base_ver = ver[: -len(leak_suffix)] + ensure_version_exists(base_ver) + shear_cfg = cc[base_ver]["shear"] + if "e1_col_corrected" not in shear_cfg or "e2_col_corrected" not in shear_cfg: + raise ValueError( + f"{base_ver} does not have e1_col_corrected/e2_col_corrected " + f"fields; cannot create {ver}" + ) + if ver not in cc: + cc[ver] = copy.deepcopy(cc[base_ver]) + cc[ver]["shear"]["e1_col"] = shear_cfg["e1_col_corrected"] + cc[ver]["shear"]["e2_col"] = shear_cfg["e2_col_corrected"] + resolve_paths_for_version(ver) + processed.add(ver) + return + + if seed_base is not None: + ensure_version_exists(seed_base) + if ver not in cc: + cc[ver] = copy.deepcopy(cc[seed_base]) + seed_path = self._materialize_seed_path( + cc[seed_base], + seed_label, + ver, + seed_base, + catalog_config, + ) + cc[ver]["shear"]["path"] = seed_path + resolve_paths_for_version(ver) + processed.add(ver) + return + + raise KeyError( + f"Version string {ver} not found in config file {catalog_config}" + ) + + for ver in versions: + ensure_version_exists(ver) + final_versions.append(ver) + + self.versions = final_versions + + # Override output directory if provided + if output_dir is not None: + cc["paths"]["output"] = output_dir + + os.makedirs(cc["paths"]["output"], exist_ok=True) + + def get_redshift(self, version): + """Load redshift distribution for a catalog version. + + Parameters + ---------- + version : str + Catalog version identifier + + Returns + ------- + z : ndarray + Redshift values + nz : ndarray + n(z) probability density + """ + redshift_path = self.cc[version]["shear"]["redshift_path"] + return np.loadtxt(redshift_path, unpack=True) + + def compute_survey_stats( + self, + ver, + weights_key_override=None, + mask_path=None, + nside=None, + overwrite_config=False, + ): + """Compute effective survey statistics for a catalog version. + + Parameters + ---------- + ver : str + Version string registered in the catalog config. + weights_key_override : str, optional + Override the weight column key (defaults to the configured `w_col`). + mask_path : str, optional + Explicit mask path to use when measuring survey area. + nside : int, optional + If provided, compute survey area from the catalog using this NSIDE when no + mask path is available. + overwrite_config : bool, optional + If True, persist the derived statistics back to the catalog configuration. + + Returns + ------- + dict + Dictionary containing: + - area_deg2: Survey area in square degrees. + - n_eff: Effective number density per arcmin^2. + - sigma_e: Per-component shape noise. + - sum_w: Sum of weights. + - sum_w2: Sum of squared weights. + - catalog_size: Number of galaxies processed. + """ + if ver not in self.cc: + raise KeyError(f"Version {ver} not found in catalog configuration") + + shear_cfg = self.cc[ver]["shear"] + cov_th = self.cc[ver].get("cov_th", {}) + + if "path" not in shear_cfg: + raise KeyError(f"No shear catalog path defined for version {ver}") - if not os.path.exists(cc["paths"]["output"]): - os.mkdir(cc["paths"]["output"]) + catalog_path = shear_cfg["path"] + if not os.path.exists(catalog_path): + raise FileNotFoundError(f"Shear catalog not found: {catalog_path}") + + data = fits.getdata(catalog_path, memmap=True) + n_rows = len(data) + + e1 = np.asarray(data[shear_cfg["e1_col"]], dtype=float) + e2 = np.asarray(data[shear_cfg["e2_col"]], dtype=float) + + weight_column = weights_key_override or shear_cfg["w_col"] + if weight_column not in data.columns.names: + raise KeyError(f"Weight column '{weight_column}' missing in {catalog_path}") + + w = np.asarray(data[weight_column], dtype=float) + + sum_w = float(np.sum(w)) + sum_w2 = float(np.sum(w**2)) + sum_w2_e2 = float(np.sum((w**2) * (e1**2 + e2**2))) + + if mask_path is not None: + if not os.path.exists(mask_path): + raise FileNotFoundError(f"Mask path not found: {mask_path}") + mask_candidate = mask_path + else: + mask_candidate = self.cc[ver].get("mask") + if isinstance(mask_candidate, str) and not os.path.isabs(mask_candidate): + mask_candidate = str(Path(self.cc[ver]["subdir"]) / mask_candidate) + if mask_candidate is not None and not os.path.exists(mask_candidate): + mask_candidate = None + + area_deg2 = None + if mask_candidate is not None and os.path.exists(mask_candidate): + area_deg2 = self._area_from_mask(mask_candidate) + elif cov_th.get("A") is not None: + area_deg2 = float(cov_th["A"]) + elif nside is not None: + area_deg2 = self._area_from_catalog(catalog_path, nside) + else: + raise ValueError( + f"Unable to determine survey area for {ver}. Provide mask_path or nside." + ) + + area_arcmin2 = area_deg2 * 3600.0 + + n_eff = (sum_w**2) / (area_arcmin2 * sum_w2) if sum_w2 > 0 else 0.0 + sigma_e = np.sqrt(sum_w2_e2 / sum_w2) if sum_w2 > 0 else 0.0 + + results = { + "area_deg2": area_deg2, + "n_eff": n_eff, + "sigma_e": sigma_e, + "sum_w": sum_w, + "sum_w2": sum_w2, + "catalog_size": n_rows, + } + + if overwrite_config: + if "cov_th" not in self.cc[ver]: + self.cc[ver]["cov_th"] = {} + self.cc[ver]["cov_th"]["A"] = float(area_deg2) + self.cc[ver]["cov_th"]["n_e"] = float(n_eff) + self.cc[ver]["cov_th"]["sigma_e"] = float(sigma_e) + self._write_catalog_config() + + return results + + def _area_from_catalog(self, catalog_path, nside): + data = fits.getdata(catalog_path, memmap=True) + ra = np.asarray(data["RA"], dtype=float) + dec = np.asarray(data["Dec"], dtype=float) + theta = np.radians(90.0 - dec) + phi = np.radians(ra) + pix = hp.ang2pix(nside, theta, phi, lonlat=False) + unique_pix = np.unique(pix) + return float(unique_pix.size * hp.nside2pixarea(nside, degrees=True)) + + def _area_from_mask(self, mask_map_path): + mask = hp.read_map(mask_map_path, dtype=np.float64) + return float(mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True)) + + def _write_catalog_config(self): + with self.catalog_config_path.open("w") as file: + yaml.dump(self.cc, file, sort_keys=False) def color_reset(self): print(colorama.Fore.BLACK, end="") @@ -163,7 +522,7 @@ def set_params_leakage_scale(self, ver): params_in["dndz_path"] = ( f"{self.cc['nz']['dndz']['path']}_{self.cc[ver]['pipeline']}_{self.cc['nz']['dndz']['blind']}.txt" ) - params_in["output_dir"] = f'{self.cc["paths"]["output"]}/leakage_{ver}' + params_in["output_dir"] = f"{self.cc['paths']['output']}/leakage_{ver}" # Note: for SP these are calibrated shear estimates params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] @@ -190,7 +549,7 @@ def set_params_leakage_object(self, ver): # Set parameters params_in["input_path_shear"] = self.cc[ver]["shear"]["path"] - params_in["output_dir"] = f'{self.cc["paths"]["output"]}/leakage_{ver}' + params_in["output_dir"] = f"{self.cc['paths']['output']}/leakage_{ver}" # Note: for SP these are calibrated shear estimates params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] @@ -209,18 +568,13 @@ def set_params_leakage_object(self, ver): + f" shear yaml entry for version {ver}" ) - if "cols" in self.cc[ver]["shear"]: - params_in["cols"] = self.cc[ver]["shear"]["cols"] - params_in["verbose"] = False return params_in def init_results(self, objectwise=False): - results = {} for ver in self.versions: - # Set parameters depending on the type of leakage if objectwise: results[ver] = run_object.LeakageObject() @@ -232,20 +586,6 @@ def init_results(self, objectwise=False): results[ver].check_params() results[ver].prepare_output() - @contextmanager - def temporarily_load_data(results_instance): - try: - self.print_start(f"Loading catalog {results_instance._params['input_path_shear']} ...", end="") - results_instance.read_data() - self.print_done(f"done") - yield - finally: - self.print_done(f"Freeing {results_instance._params['input_path_shear']} from memory") - del results_instance.dat_shear - del results_instance.dat_PSF - - results[ver].temporarily_load_data = partial(temporarily_load_data, results[ver]) - return results @property @@ -260,21 +600,34 @@ def results_objectwise(self): self._results_objectwise = self.init_results(objectwise=True) return self._results_objectwise - def calculate_rho_tau_stats(self): + def basename(self, version, treecorr_config=None, npatch=None): + cfg = treecorr_config or self.treecorr_config + patches = npatch or self.npatch + return ( + f"{version}_minsep={cfg['min_sep']}" + f"_maxsep={cfg['max_sep']}" + f"_nbins={cfg['nbins']}" + f"_npatch={patches}" + ) + + def calculate_rho_tau_stats(self): out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): os.mkdir(out_dir) self.print_start("Rho stats") for ver in self.versions: - rho_stat_handler, tau_stat_handler = utils_cosmo_val.get_rho_tau_w_cov( + base = self.basename(ver) + rho_stat_handler, tau_stat_handler = get_rho_tau_w_cov( self.cc, ver, self.treecorr_config, out_dir, + base, method=self.cov_estimate_method, cov_rho=self.compute_cov_rho, + npatch=self.npatch, ) self.print_done("Rho stats finished") @@ -296,10 +649,107 @@ def tau_stat_handler(self): @property def colors(self): return [self.cc[ver]["colour"] for ver in self.versions] + + @property + def area(self): + if not hasattr(self, "_area"): + self.calculate_area() + return self._area + + @property + def n_eff_gal(self): + if not hasattr(self, "_n_eff_gal"): + self.calculate_n_eff_gal() + return self._n_eff_gal - def plot_rho_stats(self, abs=False): + @property + def ellipticity_dispersion(self): + if not hasattr(self, "_ellipticity_dispersion"): + self.calculate_ellipticity_dispersion() + return self._ellipticity_dispersion + + @property + def pseudo_cls(self): + if not hasattr(self, "_pseudo_cls"): + self.calculate_pseudo_cl() + self.calculate_pseudo_cl_eb_cov() + return self._pseudo_cls + + @property + def pseudo_cls_onecov(self): + if not hasattr(self, "_pseudo_cls_onecov"): + self.calculate_pseudo_cl_onecovariance() + return self._pseudo_cls_onecov + + def calculate_area(self): + self.print_start("Calculating area") + area = {} + for ver in self.versions: + self.print_magenta(ver) + + if not hasattr(self.cc[ver]['shear'], 'mask'): + print("Mask not found in config file, calculating area from binned catalog") + area[ver] = self.calculate_area_from_binned_catalog(ver) + else: + mask = hp.read_map(self.cc[ver]['shear']['mask'], verbose=False) + nside_mask = hp.get_nside(mask) + print(f"nside_mask = {nside_mask}") + area[ver] = np.sum(mask) * hp.nside2pixarea(nside_mask, degrees=True) + print(f"Area = {area[ver]:.2f} deg^2") + + self._area = area + self.print_done("Area calculation finished") + + def calculate_area_from_binned_catalog(self, ver): + print(f"nside_mask = {self.nside_mask}") + with self.results[ver].temporarily_read_data(): + ra = self.results[ver].dat_shear["RA"] + dec = self.results[ver].dat_shear["Dec"] + hsp_map = hp.ang2pix( + self.nside_mask, + np.radians(90 - dec), + np.radians(ra), + lonlat=False, + ) + mask = np.bincount(hsp_map, minlength=hp.nside2npix(self.nside_mask)) > 0 + + area = np.sum(mask) * hp.nside2pixarea(self.nside_mask, degrees=True) + print(f"Area = {area:.2f} deg^2") + + return area + + def calculate_n_eff_gal(self): + self.print_start("Calculating effective number of galaxy") + n_eff_gal = {} + for ver in self.versions: + self.print_magenta(ver) + with self.results[ver].temporarily_read_data(): + w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] + n_eff_gal[ver] = 1/(self.area[ver]*60*60)* np.sum(w)**2/np.sum(w**2) + print(f"n_eff_gal = {n_eff_gal[ver]:.2f} gal./arcmin^-2") + + self._n_eff_gal = n_eff_gal + self.print_done("Effective number of galaxy calculation finished") + + def calculate_ellipticity_dispersion(self): + self.print_start("Calculating ellipticity dispersion") + ellipticity_dispersion = {} + for ver in self.versions: + self.print_magenta(ver) + with self.results[ver].temporarily_read_data(): + e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] + e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] + ellipticity_dispersion[ver] = np.sqrt( + 0.5*(np.average(e1**2, weights=w**2) + np.average(e2**2, weights=w**2)) + ) + print(f"Ellipticity dispersion = {ellipticity_dispersion[ver]:.4f}") + self._ellipticity_dispersion = ellipticity_dispersion - filenames = [f"rho_stats_{ver}.fits" for ver in self.versions] + def plot_rho_stats(self, abs=False): + filenames = [ + f"rho_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "rho_stats.png" self.rho_stat_handler.plot_rho_stats( @@ -309,16 +759,19 @@ def plot_rho_stats(self, abs=False): savefig=savefig, legend="outside", abs=abs, + show=True, + close=True, ) - plt.close() self.print_done( - f"Rho stats plot saved to " + "Rho stats plot saved to " + f"{os.path.abspath(self.rho_stat_handler.catalogs._output)}/{savefig}", ) def plot_tau_stats(self, plot_tau_m=False): - filenames = [f"tau_stats_{ver}.fits" for ver in self.versions] + filenames = [ + f"tau_stats_{self.basename(ver)}.fits" for ver in self.versions + ] savefig = "tau_stats.png" self.tau_stat_handler.plot_tau_stats( @@ -328,15 +781,17 @@ def plot_tau_stats(self, plot_tau_m=False): savefig=savefig, legend="outside", plot_tau_m=plot_tau_m, + show=True, + close=True, ) - plt.close() self.print_done( - f"Tau stats plot saved to " + "Tau stats plot saved to " + f"{os.path.abspath(self.tau_stat_handler.catalogs._output)}/{savefig}", ) def set_params_rho_tau(self, params, params_psf, survey="other"): + params = {**params} if survey in ("DES", "SP_axel_v0.0", "SP_axel_v0.0_repr"): params["patch_number"] = 120 print("DES, jackknife patch number = 120") @@ -349,8 +804,8 @@ def set_params_rho_tau(self, params, params_psf, survey="other"): else: params["patch_number"] = 150 - params["ra_col"] = params_psf["ra_col"] - params["dec_col"] = params_psf["dec_col"] + params["ra_PSF_col"] = params_psf["ra_col"] + params["dec_PSF_col"] = params_psf["dec_col"] params["e1_PSF_col"] = params_psf["e1_PSF_col"] params["e2_PSF_col"] = params_psf["e2_PSF_col"] params["e1_star_col"] = params_psf["e1_star_col"] @@ -363,6 +818,8 @@ def set_params_rho_tau(self, params, params_psf, survey="other"): params["ra_units"] = "deg" params["dec_units"] = "deg" + params["w_col"] = self.cc[survey]["shear"]["w_col"] + return params @property @@ -394,9 +851,12 @@ def calculate_rho_tau_fits(self): self.cov_estimate_method, None ) - flat_samples, result, q = utils_cosmo_val.get_samples( + base = self.basename(ver) + + flat_samples, result, q = get_samples( self.psf_fitter, ver, + base, cov_type=self.cov_estimate_method, apply_debias=npatch, sampler=self.rho_tau_method, @@ -406,7 +866,7 @@ def calculate_rho_tau_fits(self): self.rho_tau_fits["result_list"].append(result) self.rho_tau_fits["q_list"].append(q) - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] xi_psf_sys_samples = np.array([]).reshape(0, nbins) @@ -439,8 +899,9 @@ def plot_rho_tau_fits(self): legend_loc="upper right", contour_colors=self.colors, markers={"x0": 0, "x1": 1, "x2": 1}, + show=True, + close=True, ) - plt.close() self.print_done(f"Tau contours plot saved to {os.path.abspath(savefig)}") plt.figure(figsize=(15, 6)) @@ -450,16 +911,17 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for i in range(100): self.psf_fitter.plot_xi_psf_sys( flat_sample[-i + 1], ver, color, alpha=0.1 ) self.psf_fitter.plot_xi_psf_sys(mcmc_result[1], ver, color) plt.legend() - savefig = os.path.abspath(f"{out_dir}/xi_psf_sys_samples.png") - cs_plots.savefig(savefig) - self.print_done(f"xi_psf_sys samples plot saved to {savefig}") + out_path = os.path.abspath(f"{out_dir}/xi_psf_sys_samples.png") + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() + self.print_done(f"xi_psf_sys samples plot saved to {out_path}") plt.figure(figsize=(15, 6)) for mcmc_result, ver, color, flat_sample in zip( @@ -468,7 +930,6 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): - ls = self.cc[ver]["ls"] theta = self.psf_fitter.rho_stat_handler.rho_stats["theta"] xi_psf_sys = self.xi_psf_sys[ver] @@ -490,23 +951,23 @@ def plot_rho_tau_fits(self): plt.ylabel(r"$\xi^{\rm PSF}_{\rm sys}$") plt.title(f"{1 - self.quantile:.1%}, {self.quantile:.1%} quantiles") plt.legend() - plt.show() - savefig = os.path.abspath(f"{out_dir}/xi_psf_sys_quantiles.png") - cs_plots.savefig(savefig) - self.print_done(f"xi_psf_sys quantiles plot saved to {savefig}") + out_path = os.path.abspath(f"{out_dir}/xi_psf_sys_quantiles.png") + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() + self.print_done(f"xi_psf_sys quantiles plot saved to {out_path}") for mcmc_result, ver, flat_sample in zip( self.rho_tau_fits["result_list"], self.versions, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat("rho_stats_" + ver + ".fits") + self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") for yscale in ("linear", "log"): out_path = os.path.abspath( f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" ) self.psf_fitter.plot_xi_psf_sys_terms( - ver, mcmc_result[1], out_path, yscale=yscale + ver, mcmc_result[1], out_path, yscale=yscale, show=True ) self.print_done( f"{yscale}-scale xi_psf_sys terms plot saved to {out_path}" @@ -522,20 +983,17 @@ def plot_footprints(self): self.print_start("Plotting footprints:") for ver in self.versions: self.print_magenta(ver) - results = self.results[ver] - + fp = FootprintPlotter() - - for region in fp._regions: + + for region in fp._regions: out_path = os.path.abspath( f"{self.cc['paths']['output']}/footprint_{ver}_{region}.png" ) if os.path.exists(out_path): - self.print_done( - f"Skipping footprint plot, {out_path} exists" - ) + self.print_done(f"Skipping footprint plot, {out_path} exists") else: - with self.results[ver].temporarily_load_data(): + with self.results[ver].temporarily_read_data(): hsp_map = fp.create_hsp_map( self.results[ver].dat_shear["RA"], self.results[ver].dat_shear["Dec"], @@ -550,15 +1008,15 @@ def calculate_scale_dependent_leakage(self): results = self.results[ver] output_base_path = os.path.abspath( - f'{self.cc["paths"]["output"]}/leakage_{ver}/xi_for_leak_scale' + f"{self.cc['paths']['output']}/leakage_{ver}/xi_for_leak_scale" ) output_path_ab = f"{output_base_path}_a_b.txt" output_path_aa = f"{output_base_path}_a_a.txt" - - with self.results[ver].temporarily_load_data(): + with self.results[ver].temporarily_read_data(): if os.path.exists(output_path_ab) and os.path.exists(output_path_aa): self.print_green( - f"Skipping computation, reading {output_path_ab} and {output_path_aa} instead" + f"Skipping computation, reading {output_path_ab} and " + f"{output_path_aa} instead" ) results.r_corr_gp = treecorr.GGCorrelation(self.treecorr_config) @@ -598,7 +1056,6 @@ def plot_scale_dependent_leakage(self): markers.append(self.cc[ver]["marker"]) if len(theta) > 0: - # Log x out_path = os.path.abspath( f"{self.cc['paths']['output']}/alpha_leak_log.png" @@ -623,7 +1080,8 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, shift_x=True, ) - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"Log-scale alpha leakage plot saved to {out_path}") # Lin x @@ -650,7 +1108,8 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, shift_x=False, ) - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"Lin-scale alpha leakage plot saved to {out_path}") # Plot xi_sys @@ -672,7 +1131,6 @@ def plot_scale_dependent_leakage(self): ylabel = r"$\xi^{\rm sys}_+(\theta)$" title = "Cross-correlation leakage" out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_p.png") - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) cs_plots.plot_data_1d( theta, y, @@ -688,7 +1146,8 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, # shift_x=True, ) - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"xi_sys_plus plot saved to {out_path}") y = [] @@ -703,7 +1162,6 @@ def plot_scale_dependent_leakage(self): ylabel = r"$\xi^{\rm sys}_-(\theta)$" title = "Cross-correlation leakage" out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_m.png") - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) cs_plots.plot_data_1d( theta, y, @@ -720,7 +1178,8 @@ def plot_scale_dependent_leakage(self): linestyles=linestyles, # shift_x=True, ) - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"xi_sys_minus plot saved to {out_path}") def calculate_objectwise_leakage(self): @@ -738,25 +1197,31 @@ def calculate_objectwise_leakage(self): results_obj.update_params() results_obj.prepare_output() - with self.results[ver].temporarily_load_data(): - results_obj._dat = self.results[ver].dat_shear + # Skip read_data() and copy catalogue from scale leakage instance instead + # results_obj._dat = self.results[ver].dat_shear - out_base = results_obj.get_out_base(mix, order) - out_path = f"{out_base}.pkl" - if os.path.exists(out_path): - self.print_green( - f"Skipping object-wise leakage, file {out_path} exists" - ) - results_obj.par_best_fit = leakage.read_from_file(out_path) - else: - self.print_cyan("Computing object-wise leakage regression") + out_base = results_obj.get_out_base(mix, order) + out_path = f"{out_base}.pkl" + if os.path.exists(out_path): + self.print_green( + f"Skipping object-wise leakage, file {out_path} exists" + ) + results_obj.par_best_fit = leakage.read_from_file(out_path) + else: + self.print_cyan("Computing object-wise leakage regression") - # Run + # Run + with results_obj.temporarily_read_data(): + try: results_obj.PSF_leakage() + except KeyError as e: + print(f"{e}\nExpected key is missing from catalog.") + # remove the results object for this version + self.results_objectwise.pop(ver) # Gather coefficients leakage_coeff = {} - for ver in self.versions: + for ver in self.results_objectwise: leakage_coeff[ver] = {} results = self.results[ver] results_obj = self.results_objectwise[ver] @@ -794,12 +1259,12 @@ def plot_objectwise_leakage(self): self.calculate_objectwise_leakage() self.print_start("Plotting object-wise leakage:") - fig = cs_plots.figure(figsize=(15, 15)) + cs_plots.figure(figsize=(15, 15)) linestyles = ["-", "--", ":"] fillstyles = ["full", "none", "left", "right", "bottom", "top"] - for ver in self.versions: + for ver in self.results_objectwise: label = ver for key, ls, fs in zip( ["alpha_mean", "alpha_1", "alpha_0"], linestyles, fillstyles @@ -834,10 +1299,11 @@ def plot_objectwise_leakage(self): out_path = os.path.abspath( f"{self.cc['paths']['output']}/leakage_coefficients.png" ) - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"Object-wise leakage coefficients plot saved to {out_path}") - def calculate_objectwise_leakage_aux(self): + def calculate_objectwise_leakage_aux(self): self.print_start("Object-wise leakage auxiliary quantities:") for ver in self.versions: @@ -870,17 +1336,22 @@ def plot_ellipticity(self, nbins=200): self.print_start("Computing ellipticity histograms:") fig, axs = plt.subplots(1, 2, figsize=(22, 7)) + bins = np.linspace(-1.5, 1.5, nbins + 1) for ver in self.versions: - with self.results[ver].temporarily_load_data(): - self.print_magenta(ver) - R = self.cc[ver]["shear"]["R"] - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R - w = self.results[ver].dat_shear["w_col"] + self.print_magenta(ver) + R = self.cc[ver]["shear"]["R"] + with self.results[ver].temporarily_read_data(): + e1 = ( + self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R + ) + e2 = ( + self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R + ) + w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] axs[0].hist( e1, - bins=nbins, + bins=bins, density=False, histtype="step", weights=w, @@ -889,6 +1360,38 @@ def plot_ellipticity(self, nbins=200): ) axs[1].hist( e2, + bins=bins, + density=False, + histtype="step", + weights=w, + label=ver, + color=self.cc[ver]["colour"], + ) + + for idx in (0, 1): + axs[idx].set_xlabel(f"$e_{idx}$") + axs[idx].set_ylabel("frequency") + axs[idx].legend() + axs[idx].set_xlim([-1.5, 1.5]) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() + self.print_done("Ellipticity histograms saved to " + out_path) + + def plot_weights(self, nbins=200): + out_path = os.path.abspath(f"{self.cc['paths']['output']}/weight_hist.png") + if os.path.exists(out_path): + self.print_done(f"Skipping weight histograms, {out_path} exists") + else: + self.print_start("Computing weight histograms:") + + fig, ax = plt.subplots(1, 1, figsize=(10, 7)) + for ver in self.versions: + self.print_magenta(ver) + with self.results[ver].temporarily_read_data(): + w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] + + plt.hist( + w, bins=nbins, density=False, histtype="step", @@ -897,19 +1400,21 @@ def plot_ellipticity(self, nbins=200): color=self.cc[ver]["colour"], ) - for idx in (0, 1): - axs[idx].set_xlabel(f"$e_{idx}$") - axs[idx].set_ylabel("frequency") - axs[idx].legend() - axs[idx].set_xlim([-1.5, 1.5]) - cs_plots.savefig(out_path) - self.print_done("Ellipticity histograms saved to " + out_path) + plt.xlabel("$w$") + plt.ylabel("frequency") + plt.yscale("log") + plt.legend() + # plt.xlim([-0.01, 1.2]) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() + self.print_done("Weight histograms saved to " + out_path) def plot_separation(self, nbins=200): self.print_start("Separation histograms") if "SP_matched_MP_v1.0" in self.versions: fig, axs = plt.subplots(1, 1, figsize=(10, 7)) - sep = self.results["SP_matched_MP_v1.0"].dat_shear["Separation"] + with self.results["SP_matched_MP_v1.0"].temporarily_read_data(): + sep = self.results["SP_matched_MP_v1.0"].dat_shear["Separation"] axs.hist( sep, bins=nbins, @@ -930,128 +1435,218 @@ def calculate_additive_bias(self): self._c2 = {} for ver in self.versions: self.print_magenta(ver) - with self.results[ver].temporarily_load_data(): - R = self.cc[ver]["shear"]["R"] + R = self.cc[ver]["shear"]["R"] + e1_col, e2_col, w_col = [ + self.cc[ver]["shear"][k] for k in ["e1_col", "e2_col", "w_col"] + ] + with self.results[ver].temporarily_read_data(): self._c1[ver] = np.average( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R, - weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], + self.results[ver].dat_shear[e1_col] / R, + weights=self.results[ver].dat_shear[w_col], ) self._c2[ver] = np.average( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R, - weights=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], + self.results[ver].dat_shear[e2_col] / R, + weights=self.results[ver].dat_shear[w_col], ) - self.print_done("Finished additive bias calculation.") + self.print_done("Finished additive bias calculation.") @property def c1(self): if not hasattr(self, "_c1"): - print("MKDEBUG calc c1") self.calculate_additive_bias() return self._c1 @property def c2(self): if not hasattr(self, "_c2"): - print("MKDEBUG calc c2") self.calculate_additive_bias() return self._c2 - def calculate_2pcf(self): - self.print_start(f"Computing 2PCF") + def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): + """ + Calculate the two-point correlation function (2PCF) ξ± for a given catalog + version with TreeCorr. + + By default the class instance's `npatch` and `treecorr_config` entries are + used to + initialize the TreeCorr Catalog and GGCorrelation objects, but may be + overridden + by passing keyword arguments. + + Parameters: + ver (str): The catalog version to process. + + npatch (int, optional): The number of patches to use for the calculation. + Defaults to the instance's `npatch` attribute. + + save_fits (bool, optional): Whether to save the ξ± results to FITS files. + Defaults to False. + + **treecorr_config: Additional TreeCorr configuration parameters that will + override the instance's default `treecorr_config`. For example, `min_sep=1`. + + Returns: + treecorr.GGCorrelation: The TreeCorr GGCorrelation object containing the + computed 2PCF results. + + Notes: + - If the output file for the given configuration already exists, the + calculation is skipped, and the results are loaded from the file. + - If a patch file for the given configuration does not exist, it is + created during the process. + - FITS files for ξ+ and ξ− are saved with additional metadata in their + headers if `save_fits` is True. + """ - self._cat_ggs = {} - for ver in self.versions: - self.print_magenta(ver) - gg = self._cat_ggs[ver] = treecorr.GGCorrelation(self.treecorr_config) + self.print_magenta(f"Computing {ver} ξ±") - out_fname = os.path.abspath(f"{self.cc['paths']['output']}/xi_pm_{ver}.txt") - if os.path.exists(out_fname): - self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") - gg.read(out_fname) - else: - # Run TreeCorr - with self.results[ver].temporarily_load_data(): - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - if ver != "DES": - R = self.cc[ver]["shear"]["R"] - g1 = (e1 - self.c1[ver]) / R - g2 = (e2 - self.c2[ver]) / R - else: - R11 = self.cc[ver]["shear"]["R11"] - R22 = self.cc[ver]["shear"]["R22"] - g1 = (e1 - self.c1[ver]) / np.average( - self.results[ver].dat_shear[R11] - ) - g2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - - self.c2[ver] - ) / np.average(self.results[ver].dat_shear[R22]) - cat_gal = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=self.npatch, - ) - gg.process(cat_gal) - gg.write(out_fname) + npatch = npatch or self.npatch + treecorr_config = { + **self.treecorr_config, + **treecorr_config, + "var_method": "jackknife" if int(npatch) > 1 else "shot", + } - # Save xi_p and xi_m results to fits file - lst = np.arange(1, self.treecorr_config["nbins"] + 1) + gg = treecorr.GGCorrelation(treecorr_config) - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column( - name="ANG", format="D", unit="arcmin", array=gg.meanr + # If the output file already exists, skip the calculation + out_fname = os.path.abspath( + f"{self.cc['paths']['output']}/{ver}_xi_minsep={treecorr_config['min_sep']}_maxsep={treecorr_config['max_sep']}_nbins={treecorr_config['nbins']}_npatch={npatch}.txt" + ) + + if os.path.exists(out_fname): + self.print_done(f"Skipping 2PCF calculation, {out_fname} exists") + gg.read(out_fname) + + else: + # Load data and create a catalog + with self.results[ver].temporarily_read_data(): + e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] + e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] + if ver != "DES": + R = self.cc[ver]["shear"]["R"] + g1 = (e1 - self.c1[ver]) / R + g2 = (e2 - self.c2[ver]) / R + else: + R11 = self.cc[ver]["shear"]["R11"] + R22 = self.cc[ver]["shear"]["R22"] + g1 = (e1 - self.c1[ver]) / np.average( + self.results[ver].dat_shear[R11] ) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - xiplus_hdu.writeto( - f"{self.cc['paths']['output']}/xi_plus_{ver}.fits", overwrite=True + g2 = (e2 - self.c2[ver]) / np.average( + self.results[ver].dat_shear[R22] ) - # append xi_minus header info - ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} - for key in ximinus_dict: - ximinus_hdu.header[key] = ximinus_dict[key] - ximinus_hdu.writeto( - f"{self.cc['paths']['output']}/xi_minus_{ver}.fits", overwrite=True - ) + # Use patch file if it exists + patch_file = os.path.abspath( + f"{self.cc['paths']['output']}/{ver}_patches_npatch={npatch}.dat" + ) - self.print_done("Done 2PCF") + cat_gal = treecorr.Catalog( + ra=self.results[ver].dat_shear["RA"], + dec=self.results[ver].dat_shear["Dec"], + g1=g1, + g2=g2, + w=w, + ra_units=self.treecorr_config["ra_units"], + dec_units=self.treecorr_config["dec_units"], + npatch=npatch, + patch_centers=patch_file if os.path.exists(patch_file) else None, + ) - @property - def cat_ggs(self): - if not hasattr(self, "_cat_ggs"): - self.calculate_2pcf() - return self._cat_ggs + # If no patch file exists, save the current patches + if not os.path.exists(patch_file): + cat_gal.write_patch_centers(patch_file) + + # Process the catalog & write the correlation functions + gg.process(cat_gal) + gg.write(out_fname, write_patch_results=True, write_cov=True) + + # Save xi_p and xi_m results to fits file + # (moved outside so it runs even if txt exists) + if save_fits: + lst = np.arange(1, treecorr_config["nbins"] + 1) + + col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) + col3 = fits.Column(name="ANGBIN", format="K", array=lst) + col4 = fits.Column(name="VALUE", format="D", array=gg.xip) + col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") + + col4 = fits.Column(name="VALUE", format="D", array=gg.xim) + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") + + # append xi_plus header info + xiplus_dict = { + "2PTDATA": "T", + "QUANT1": "G+R", + "QUANT2": "G+R", + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + } + for key in xiplus_dict: + xiplus_hdu.header[key] = xiplus_dict[key] + + col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) + col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) + col3 = fits.Column(name="ANGBIN", format="K", array=lst) + col4 = fits.Column(name="VALUE", format="D", array=gg.xip) + col5 = fits.Column( + name="ANG", format="D", unit="arcmin", array=gg.rnom + ) + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") + + col4 = fits.Column(name="VALUE", format="D", array=gg.xim) + coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) + ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") + + # append xi_plus header info + xiplus_dict = { + "2PTDATA": "T", + "QUANT1": "G+R", + "QUANT2": "G+R", + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + } + for key in xiplus_dict: + xiplus_hdu.header[key] = xiplus_dict[key] + # Use same naming format as txt output + fits_base = out_fname.replace(".txt", "").replace("_xi_", "_") + xiplus_hdu.writeto( + f"{fits_base.replace(ver, f'xi_plus_{ver}')}.fits", + overwrite=True, + ) + + # append xi_minus header info + ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} + for key in ximinus_dict: + ximinus_hdu.header[key] = ximinus_dict[key] + ximinus_hdu.writeto( + f"{fits_base.replace(ver, f'xi_minus_{ver}')}.fits", + overwrite=True, + ) + + # Add correlation object to class + if not hasattr(self, "cat_ggs"): + self.cat_ggs = {} + self.cat_ggs[ver] = gg + + self.print_done("Done 2PCF") + + return gg def plot_2pcf(self): # Plot of n_pairs fig, ax = plt.subplots(ncols=1, nrows=1) for ver in self.versions: + self.calculate_2pcf(ver) plt.plot( self.cat_ggs[ver].meanr, self.cat_ggs[ver].npairs, @@ -1063,7 +1658,8 @@ def plot_2pcf(self): plt.ylabel(r"$n_{\rm pair}$") plt.legend() out_path = os.path.abspath(f"{self.cc['paths']['output']}/n_pair.png") - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"n_pair plot saved to {out_path}") # Plot of xi_+ @@ -1079,13 +1675,14 @@ def plot_2pcf(self): ) plt.xscale("log") plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") + plt.legend() plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\xi_+(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_p.png") - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"xi_plus plot saved to {out_path}") # Plot of xi_- @@ -1101,13 +1698,14 @@ def plot_2pcf(self): ) plt.xscale("log") plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") + plt.legend() plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\xi_-(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_m.png") - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"xi_minus plot saved to {out_path}") # Plot of xi_+(theta) * theta @@ -1122,14 +1720,14 @@ def plot_2pcf(self): color=self.cc[ver]["colour"], ) plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") + plt.legend() plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\theta \xi_+(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_p_theta.png") - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"xi_plus_theta plot saved to {out_path}") # Plot of xi_- * theta @@ -1144,57 +1742,121 @@ def plot_2pcf(self): color=self.cc[ver]["colour"], ) plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") + plt.legend() plt.ticklabel_format(axis="y") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\theta \xi_-(\theta)$") out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_m_theta.png") - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"xi_minus_theta plot saved to {out_path}") # Plot of xi_+ with and without xi_psf_sys + # but skip if xi_psf_sys is not calculated since that takes forever + if hasattr(self, "_xi_psf_sys"): + for idx, ver in enumerate(self.versions): + fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) + plt.errorbar( + self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), + self.cat_ggs[ver].xip, + yerr=np.sqrt(self.cat_ggs[ver].varxim), + label=r"$\xi_+$", + ls="solid", + color="green", + ) + plt.errorbar( + self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), + self.xi_psf_sys[ver]["mean"], + yerr=np.sqrt(self.xi_psf_sys[ver]["var"]), + label=r"$\xi^{\rm psf}_{+, {\rm sys}}$", + ls="dotted", + color="red", + ) + plt.errorbar( + self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), + self.cat_ggs[ver].xip + self.xi_psf_sys[ver]["mean"], + yerr=np.sqrt( + self.cat_ggs[ver].varxip + self.xi_psf_sys[ver]["var"] + ), + label=r"$\xi_+ + \xi^{\rm psf}_{+, {\rm sys}}$", + ls="dashdot", + color="magenta", + ) + + plt.xscale("log") + plt.yscale("log") + plt.legend() + plt.ticklabel_format(axis="y") + plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") + plt.xlim([self.theta_min_plot, self.theta_max_plot]) + plt.ylim(1e-8, 5e-4) + plt.ylabel(r"$\xi_+(\theta)$") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/xi_p_xi_psf_sys_{ver}.png" + ) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() + self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") + + def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): + + fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(10, 7)) + + for idx, ver in enumerate(self.versions): - fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(7, 7)) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip, - yerr=np.sqrt(self.cat_ggs[ver].varxim), - label=r"$\xi_+$", - ls="solid", - color="green", - ) - plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt(self.xi_psf_sys[ver]["var"]), - label=r"$\xi^{\rm psf}_{+, {\rm sys}}$", - ls="dotted", - color="red", + xi_psf_sys = self.xi_psf_sys[ver] + gg = self.cat_ggs[ver] + + ratio = xi_psf_sys["mean"] / gg.xip + ratio_err = np.sqrt( + (np.sqrt(xi_psf_sys["var"]) / gg.xip) ** 2 + + (xi_psf_sys["mean"] * np.sqrt(gg.varxip) / gg.xip**2) ** 2 ) + + theta = gg.meanr + jittered_theta = theta * (1+idx*offset) + plt.errorbar( - self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), - self.cat_ggs[ver].xip + self.xi_psf_sys[ver]["mean"], - yerr=np.sqrt(self.cat_ggs[ver].varxip + self.xi_psf_sys[ver]["var"]), - label=r"$\xi_+ + \xi^{\rm psf}_{+, {\rm sys}}$", - ls="dashdot", - color="magenta", + jittered_theta, + ratio, + yerr=ratio_err, + label=ver, + ls=self.cc[ver]["ls"], + color=self.cc[ver]["colour"], + fmt=self.cc[ver].get("marker", None), + capsize=5 ) - plt.xscale("log") - plt.yscale("log") - plt.legend(fontsize=20, bbox_to_anchor=(1.05, 1), loc="upper left") - plt.ticklabel_format(axis="y") - plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") - plt.xlim([self.theta_min_plot, self.theta_max_plot]) - plt.ylim(1e-8, 5e-4) - plt.ylabel(r"$\xi_+(\theta)$") - out_path = os.path.abspath( - f"{self.cc['paths']['output']}/xi_p_xi_psf_sys_{ver}.png" - ) - cs_plots.savefig(out_path) - self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") + plt.fill_between( + [self.theta_min_plot, self.theta_max_plot], + - threshold, + + threshold, + color="black", + alpha=0.1, + label=f"{threshold:.0%} threshold", + ) + plt.plot( + [self.theta_min_plot, self.theta_max_plot], + [threshold, threshold], + ls="dashed", + color="black") + plt.plot( + [self.theta_min_plot, self.theta_max_plot], + [-threshold, -threshold], + ls="dashed", + color="black") + plt.xscale("log") + plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") + plt.ylabel(r"$\xi^{\rm psf}_{+, {\rm sys}} / \xi_+$") + plt.gca().yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1)) + plt.legend() + plt.title("Ratio of PSF systematics to cosmic shear signal") + out_path = os.path.abspath(f"{self.cc['paths']['output']}/ratio_xi_sys_xi.png") + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() + print(f"Ratio of xi_psf_sys to xi plot saved to {out_path}") + def calculate_aperture_mass_dispersion( self, theta_min=0.3, theta_max=200, nbins=500, nbins_map=15, npatch=25 @@ -1224,7 +1886,7 @@ def calculate_aperture_mass_dispersion( self.print_green(f"Skipping xi for Map2, {out_fname} exists") gg.read(out_fname) else: - with self.results[ver].temporarily_load_data(): + with self.results[ver].temporarily_read_data(): R = self.cc[ver]["shear"]["R"] g1 = ( self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] @@ -1315,7 +1977,8 @@ def plot_aperture_mass_dispersion(self): linestyles=linestyles, shift_x=True, ) - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"linear-scale {mode} plot saved to {out_path}") for mode in ["mapsq", "mapsq_im", "mxsq", "mxsq_im"]: @@ -1347,395 +2010,601 @@ def plot_aperture_mass_dispersion(self): linestyles=linestyles, shift_x=True, ) - cs_plots.savefig(out_path) + cs_plots.savefig(out_path, close_fig=False) + cs_plots.show() self.print_done(f"log-scale {mode} plot saved to {out_path}") def calculate_pure_eb( self, - theta_min=0.1, - theta_max=250, - nbins=20, - theta_min_int=0.04, - theta_max_int=500, - nbins_int=1000, + version, + min_sep=None, + max_sep=None, + nbins=None, + min_sep_int=0.08, + max_sep_int=300, + nbins_int=100, + npatch=256, + var_method="jackknife", + cov_path_int=None, + cosmo_cov=None, + n_samples=1000, ): - self.print_start("Computing pure E/B") + """ + Calculate the pure E/B modes for the given catalog version. + The class instance's treecorr_config will be used for the "reporting" binning + by default, but any kwargs passed to this function will overwrite the defaults. + Parameters + ---------- + version : str + The catalog version to compute the pure E/B modes for. + min_sep : float, optional + Minimum separation for the reporting binning. Defaults to the value in + self.treecorr_config if not provided. + max_sep : float, optional + Maximum separation for the reporting binning. Defaults to the value in + self.treecorr_config if not provided. + nbins : int, optional + Number of bins for the reporting binning. Defaults to the value in + self.treecorr_config if not provided. + min_sep_int : float, optional + Minimum separation for the integration binning. Defaults to 0.08. + max_sep_int : float, optional + Maximum separation for the integration binning. Defaults to 300. + nbins_int : int, optional + Number of bins for the integration binning. Defaults to 100. + npatch : int, optional + Number of patches for the jackknife or bootstrap resampling. Defaults to + the value in self.npatch if not provided. + var_method : str, optional + Variance estimation method. Defaults to "jackknife". + cov_path_int : str, optional + Path to the covariance matrix for the reporting binning. Replaces the + treecorr covariance matrix if provided, meaning that var_method has no + effect on the results although it is still passed to + CosmologyValidation.calculate_2pcf. + cosmo_cov : pyccl.Cosmology, optional + Cosmology object to use for theoretical xi+/xi- predictions in the + semi-analytical covariance calculation. Defaults to self.cosmo if not + provided. + n_samples : int, optional + Number of Monte Carlo samples for semi-analytical covariance propagation. + Defaults to 1000. + + Returns + ------- + dict + A dictionary containing the following keys: + - "xip_E": Pure E-mode correlation function for xi+. + - "xim_E": Pure E-mode correlation function for xi-. + - "xip_B": Pure B-mode correlation function for xi+. + - "xim_B": Pure B-mode correlation function for xi-. + - "xip_amb": Ambiguity mode for xi+. + - "xim_amb": Ambiguity mode for xi-. + - "cov": Covariance matrix for the pure E/B modes. + - "gg": The two-point correlation function object for the reporting binning. + - "gg_int": The two-point correlation function object for the + integration binning. + - "eb_samples": (only when using semi-analytical covariance) Semi-analytic + EB samples used for covariance calculation. Shape: (n_samples, 6*nbins) + + Notes + ----- + - A shared patch file is used for the reporting and integration binning, + and is created if it does not exist. + """ + from .b_modes import calculate_pure_eb_correlation + + self.print_start(f"Computing {version} pure E/B") + + # Set up parameters with defaults + npatch = npatch or self.npatch + min_sep = min_sep or self.treecorr_config["min_sep"] + max_sep = max_sep or self.treecorr_config["max_sep"] + nbins = nbins or self.treecorr_config["nbins"] + + # Create TreeCorr configurations treecorr_config = { **self.treecorr_config, - "min_sep": theta_min, - "max_sep": theta_max, + "min_sep": min_sep, + "max_sep": max_sep, "nbins": nbins, } - # nbins_int = (nbins - 1) * integration_oversample + 1 treecorr_config_int = { **treecorr_config, - "min_sep": theta_min_int, - "max_sep": theta_max_int, + "min_sep": min_sep_int, + "max_sep": max_sep_int, "nbins": nbins_int, } - print("Integration correlation function parameters:") - print(treecorr_config_int) - - print("Output correlation function parameters:") - print(treecorr_config) + # Calculate correlation functions + gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) + gg_int = self.calculate_2pcf(version, npatch=npatch, **treecorr_config_int) - for ver in self.versions: - self.print_magenta(ver) - - gg_int = treecorr.GGCorrelation(treecorr_config_int) - gg = treecorr.GGCorrelation(treecorr_config) - - out_fname = os.path.abspath( - # f"{self.cc['paths']['output']}/xi_for_pure_eb_{ver}.txt" - f"{self.cc['paths']['output']}/xi_for_pure_eb_thetamin={theta_min}_thetamax={theta_max}_nbins={nbins}_npatch={self.npatch}_{ver}.txt" - ) - out_fname_int = os.path.abspath( - f"{self.cc['paths']['output']}/xi_for_pure_eb_int_thetaminint={theta_min_int}_thetamaxint_{theta_max_int}_nbinsint={nbins_int}_npatchint={self.npatch}_{ver}.txt" - # f"{self.cc['paths']['output']}/xi_for_pure_eb_{ver}_int.txt" - ) - if os.path.exists(out_fname) and os.path.exists(out_fname_int): - self.print_green( - f"Skipping xi for COSEBIs:\n{out_fname}\n{out_fname_int}\nexist." - ) - gg.read(out_fname) - gg_int.read(out_fname_int) - else: - with self.results[ver].temporarily_load_data(): - R = self.cc[ver]["shear"]["R"] - g1 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - - self.c1[ver] - ) / R - g2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] - - self.c2[ver] - ) / R - - cat = treecorr.Catalog( - ra=self.results[ver].dat_shear["RA"], - dec=self.results[ver].dat_shear["Dec"], - g1=g1, - g2=g2, - w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], - ra_units=self.treecorr_config["ra_units"], - dec_units=self.treecorr_config["dec_units"], - npatch=self.npatch, - ) - del g1, g2, R - - self.print_cyan("Integration binning") - gg_int.process(cat) - gg_int.write(out_fname_int, write_patch_results=True, write_cov=True) - - self.print_cyan("Output binning") - gg.process(cat) - gg.write(out_fname, write_patch_results=True, write_cov=True) - - def pure_EB(corrs): - gg, gg_int = corrs - return get_pure_EB_modes( - theta=gg.meanr, - xip=gg.xip, - xim=gg.xim, - theta_int=gg_int.meanr, - xip_int=gg_int.xip, - xim_int=gg_int.xim, - tmin=theta_min, - tmax=theta_max, - ) - - xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb = pure_EB([gg, gg_int]) - cov = treecorr.estimate_multi_cov( - [gg, gg_int], "jackknife", func=lambda x: np.hstack(pure_EB(x)) - ) + # Get redshift distribution if using analytic covariance + if cov_path_int is not None: + z, nz = self.get_redshift(version) + z_dist = np.column_stack([z, nz]) + else: + z_dist = None + + # Delegate to b_modes module + results = calculate_pure_eb_correlation( + gg=gg, + gg_int=gg_int, + var_method=var_method, + cov_path_int=cov_path_int, + cosmo_cov=cosmo_cov, + n_samples=n_samples, + z_dist=z_dist + ) - results = { - "xip_E": xip_E, - "xim_E": xim_E, - "xip_B": xip_B, - "xim_B": xim_B, - "xip_amb": xip_amb, - "xim_amb": xim_amb, - "cov": cov, - } + return results - return results + def plot_pure_eb( + self, + versions=None, + output_dir=None, + fiducial_xip_scale_cut=None, + fiducial_xim_scale_cut=None, + min_sep=None, + max_sep=None, + nbins=None, + min_sep_int=0.08, + max_sep_int=300, + nbins_int=100, + npatch=None, + var_method="jackknife", + cov_path_int=None, + cosmo_cov=None, + n_samples=1000, + results=None, + **kwargs + ): + """ + Generate comprehensive pure E/B mode analysis plots. -class FootprintPlotter: - """Class to create footprint plots. - - Parameters - ----------- - nside_coverage: int, optional - basic resolution of map; default is 32 - nside_map: - fine resolution for plotting; default is 2048 + Creates four types of plots for each version: + 1. Integration vs Reporting comparison + 2. E/B/Ambiguous correlation functions + 3. 2D PTE heatmaps + 4. Covariance matrix visualization - """ - - # Dictionary storing region parameters - _regions = { - "NGC": {"ra_0": 180, "extend": [120, 270, 20, 70], "vmax": 60}, - "SGC": {"ra_0": 15, "extend": [-20, 45, 20, 45], "vmax": 60}, - "fullsky": {"ra_0": 150, "extend": [0, 360, -90, 90], "vmax": 60}, - } - - def __init__(self, nside_coverage=32, nside_map=2048): - - self._nside_coverage = nside_coverage - self._nside_map = nside_map - - def create_hsp_map(self, ra, dec): - """Create Hsp Map. - - Create healsparse map. - Parameters ---------- - ra : numpy.ndarray - right ascension values - dec : numpy.ndarray - declination values - - Returns - ------- - hsp.HealSparseMap - map - + versions : list, optional + List of catalog versions to process. Uses self.versions if None. + output_dir : str, optional + Output directory for plots. Uses configured output path if None. + fiducial_xip_scale_cut : tuple, optional + (min_scale, max_scale) for xi+ fiducial analysis, shown as gray regions + fiducial_xim_scale_cut : tuple, optional + (min_scale, max_scale) for xi- fiducial analysis, shown as gray regions + min_sep, max_sep, nbins : float, float, int, optional + Binning parameters for reporting scale. Uses treecorr_config if None. + min_sep_int, max_sep_int, nbins_int : float, float, int + Binning parameters for integration scale + (default: 0.08-300 arcmin, 100 bins) + npatch : int, optional + Number of patches for jackknife covariance. Uses self.npatch if None. + var_method : str + Variance method ("jackknife" or "semi-analytic"). + Automatically set to "semi-analytic" when cov_path_int is provided. + cov_path_int : str, optional + Path to integration covariance matrix for semi-analytical calculation + cosmo_cov : pyccl.Cosmology, optional + Cosmology for theoretical predictions in semi-analytical covariance + n_samples : int + Number of Monte Carlo samples for semi-analytical covariance (default: 1000) + results : dict or list, optional + Precalculated results to avoid recomputation. Can be a single results dict + for one version, or a list of results dicts for multiple versions. + If None (default), results will be calculated using calculate_pure_eb. + **kwargs : dict + Additional arguments passed to calculate_eb_statistics + + Notes + ----- + This function orchestrates the full E/B mode analysis workflow: + - Uses instance configuration as defaults for unspecified parameters + - Automatically switches to analytical variance when theoretical + covariance provided + - Generates standardized output file naming based on all analysis + parameters + - Delegates individual plot generation to specialized functions in + b_modes module """ - # Create empty map - hsp_map = hsp.HealSparseMap.make_empty( - self._nside_coverage, - self._nside_map, - dtype=np.float32, - sentinel=np.nan + from .b_modes import ( + calculate_eb_statistics, + plot_eb_covariance_matrix, + plot_integration_vs_reporting, + plot_pte_2d_heatmaps, + plot_pure_eb_correlations, ) - # Get pixel list corresponding to coordinates - hpix = hp.ang2pix(self._nside_map, ra, dec, nest=True, lonlat=True) + # Use instance defaults for unspecified parameters + versions = versions or self.versions + output_dir = output_dir or self.cc['paths']['output'] + npatch = npatch or self.npatch + + # Override var_method to analytic when cov_path_int is provided + if cov_path_int is not None: + var_method = "semi-analytic" + + # Use treecorr_config defaults for reporting scale binning + min_sep = min_sep or self.treecorr_config["min_sep"] + max_sep = max_sep or self.treecorr_config["max_sep"] + nbins = nbins or self.treecorr_config["nbins"] + + # Handle results parameter - convert to list format for consistent processing + if results is not None: + if isinstance(results, dict): + # Single results dict provided - should match single version + if len(versions) != 1: + raise ValueError( + "Single results dict provided but multiple versions specified. " + "Provide results list matching versions length." + ) + results_list = [results] + elif isinstance(results, list): + # List of results provided + if len(results) != len(versions): + raise ValueError( + f"Results list length ({len(results)}) does not match versions " + f"length ({len(versions)})" + ) + results_list = results + else: + raise TypeError("Results must be dict, list, or None") + else: + results_list = [None] * len(versions) + + for idx, version in enumerate(versions): + # Generate standardized output filename stub + out_stub = ( + f"{output_dir}/{version}_eb_minsep={min_sep}_" + f"maxsep={max_sep}_nbins={nbins}_minsepint={min_sep_int}_" + f"maxsepint={max_sep_int}_nbinsint={nbins_int}_npatch={npatch}_" + f"varmethod={var_method}" + ) - # Get count of objects per pixel - pixel_counts = Counter(hpix) + # Get or calculate results for this version + version_results = results_list[idx] or self.calculate_pure_eb( + version, + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + min_sep_int=min_sep_int, + max_sep_int=max_sep_int, + nbins_int=nbins_int, + npatch=npatch, + var_method=var_method, + cov_path_int=cov_path_int, + cosmo_cov=cosmo_cov, + n_samples=n_samples, + ) - # List of unique pixels - unique_hpix = np.array(list(pixel_counts.keys())) + # Calculate E/B statistics for all bin combinations (only if not provided) + version_results = calculate_eb_statistics( + version_results, + cov_path_int=cov_path_int, + n_samples=n_samples, + **kwargs + ) - # Number of objects - values = np.array(list(pixel_counts.values()), dtype=np.float32) + # Generate all plots using specialized plotting functions + gg, gg_int = version_results["gg"], version_results["gg_int"] - # Create maps with numbers per pixel - hsp_map[unique_hpix] = values - - return hsp_map - - def plot_area( - self, - hsp_map, - ra_0=0, - extend=[120, 270, 29, 70], - vmax=60, - projection=None, - outpath=None, - title=None, - ): - """Plot Area. - - Plot catalogue in an area on the sky. - - Parameters - ---------- - hsp_map : hsp_HealSparseMap - input map - ra_0 : float, optional - anchor point in R.A.; default is 0 - extend : list, optional - sky region, extend=[ra_low, ra_high, dec_low, dec_high]; - default is [120, 270, 29, 70] - vmax : float, optional - maximum pixel value to plot with color; default is 60 - projection : skyproj.McBrydeSkyproj - if ``None`` (default), a new plot is created - outpath : str, optional - output path, default is ``None`` - title : str, optional - print title if not ``None`` (default) - - Returns - -------- - skyproj.McBrydeSkyproj - projection instance - plt.axes.Axes - axes instance - - Raises - ------ - ValueError - if no object found in region - - """ - if not projection: - - # Create new figure and axes - fig, ax = plt.subplots(figsize=(10, 10)) - - # Create new projection - projection = skyproj.McBrydeSkyproj( - ax=ax, - lon_0=ra_0, - extent=extend, - autorescale=True, - vmax=vmax + # Integration vs Reporting comparison plot + plot_integration_vs_reporting( + gg, gg_int, + out_stub + "_integration_vs_reporting.png", + version ) - else: - ax = None - try: - _ = projection.draw_hspmap( - hsp_map, lon_range=extend[0:2], - lat_range=extend[2:] + # E/B/Ambiguous correlation functions plot + plot_pure_eb_correlations( + version_results, + out_stub + "_xis.png", + version, + fiducial_xip_scale_cut=fiducial_xip_scale_cut, + fiducial_xim_scale_cut=fiducial_xim_scale_cut ) - except ValueError: - msg = "No object found in region to draw" - print(f"{msg}, continuing...") - #raise ValueError(msg) - - projection.draw_milky_way(width=25, linewidth=1.5, color='black', linestyle='-') - - if title: - plt.title(title, pad=5) - if outpath: - plt.savefig(outpath) - - return projection, ax - - def plot_region(self, hsp_map, region, projection=None, outpath=None, title=None): - - return self.plot_area( - hsp_map, - region["ra_0"], - region["extend"], - region["vmax"], - projection=projection, - outpath=outpath, - title=title, - ) + # 2D PTE heatmaps plot + plot_pte_2d_heatmaps( + version_results, + version, + out_stub + "_ptes.png", + fiducial_xip_scale_cut=fiducial_xip_scale_cut, + fiducial_xim_scale_cut=fiducial_xim_scale_cut + ) - def plot_all_regions(self, hsp_map, outbase=None): + # Covariance matrix plot + plot_eb_covariance_matrix( + version_results["cov"], + var_method, + out_stub + "_covariance.png", + version + ) - for region in self._regions: - if outbase: - outpath = f"{outbase}_{region}.png" - else: - outpath = None - self.plot_region(hsp_map, self._regions[region], outpath=outpath) - + def calculate_cosebis( + self, + version, + min_sep_int=0.5, + max_sep_int=500, + nbins_int=1000, + npatch=None, + nmodes=10, + cov_path=None, + evaluate_all_scale_cuts=False, + min_sep=None, + max_sep=None, + nbins=None, + ): + """ + Calculate COSEBIs from a finely-binned correlation function. - @classmethod - def hp_pixel_centers(cls, nside, nest=False): - - # Get number of pixels for given nside - npix = hp.nside2npix(nside) - - # Get pixel indices - pix_indices = np.arange(npix) - - # Get coordinates of pixel centers - ra, dec = hp.pix2ang(nside, pix_indices, nest=nest, lonlat=True) + COSEBIs fundamentally require fine binning for accurate transformations. + This function computes a single, finely-binned correlation function using + integration binning parameters and can evaluate either a single scale cut + (full range) or multiple scale cuts systematically. - return ra, dec, npix + Parameters + ---------- + version : str + The catalog version to compute the COSEBIs for. + min_sep_int : float, optional + Minimum separation for integration binning (fine binning for COSEBIs). + Defaults to 0.5 arcmin. + max_sep_int : float, optional + Maximum separation for integration binning (fine binning for COSEBIs). + Defaults to 500 arcmin. + nbins_int : int, optional + Number of bins for integration binning (fine binning for COSEBIs). + Defaults to 1000. + npatch : int, optional + Number of patches for the jackknife resampling. Defaults to self.npatch. + nmodes : int, optional + Number of COSEBIs modes to compute. Defaults to 10. + cov_path : str, optional + Path to theoretical covariance matrix. When provided, enables analytic + covariance calculation. + evaluate_all_scale_cuts : bool, optional + If True, evaluates COSEBIs for all possible scale cut combinations + using the reporting binning parameters. If False, uses the full + integration range as a single scale cut. Defaults to False. + min_sep : float, optional + Minimum separation for reporting binning (only used when + evaluate_all_scale_cuts=True). Defaults to self.treecorr_config["min_sep"]. + max_sep : float, optional + Maximum separation for reporting binning (only used when + evaluate_all_scale_cuts=True). Defaults to self.treecorr_config["max_sep"]. + nbins : int, optional + Number of bins for reporting binning (only used when + evaluate_all_scale_cuts=True). Defaults to self.treecorr_config["nbins"]. - @classmethod - def plot_footprint_as_hp(cls, hsp_map, nside, outpath=None, title=None): + Returns + ------- + dict + When evaluate_all_scale_cuts=False: Dictionary containing COSEBIs results + with E/B modes, covariances, and statistics for the full range. + When evaluate_all_scale_cuts=True: Dictionary with scale cut tuples as + keys and results dictionaries as values, containing results for all + possible scale cut combinations. + + Notes + ----- + """ + from .b_modes import calculate_cosebis - ra, dec, npix = cls.hp_pixel_centers(nside) + self.print_start(f"Computing {version} COSEBIs") - # Create an empty HEALPix map - m = np.full(npix, np.nan) + # Set up parameters with defaults + npatch = npatch or self.npatch - fig, ax = plt.subplots(figsize=(10, 10)) + # Always use integration binning for COSEBIs calculation (fine binning) + treecorr_config = { + **self.treecorr_config, + "min_sep": min_sep_int, + "max_sep": max_sep_int, + "nbins": nbins_int, + } - # Plot the HEALPix grid - hp.mollview(m, title=title, coord="C", notext=True, rot=(180, 0, 0)) - - # Define the Galactic Plane: l = [0, 360], b = 0° - for l0, ls in zip((-5, 0, 5), (":", "-", ":")): - l_values = np.linspace(0, 360, 500) # 500 points along the plane - b_values = np.zeros_like(l_values) # Galactic latitude is 0 (the plane) + # Calculate single fine-binned correlation function for COSEBIs + print( + f"Computing fine-binned 2PCF with {nbins_int} bins from {min_sep_int} to " + f"{max_sep_int} arcmin" + ) + gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) + + if evaluate_all_scale_cuts: + # Use reporting binning parameters or inherit from class config + min_sep = min_sep or self.treecorr_config["min_sep"] + max_sep = max_sep or self.treecorr_config["max_sep"] + nbins = nbins or self.treecorr_config["nbins"] + + # Generate scale cuts using np.geomspace (no TreeCorr needed) + bin_edges = np.geomspace(min_sep, max_sep, nbins + 1) + scale_cuts = [ + (bin_edges[start], bin_edges[stop]) + for start in range(nbins) + for stop in range(start+1, nbins+1) + ] + + print(f"Evaluating {len(scale_cuts)} scale cut combinations") + + # Call b_modes function with scale cuts list + results = calculate_cosebis( + gg=gg, nmodes=nmodes, scale_cuts=scale_cuts, cov_path=cov_path + ) + else: + # Single scale cut behavior: use full range + results = calculate_cosebis( + gg=gg, nmodes=nmodes, scale_cuts=None, cov_path=cov_path + ) + # Extract single results dict from scale_cuts dictionary + results = list(results.values())[0] - # Convert (l, b) to (λ, β) - Ecliptic coordinates - coords = SkyCoord(l=l_values*u.degree, b=b_values*u.degree, frame='galactic') - ecl_coords = coords.transform_to('barycentrictrueecliptic') # Ecliptic frame + return results - # Extract Ecliptic longitude (λ) and latitude (β) - lambda_ecl = ecl_coords.lon.deg # Ecliptic longitude - beta_ecl = ecl_coords.lat.deg # Ecliptic latitude + def plot_cosebis( + self, + version=None, + output_dir=None, + min_sep_int=0.5, max_sep_int=500, nbins_int=1000, # Integration binning + npatch=None, nmodes=10, cov_path=None, + evaluate_all_scale_cuts=False, # New parameter + min_sep=None, max_sep=None, nbins=None, # Reporting binning + fiducial_scale_cut=None, # For plotting reference + results=None, + ): + """ + Generate comprehensive COSEBIs analysis plots for a single version. - # Convert to HEALPix projection coordinates (colatitude, longitude) - theta = np.radians(90 - beta_ecl) # HEALPix uses colatitude - phi = np.radians(lambda_ecl) # HEALPix uses longitude + Creates two types of plots: + 1. COSEBIs E/B mode correlation functions + 2. COSEBIs covariance matrix - # Create a healpy Mollweide projection in Ecliptic coordinates - hp.projplot(theta, phi, linestyle=ls, color='black', linewidth=1) # Plot the outline + Parameters + ---------- + version : str, optional + Version string to process. Defaults to first version in self.versions. + output_dir : str, optional + Output directory for plots. Defaults to self.cc['paths']['output']. + min_sep_int, max_sep_int, nbins_int : float, float, int + Integration binning parameters for correlation function + (default: 0.5, 500, 1000) + npatch : int, optional + Number of patches for jackknife covariance. Defaults to instance value. + nmodes : int + Number of COSEBIs modes to compute (default: 10) + cov_path : str, optional + Path to theoretical covariance matrix. When provided, analytic + covariance is used. + evaluate_all_scale_cuts : bool + Whether to evaluate all scale cuts (default: False) + min_sep, max_sep, nbins : float, float, int, optional + Reporting binning parameters. Only used when evaluate_all_scale_cuts=True. + fiducial_scale_cut : tuple, optional + (min_scale, max_scale) reference scale cut for plotting when + evaluate_all_scale_cuts=True + results : dict, optional + Precalculated results to avoid recomputation. If None (default), + results will be calculated using calculate_cosebis. + + Notes + ----- + This function orchestrates the full COSEBIs analysis workflow: + - Uses instance configuration as defaults for unspecified parameters + - Calculates COSEBIs for the version using the updated parameter interface + - Generates mode plots and covariance visualization + - Output files are named with analysis parameters for reproducibility + - When evaluate_all_scale_cuts=True, results contain multiple scale cuts; + fiducial_scale_cut determines which one is used for plotting + """ + from .b_modes import ( + find_conservative_scale_cut_key, + plot_cosebis_covariance_matrix, + plot_cosebis_modes, + plot_cosebis_scale_cut_heatmap, + ) - # Apply mask - mask_values = hsp_map.get_values_pos(ra, dec, valid_mask=True, lonlat=True) + # Use instance defaults if not specified + version = version or self.versions[0] + output_dir = output_dir or self.cc['paths']['output'] + npatch = npatch or self.treecorr_config.get('npatch', 256) - ok = np.where(mask_values == False)[0] - #nok = np.where(mask_values == False)[0] + # Determine variance method based on whether theoretical covariance is used + var_method = "analytic" if cov_path is not None else "jackknife" - hp.projscatter(ra[ok], dec[ok], lonlat=True, color="green", s=1, marker=".") - #hp.projscatter(ra[nok], dec[nok], lonlat=True, color="red", s=1, marker=".") - - plt.tight_layout() - - if outpath: - plt.savefig(outpath) - - plt.show() + # Create output filename with integration parameters to match Snakemake + out_stub = ( + f"{output_dir}/{version}_cosebis_minsep={min_sep_int}_" + f"maxsep={max_sep_int}_nbins={nbins_int}_npatch={npatch}_" + f"varmethod={var_method}_nmodes={nmodes}" + ) -def hsp_map_logical_or(maps, verbose=False): - """ - Hsp Map Logical Or. - - Logical AND of HealSparseMaps. - - """ - if verbose: - print("Combine all maps...") - - # Ensure consistency in coverage and data type - nside_coverage = maps[0].nside_coverage - nside_sparse = maps[0].nside_sparse - dtype = maps[0].dtype - - for m in maps: - # MKDEBUG TODO: Change nside if possible - if m.nside_coverage != nside_coverage: - raise ValueError( - f"Coverage nside={m.nside_coverage} does not match {nside_coverage}" + # Add scale cut info if provided + if fiducial_scale_cut is not None: + out_stub += f"_scalecut={fiducial_scale_cut[0]}-{fiducial_scale_cut[1]}" + + # if evaluate_all_scale_cuts: + # out_stub += f"_allcuts_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}" + + # Get or calculate results for this version + if results is None: + # Calculate COSEBIs using instance method + results = self.calculate_cosebis( + version, + min_sep_int=min_sep_int, + max_sep_int=max_sep_int, + nbins_int=nbins_int, + npatch=npatch, + nmodes=nmodes, + cov_path=cov_path, + evaluate_all_scale_cuts=evaluate_all_scale_cuts, + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, ) - if m.dtype != dtype: - raise ValueError( - f"Data type {m.dtype} does not match {dtype}" + + # Generate plots using specialized plotting functions + # Extract single result for plotting if multiple scale cuts were evaluated + if (isinstance(results, dict) and + all(isinstance(k, tuple) for k in results.keys())): + # Multiple scale cuts: use fiducial_scale_cut if provided, otherwise use + # full range + if fiducial_scale_cut is not None: + plot_results = results[ + find_conservative_scale_cut_key(results, fiducial_scale_cut) + ] + else: + # Use full range result (largest scale cut) + max_range_key = max(results.keys(), key=lambda x: x[1] - x[0]) + plot_results = results[max_range_key] + else: + # Single result + plot_results = results + + plot_cosebis_modes( + plot_results, + version, + out_stub + "_cosebis.png", + fiducial_scale_cut=fiducial_scale_cut + ) + + plot_cosebis_covariance_matrix( + plot_results, + version, + var_method, + out_stub + "_covariance.png" + ) + + # Generate scale cut heatmap if we have multiple scale cuts + if (isinstance(results, dict) and + all(isinstance(k, tuple) for k in results.keys()) and + len(results) > 1): + # Create temporary gg object with correct binning for mapping + treecorr_config_temp = { + **self.treecorr_config, + "min_sep": min_sep or self.treecorr_config["min_sep"], + "max_sep": max_sep or self.treecorr_config["max_sep"], + "nbins": nbins or self.treecorr_config["nbins"], + } + gg_temp = self.calculate_2pcf( + version, npatch=npatch, **treecorr_config_temp ) - # Create an empty HealSparse map - map_comb = hsp.HealSparseMap.make_empty(nside_coverage, nside_sparse, dtype=dtype) - for idx, m in enumerate(maps): - map_comb |= m - - if verbose: - valid_pixels = map_comb.valid_pixels - n_tot = np.sum(valid_pixels) - n_true = np.count_nonzero(valid_pixels) - n_false = n_tot - n_true - print(f"after map {idx}: frac_true={n_true / n_tot:g}, frac_false={n_false / n_tot:g}") + plot_cosebis_scale_cut_heatmap( + results, + gg_temp, + version, + out_stub + "_scalecut_ptes.png", + fiducial_scale_cut=fiducial_scale_cut + ) - return map_comb def calculate_pseudo_cl_eb_cov(self): """ @@ -1752,7 +2621,7 @@ def calculate_pseudo_cl_eb_cov(self): for ver in self.versions: self.print_magenta(ver) - if not ver in self._pseudo_cls.keys(): + if ver not in self._pseudo_cls.keys(): self._pseudo_cls[ver] = {} out_path = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_{ver}.fits") @@ -1760,15 +2629,32 @@ def calculate_pseudo_cl_eb_cov(self): self.print_done(f"Skipping Pseudo-Cl covariance calculation, {out_path} exists") self._pseudo_cls[ver]['cov'] = fits.open(out_path) else: - - params = utils_cosmo_val.get_params_rho_tau(self.cc[ver], survey=ver) + params = get_params_rho_tau(self.cc[ver], survey=ver) self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") - lmax = 2*self.nside - path_redshift_distr = self.data_base_dir + self.cc[ver]["shear"]["redshift_distr"] + lmax = 2 * self.nside + z, dndz = self.get_redshift(ver) + ell = np.arange(1, lmax + 1) pw = hp.pixwin(nside, lmax=lmax) - fiducial_cl = self.get_fiducial(lmax, path_redshift_distr)*pw**2 + if pw.shape[0] != len(ell) + 1: + raise ValueError( + "Unexpected pixwin length for lmax=" + f"{lmax}: got {pw.shape[0]}, expected {len(ell)+1}" + ) + pw = pw[1:len(ell)+1] + + # Load redshift distribution and calculate theory C_ell + fiducial_cl = ( + get_theo_c_ell( + ell=ell, + z=z, + nz=dndz, + backend="camb", + cosmo=self.cosmo, + ) + * pw**2 + ) self.print_cyan("Getting a sample of the fiducial Cl's with noise") @@ -1776,12 +2662,14 @@ def calculate_pseudo_cl_eb_cov(self): lmax = 2*self.nside b_lmax = lmax - 1 + ells = np.arange(lmin, lmax+1) + if self.binning == 'linear': - step = 10 - b = nmt.NmtBin.from_nside_linear(self.nside, step) + # Linear bands of width ell_step, respecting actual lmax + bpws = (ells - lmin) // self.ell_step + bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all + b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) elif self.binning == 'powspace': - ells = np.arange(lmin, lmax+1) - start = np.power(lmin, self.power) end = np.power(lmax, self.power) bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) @@ -1858,6 +2746,150 @@ def calculate_pseudo_cl_eb_cov(self): self.print_done("Done Pseudo-Cl covariance") + def calculate_pseudo_cl_onecovariance(self): + """ + Compute the pseudo-Cl covariance using OneCovariance. + """ + self.print_start("Computing Pseudo-Cl covariance with OneCovariance") + + if self.path_onecovariance is None: + raise ValueError("path_onecovariance must be provided to use OneCovariance") + + if not os.path.exists(self.path_onecovariance): + raise ValueError(f"OneCovariance path {self.path_onecovariance} does not exist") + + template_config = os.path.join(self.path_onecovariance, "config_files", "config_3x2pt_pure_Cell_UNIONS.ini") + if not os.path.exists(template_config): + raise ValueError(f"Template config file {template_config} does not exist") + + self._pseudo_cls_onecov = {} + for ver in self.versions: + self.print_magenta(ver) + + out_dir = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_onecov_{ver}/") + os.makedirs(out_dir, exist_ok=True) + + if os.path.exists(os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat")): + self.print_done(f"Skipping OneCovariance calculation, {out_dir} exists") + self._load_onecovariance_cov(out_dir, ver) + else: + + mask_path = self.cc[ver]['shear']['mask'] + redshift_distr_path = os.path.join(self.data_base_dir, self.cc[ver]['shear']['redshift_distr']) + + config_path = os.path.join(out_dir, f"config_onecov_{ver}.ini") + + self.print_cyan(f"Modifying OneCovariance config file and saving it to {config_path}") + self._modify_onecov_config(template_config, config_path, out_dir, mask_path, redshift_distr_path, ver) + + self.print_cyan("Running OneCovariance...") + cmd = f"python {os.path.join(self.path_onecovariance, 'covariance.py')} {config_path}" + self.print_cyan(f"Command: {cmd}") + ret = os.system(cmd) + if ret != 0: + raise RuntimeError(f"OneCovariance command failed with return code {ret}") + self.print_cyan("OneCovariance completed successfully.") + self._load_onecovariance_cov(out_dir, ver) + + self.print_done("Done Pseudo-Cl covariance with OneCovariance") + + def _modify_onecov_config(self, template_config, config_path, out_dir, mask_path, redshift_distr_path, ver): + """ + Modify OneCovariance configuration file with correct mask, redshift distribution, + and ellipticity dispersion parameters. + + Parameters + ---------- + template_config : str + Path to the template configuration file + config_path : str + Path where the modified configuration will be saved + mask_path : str + Path to the mask file + redshift_distr_path : str + Path to the redshift distribution file + """ + config = configparser.ConfigParser() + # Load the template configuration + config.read(template_config) + + # Update mask path + mask_base = os.path.basename(os.path.abspath(mask_path)) + mask_folder = os.path.dirname(os.path.abspath(mask_path)) + config['survey specs']['mask_directory'] = mask_folder + config['survey specs']['mask_file_lensing'] = mask_base + config['survey specs']['survey_area_lensing_in_deg2'] = str(self.area[ver]) + config['survey specs']['ellipticity_dispersion'] = str(self.ellipticity_dispersion[ver]) + config['survey specs']['n_eff_lensing'] = str(self.n_eff_gal[ver]) + + # Update redshift distribution path + redshift_distr_base = os.path.basename(os.path.abspath(redshift_distr_path)) + redshift_distr_folder = os.path.dirname(os.path.abspath(redshift_distr_path)) + config['redshift']['z_directory'] = redshift_distr_folder + config['redshift']['zlens_file'] = redshift_distr_base + + #Update output directory + config['output settings']['directory'] = out_dir + + # Save the modified configuration + with open(config_path, 'w') as f: + config.write(f) + + def get_cov_from_onecov(self, cov_one_cov, gaussian=True): + n_bins = np.sqrt(cov_one_cov.shape[0]).astype(int) + cov = np.zeros((n_bins, n_bins)) + + index_value = 10 if gaussian else 9 + for i in range(n_bins): + for j in range(n_bins): + cov[i, j] = cov_one_cov[i * n_bins + j, index_value] + + return cov + + def _load_onecovariance_cov(self, out_dir, ver): + self.print_cyan(f"Loading OneCovariance results from {out_dir}") + cov_one_cov = np.genfromtxt(os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat")) + gaussian_one_cov = self.get_cov_from_onecov(cov_one_cov, gaussian=True) + all_one_cov = self.get_cov_from_onecov(cov_one_cov, gaussian=False) + + self._pseudo_cls_onecov[ver] = { + 'gaussian_cov': gaussian_one_cov, + 'all_cov': all_one_cov + } + + def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): + assert gaussian_part in ["iNKA", "OneCovariance"], "gaussian_part must be 'iNKA' or 'OneCovariance'" + self.print_start(f"Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part") + + self._pseudo_cls_cov_g_ng = {} + + for ver in self.versions: + self.print_magenta(ver) + out_file = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_g_ng_{gaussian_part}_{ver}.fits") + if os.path.exists(out_file): + self.print_done(f"Skipping Gaussian and Non-Gaussian covariance calculation, {out_file} exists") + cov_hdu = fits.open(out_file) + self._pseudo_cls_cov_g_ng[ver] = cov_hdu + continue + if gaussian_part == "iNKA": + gaussian_cov = self.pseudo_cls[ver]['cov']['COVAR_EE_EE'].data + non_gaussian_cov = self.pseudo_cls_onecov[ver]['all_cov'] - self.pseudo_cls_onecov[ver]['gaussian_cov'] + full_cov = gaussian_cov + non_gaussian_cov + elif gaussian_part == "OneCovariance": + gaussian_cov = self.pseudo_cls_onecov[ver]['gaussian_cov'] + non_gaussian_cov = self.pseudo_cls_onecov[ver]['all_cov'] - self.pseudo_cls_onecov[ver]['gaussian_cov'] + full_cov = self.pseudo_cls_onecov[ver]['all_cov'] + else: + raise ValueError(f"Unknown gaussian_part: {gaussian_part}") + self.print_cyan("Saving Gaussian and Non-Gaussian covariance...") + hdu = fits.HDUList() + hdu.append(fits.ImageHDU(gaussian_cov, name="COVAR_GAUSSIAN")) + hdu.append(fits.ImageHDU(non_gaussian_cov, name="COVAR_NON_GAUSSIAN")) + hdu.append(fits.ImageHDU(full_cov, name="COVAR_FULL")) + hdu.writeto(out_file, overwrite=True) + self._pseudo_cls_cov_g_ng[ver] = hdu + self.print_done(f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part") + def calculate_pseudo_cl(self): """ Compute the pseudo-Cl of given catalogs. @@ -1880,104 +2912,98 @@ def calculate_pseudo_cl(self): self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") cl_shear = fits.getdata(out_path) self._pseudo_cls[ver]['pseudo_cl'] = cl_shear + elif self.cell_method == 'map': + self.calculate_pseudo_cl_map(ver, nside, out_path) + elif self.cell_method == 'catalog': + self.calculate_pseudo_cl_catalog(ver, out_path) else: - params = utils_cosmo_val.get_params_rho_tau(self.cc[ver], survey=ver) + raise ValueError(f"Unknown cell method: {self.cell_method}") - #Load data and create shear and noise maps - cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) + self.print_done("Done pseudo-Cl's") - w = cat_gal[params['w_col']] - self.print_cyan("Creating maps and computing Cl's...") - n_gal_map, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) - mask = n_gal_map != 0 + def calculate_pseudo_cl_map(self, ver, nside, out_path): + params = get_params_rho_tau(self.cc[ver], survey=ver) - shear_map_e1 = np.zeros(hp.nside2npix(nside)) - shear_map_e2 = np.zeros(hp.nside2npix(nside)) + #Load data and create shear and noise maps + cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - e1 = cat_gal[params['e1_col']] - e2 = cat_gal[params['e2_col']] + w = cat_gal[params['w_col']] + self.print_cyan("Creating maps and computing Cl's...") + n_gal_map, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) + mask = n_gal_map != 0 - del cat_gal - - shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1*w) - shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2*w) - shear_map_e1[mask] /= n_gal_map[mask] - shear_map_e2[mask] /= n_gal_map[mask] + shear_map_e1 = np.zeros(hp.nside2npix(nside)) + shear_map_e2 = np.zeros(hp.nside2npix(nside)) - shear_map = shear_map_e1 + 1j*shear_map_e2 + e1 = cat_gal[params['e1_col']] + e2 = cat_gal[params['e2_col']] - del shear_map_e1, shear_map_e2 + del cat_gal + + shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1*w) + shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2*w) + shear_map_e1[mask] /= n_gal_map[mask] + shear_map_e2[mask] /= n_gal_map[mask] - ell_eff, cl_shear, wsp = self.get_pseudo_cls(shear_map) + shear_map = shear_map_e1 + 1j*shear_map_e2 - cl_noise = np.zeros((4, self.n_ell_bins)) - - for i in range(self.nrandom_cell): + del shear_map_e1, shear_map_e2 - noise_map_e1 = np.zeros(hp.nside2npix(nside)) - noise_map_e2 = np.zeros(hp.nside2npix(nside)) + ell_eff, cl_shear, wsp = self.get_pseudo_cls_map(shear_map) - e1_rot, e2_rot = self.apply_random_rotation(e1, e2) + cl_noise = np.zeros_like(cl_shear) + + for i in range(self.nrandom_cell): - - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) + noise_map_e1 = np.zeros(hp.nside2npix(nside)) + noise_map_e2 = np.zeros(hp.nside2npix(nside)) - noise_map_e1[mask] /= n_gal_map[mask] - noise_map_e2[mask] /= n_gal_map[mask] + e1_rot, e2_rot = self.apply_random_rotation(e1, e2) - noise_map = noise_map_e1 + 1j*noise_map_e2 - del noise_map_e1, noise_map_e2 + + noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) + noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) - _, cl_noise_, _ = self.get_pseudo_cls(noise_map, wsp) - cl_noise += cl_noise_ - - cl_noise /= self.nrandom_cell - del e1, e2, e1_rot, e2_rot, w - del n_gal_map + noise_map_e1[mask] /= n_gal_map[mask] + noise_map_e2[mask] /= n_gal_map[mask] - #This is a problem because the measurement depends on the seed. To be fixed. - #cl_shear = cl_shear - np.mean(cl_noise, axis=1, keepdims=True) - cl_shear = cl_shear - cl_noise + noise_map = noise_map_e1 + 1j*noise_map_e2 + del noise_map_e1, noise_map_e2 - self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + _, cl_noise_, _ = self.get_pseudo_cls_map(noise_map, wsp) + cl_noise += cl_noise_ + + cl_noise /= self.nrandom_cell + del e1, e2, w + try: + del e1_rot, e2_rot + except NameError: #Continue if the random generation has been skipped. + pass + del n_gal_map - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear + #This is a problem because the measurement depends on the seed. To be fixed. + #cl_shear = cl_shear - np.mean(cl_noise, axis=1, keepdims=True) + cl_shear = cl_shear - cl_noise - self.print_done("Done pseudo-Cl's") + self.print_cyan("Saving pseudo-Cl's...") + self.save_pseudo_cl(ell_eff, cl_shear, out_path) - def get_fiducial(self, lmax, redshift_distr): - """ - Get the power spectrum at Planck18 cosmology using CAMB. - """ - planck = Planck18 - - h = planck.H0.value/100 - Om = planck.Om0 - Ob = planck.Ob0 - Oc = Om - Ob - ns = 0.965 - As = 2.1e-9 - m_nu = 0.06 - w = -1 - - pars = camb.set_params(H0=100*h, omch2=Oc*h**2, ombh2=Ob*h**2, ns=ns, mnu=m_nu, w=w, As=As, WantTransfer=True, NonLinear=camb.model.NonLinear_both) - Onu = pars.omeganu - Oc = Om - Ob - Onu - pars = camb.set_params(H0=100*h, omch2=Oc*h**2, ombh2=Ob*h**2, ns=ns, mnu=m_nu, w=w, As=As, WantTransfer=True, NonLinear=camb.model.NonLinear_both) - - z, dndz = np.loadtxt(redshift_distr, unpack=True) - - #getthe expected cl's from CAMB - pars.min_l = 1 - pars.set_for_lmax(lmax) - pars.SourceWindows = [ - camb.sources.SplinedSourceWindow(z=z, W=dndz, source_type='lensing') - ] - theory_cls = camb.get_results(pars).get_source_cls_dict(lmax=lmax, raw_cl=True) - return theory_cls['W1xW1'] + cl_shear = fits.getdata(out_path) + self._pseudo_cls[ver]['pseudo_cl'] = cl_shear + + def calculate_pseudo_cl_catalog(self, ver, out_path): + params = get_params_rho_tau(self.cc[ver], survey=ver) + + #Load data and create shear and noise maps + cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) + + ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog(catalog=cat_gal, params=params) + + self.print_cyan("Saving pseudo-Cl's...") + self.save_pseudo_cl(ell_eff, cl_shear, out_path) + + cl_shear = fits.getdata(out_path) + self._pseudo_cls[ver]['pseudo_cl'] = cl_shear def get_n_gal_map(self, params, nside, cat_gal): """ @@ -1996,9 +3022,12 @@ def get_n_gal_map(self, params, nside, cat_gal): n_gal[unique_pix] = np.bincount(idx_rep, weights=w) return n_gal, unique_pix, idx, idx_rep - def get_gaussian_real(self, params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep): - - e1_rot, e2_rot = self.apply_random_rotation(cat_gal[params['e1_col']], cat_gal[params['e2_col']]) + def get_gaussian_real( + self, params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep + ): + e1_rot, e2_rot = self.apply_random_rotation( + cat_gal[params["e1_col"]], cat_gal[params["e2_col"]] + ) noise_map_e1 = np.zeros(hp.nside2npix(nside)) noise_map_e2 = np.zeros(hp.nside2npix(nside)) @@ -2022,7 +3051,7 @@ def get_sample(self, params, nside, lmax, b, cat_gal, n_gal, mask, unique_pix, i return cl_noise, f, wsp - def get_pseudo_cls(self, map, wsp=None): + def get_pseudo_cls_map(self, map, wsp=None): """ Compute the pseudo-cl for a given map. """ @@ -2031,12 +3060,14 @@ def get_pseudo_cls(self, map, wsp=None): lmax = 2*self.nside b_lmax = lmax - 1 + ells = np.arange(lmin, lmax+1) + if self.binning == 'linear': - step = 10 - b = nmt.NmtBin.from_nside_linear(self.nside, step) + # Linear bands of width ell_step, respecting actual lmax + bpws = (ells - lmin) // self.ell_step + bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all + b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) elif self.binning == 'powspace': - ells = np.arange(lmin, lmax+1) - start = np.power(lmin, self.power) end = np.power(lmax, self.power) bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) @@ -2053,6 +3084,54 @@ def get_pseudo_cls(self, map, wsp=None): factor = -1 if self.pol_factor else 1 f_all = nmt.NmtField(mask=(map!=0), maps=[map.real, factor*map.imag], lmax=b_lmax) + + if wsp is None: + wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) + + cl_coupled = nmt.compute_coupled_cell(f_all, f_all) + cl_all = wsp.decouple_cell(cl_coupled) + + return ell_eff, cl_all, wsp + + def get_pseudo_cls_catalog(self, catalog, params, wsp=None): + """ + Compute the pseudo-cl for a given catalog. + """ + + lmin = 8 + lmax = 2*self.nside + b_lmax = lmax - 1 + + ells = np.arange(lmin, lmax+1) + + if self.binning == 'linear': + # Linear bands of width ell_step, respecting actual lmax + bpws = (ells - lmin) // self.ell_step + bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all + b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) + elif self.binning == 'powspace': + start = np.power(lmin, self.power) + end = np.power(lmax, self.power) + bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) + + #Get bandpowers + bpws = np.digitize(ells.astype(float), bins_ell) - 1 + bpws[0] = 0 + bpws[-1] = self.n_ell_bins-1 + + b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) + + ell_eff = b.get_effective_ells() + + factor = -1 if self.pol_factor else 1 + + f_all = nmt.NmtFieldCatalog(positions=[catalog[params['ra_col']], catalog[params['dec_col']]], + weights=catalog[params['w_col']], + field=[catalog[params['e1_col']], factor*catalog[params['e2_col']]], + lmax=b_lmax, + lmax_mask=b_lmax, + spin=2, + lonlat=True) if wsp is None: wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) @@ -2129,7 +3208,7 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"], capsize=2) - ax[0].set_ylabel('$\ell C_\ell$') + ax[0].set_ylabel(r"$\ell C_\ell$") ax[0].set_xlim(ell.min()-10, ell.max()+100) ax[0].set_xscale('squareroot') @@ -2144,8 +3223,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"]) - ax[1].set_xlabel('$\ell$') - ax[1].set_ylabel('$C_\ell$') + ax[1].set_xlabel(r"$\ell$") + ax[1].set_ylabel(r"$C_\ell$") ax[1].set_xlim(ell.min()-10, ell.max()+100) ax[1].set_xscale('squareroot') @@ -2170,8 +3249,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"], capsize=2) - ax[0].axhline(0, color='black', linestyle='--') - ax[0].set_ylabel('$\ell C_\ell$') + ax[0].axhline(0, color="black", linestyle="--") + ax[0].set_ylabel(r"$\ell C_\ell$") ax[0].set_xlim(ell.min()-10, ell.max()+100) ax[0].set_xscale('squareroot') @@ -2186,8 +3265,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"]) - ax[1].set_xlabel('$\ell$') - ax[1].set_ylabel('$C_\ell$') + ax[1].set_xlabel(r"$\ell$") + ax[1].set_ylabel(r"$C_\ell$") ax[1].set_xlim(ell.min()-10, ell.max()+100) ax[1].set_xscale('squareroot') @@ -2212,8 +3291,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"], capsize=2) - ax[0].axhline(0, color='black', linestyle='--') - ax[0].set_ylabel('$\ell C_\ell$') + ax[0].axhline(0, color="black", linestyle="--") + ax[0].set_ylabel(r"$\ell C_\ell$") ax[0].set_xlim(ell.min()-10, ell.max()+100) ax[0].set_xscale('squareroot') @@ -2228,8 +3307,8 @@ def plot_pseudo_cl(self): cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"]) - ax[1].set_xlabel('$\ell$') - ax[1].set_ylabel('$C_\ell$') + ax[1].set_xlabel(r"$\ell$") + ax[1].set_ylabel(r"$C_\ell$") ax[1].set_xlim(ell.min()-10, ell.max()+100) ax[1].set_xscale('squareroot') From 073e15e0cc4b14e632d52439b821ea4af128de52 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 26 Jan 2026 13:40:52 +0100 Subject: [PATCH 12/15] ran "black" formatting --- src/sp_validation/cosmo_val.py | 994 ++++++++++++++++++++++----------- 1 file changed, 660 insertions(+), 334 deletions(-) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index edc9a255..1dd423df 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -203,13 +203,13 @@ def __init__( ylim_alpha=[-0.005, 0.05], ylim_xi_sys_ratio=[-0.02, 0.5], nside=1024, - nside_mask = 2**12, + nside_mask=2**12, binning="powspace", power=1 / 2, n_ell_bins=32, ell_step=10, pol_factor=True, - cell_method='map', + cell_method="map", nrandom_cell=10, path_onecovariance=None, cosmo_params=None, @@ -239,7 +239,10 @@ def __init__( self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance - assert self.cell_method in ["map", "catalog"], "cell_method must be 'map' or 'catalog'" + assert self.cell_method in [ + "map", + "catalog", + ], "cell_method must be 'map' or 'catalog'" # For theory calculations: # Create cosmology object using new functionality @@ -249,7 +252,6 @@ def __init__( # Use Planck 2018 defaults self.cosmo = get_cosmo() - self.treecorr_config = { "ra_units": "degrees", "dec_units": "degrees", @@ -258,7 +260,9 @@ def __init__( "sep_units": "arcmin", "nbins": nbins, "var_method": var_method, - "cross_patch_weight": "match" if var_method == "jackknife" else "simple", + "cross_patch_weight": ( + "match" if var_method == "jackknife" else "simple" + ), } self.catalog_config_path = Path(catalog_config) @@ -295,7 +299,10 @@ def ensure_version_exists(ver): base_ver = ver[: -len(leak_suffix)] ensure_version_exists(base_ver) shear_cfg = cc[base_ver]["shear"] - if "e1_col_corrected" not in shear_cfg or "e2_col_corrected" not in shear_cfg: + if ( + "e1_col_corrected" not in shear_cfg + or "e2_col_corrected" not in shear_cfg + ): raise ValueError( f"{base_ver} does not have e1_col_corrected/e2_col_corrected " f"fields; cannot create {ver}" @@ -414,7 +421,9 @@ def compute_survey_stats( weight_column = weights_key_override or shear_cfg["w_col"] if weight_column not in data.columns.names: - raise KeyError(f"Weight column '{weight_column}' missing in {catalog_path}") + raise KeyError( + f"Weight column '{weight_column}' missing in {catalog_path}" + ) w = np.asarray(data[weight_column], dtype=float) @@ -428,9 +437,15 @@ def compute_survey_stats( mask_candidate = mask_path else: mask_candidate = self.cc[ver].get("mask") - if isinstance(mask_candidate, str) and not os.path.isabs(mask_candidate): - mask_candidate = str(Path(self.cc[ver]["subdir"]) / mask_candidate) - if mask_candidate is not None and not os.path.exists(mask_candidate): + if isinstance(mask_candidate, str) and not os.path.isabs( + mask_candidate + ): + mask_candidate = str( + Path(self.cc[ver]["subdir"]) / mask_candidate + ) + if mask_candidate is not None and not os.path.exists( + mask_candidate + ): mask_candidate = None area_deg2 = None @@ -481,7 +496,9 @@ def _area_from_catalog(self, catalog_path, nside): def _area_from_mask(self, mask_map_path): mask = hp.read_map(mask_map_path, dtype=np.float64) - return float(mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True)) + return float( + mask.sum() * hp.nside2pixarea(hp.get_nside(mask), degrees=True) + ) def _write_catalog_config(self): with self.catalog_config_path.open("w") as file: @@ -528,8 +545,12 @@ def set_params_leakage_scale(self, ver): params_in["e1_col"] = self.cc[ver]["shear"]["e1_col"] params_in["e2_col"] = self.cc[ver]["shear"]["e2_col"] params_in["w_col"] = self.cc[ver]["shear"]["w_col"] - params_in["R11"] = None if ver != "DES" else self.cc[ver]["shear"]["R11"] - params_in["R22"] = None if ver != "DES" else self.cc[ver]["shear"]["R22"] + params_in["R11"] = ( + None if ver != "DES" else self.cc[ver]["shear"]["R11"] + ) + params_in["R22"] = ( + None if ver != "DES" else self.cc[ver]["shear"]["R22"] + ) params_in["ra_star_col"] = self.cc[ver]["star"]["ra_col"] params_in["dec_star_col"] = self.cc[ver]["star"]["dec_col"] @@ -610,7 +631,6 @@ def basename(self, version, treecorr_config=None, npatch=None): f"_npatch={patches}" ) - def calculate_rho_tau_stats(self): out_dir = f"{self.cc['paths']['output']}/rho_tau_stats" if not os.path.exists(out_dir): @@ -649,13 +669,13 @@ def tau_stat_handler(self): @property def colors(self): return [self.cc[ver]["colour"] for ver in self.versions] - + @property def area(self): if not hasattr(self, "_area"): self.calculate_area() return self._area - + @property def n_eff_gal(self): if not hasattr(self, "_n_eff_gal"): @@ -674,7 +694,7 @@ def pseudo_cls(self): self.calculate_pseudo_cl() self.calculate_pseudo_cl_eb_cov() return self._pseudo_cls - + @property def pseudo_cls_onecov(self): if not hasattr(self, "_pseudo_cls_onecov"): @@ -687,33 +707,42 @@ def calculate_area(self): for ver in self.versions: self.print_magenta(ver) - if not hasattr(self.cc[ver]['shear'], 'mask'): - print("Mask not found in config file, calculating area from binned catalog") + if not hasattr(self.cc[ver]["shear"], "mask"): + print( + "Mask not found in config file, calculating area from binned catalog" + ) area[ver] = self.calculate_area_from_binned_catalog(ver) else: - mask = hp.read_map(self.cc[ver]['shear']['mask'], verbose=False) + mask = hp.read_map(self.cc[ver]["shear"]["mask"], verbose=False) nside_mask = hp.get_nside(mask) print(f"nside_mask = {nside_mask}") - area[ver] = np.sum(mask) * hp.nside2pixarea(nside_mask, degrees=True) + area[ver] = np.sum(mask) * hp.nside2pixarea( + nside_mask, degrees=True + ) print(f"Area = {area[ver]:.2f} deg^2") self._area = area self.print_done("Area calculation finished") - + def calculate_area_from_binned_catalog(self, ver): print(f"nside_mask = {self.nside_mask}") with self.results[ver].temporarily_read_data(): ra = self.results[ver].dat_shear["RA"] dec = self.results[ver].dat_shear["Dec"] hsp_map = hp.ang2pix( - self.nside_mask, + self.nside_mask, np.radians(90 - dec), np.radians(ra), lonlat=False, ) - mask = np.bincount(hsp_map, minlength=hp.nside2npix(self.nside_mask)) > 0 + mask = ( + np.bincount(hsp_map, minlength=hp.nside2npix(self.nside_mask)) + > 0 + ) - area = np.sum(mask) * hp.nside2pixarea(self.nside_mask, degrees=True) + area = np.sum(mask) * hp.nside2pixarea( + self.nside_mask, degrees=True + ) print(f"Area = {area:.2f} deg^2") return area @@ -725,9 +754,14 @@ def calculate_n_eff_gal(self): self.print_magenta(ver) with self.results[ver].temporarily_read_data(): w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] - n_eff_gal[ver] = 1/(self.area[ver]*60*60)* np.sum(w)**2/np.sum(w**2) + n_eff_gal[ver] = ( + 1 + / (self.area[ver] * 60 * 60) + * np.sum(w) ** 2 + / np.sum(w**2) + ) print(f"n_eff_gal = {n_eff_gal[ver]:.2f} gal./arcmin^-2") - + self._n_eff_gal = n_eff_gal self.print_done("Effective number of galaxy calculation finished") @@ -737,13 +771,23 @@ def calculate_ellipticity_dispersion(self): for ver in self.versions: self.print_magenta(ver) with self.results[ver].temporarily_read_data(): - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + e1 = self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e1_col"] + ] + e2 = self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e2_col"] + ] w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] ellipticity_dispersion[ver] = np.sqrt( - 0.5*(np.average(e1**2, weights=w**2) + np.average(e2**2, weights=w**2)) + 0.5 + * ( + np.average(e1**2, weights=w**2) + + np.average(e2**2, weights=w**2) + ) + ) + print( + f"Ellipticity dispersion = {ellipticity_dispersion[ver]:.4f}" ) - print(f"Ellipticity dispersion = {ellipticity_dispersion[ver]:.4f}") self._ellipticity_dispersion = ellipticity_dispersion def plot_rho_stats(self, abs=False): @@ -836,7 +880,11 @@ def calculate_rho_tau_fits(self): assert self.rho_tau_method != "none" # this initializes the rho_tau_fits attribute - self._rho_tau_fits = {"flat_sample_list": [], "result_list": [], "q_list": []} + self._rho_tau_fits = { + "flat_sample_list": [], + "result_list": [], + "q_list": [], + } quantiles = [1 - self.quantile, self.quantile] self._xi_psf_sys = {} @@ -866,7 +914,9 @@ def calculate_rho_tau_fits(self): self.rho_tau_fits["result_list"].append(result) self.rho_tau_fits["q_list"].append(q) - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") + self.psf_fitter.load_rho_stat( + f"rho_stats_{self.basename(ver)}.fits" + ) nbins = self.psf_fitter.rho_stat_handler._treecorr_config["nbins"] xi_psf_sys_samples = np.array([]).reshape(0, nbins) @@ -902,7 +952,9 @@ def plot_rho_tau_fits(self): show=True, close=True, ) - self.print_done(f"Tau contours plot saved to {os.path.abspath(savefig)}") + self.print_done( + f"Tau contours plot saved to {os.path.abspath(savefig)}" + ) plt.figure(figsize=(15, 6)) for mcmc_result, ver, color, flat_sample in zip( @@ -911,7 +963,9 @@ def plot_rho_tau_fits(self): self.colors, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") + self.psf_fitter.load_rho_stat( + f"rho_stats_{self.basename(ver)}.fits" + ) for i in range(100): self.psf_fitter.plot_xi_psf_sys( flat_sample[-i + 1], ver, color, alpha=0.1 @@ -934,8 +988,12 @@ def plot_rho_tau_fits(self): theta = self.psf_fitter.rho_stat_handler.rho_stats["theta"] xi_psf_sys = self.xi_psf_sys[ver] plt.plot(theta, xi_psf_sys["mean"], linestyle=ls, color=color) - plt.plot(theta, xi_psf_sys["quantiles"][0], linestyle=ls, color=color) - plt.plot(theta, xi_psf_sys["quantiles"][1], linestyle=ls, color=color) + plt.plot( + theta, xi_psf_sys["quantiles"][0], linestyle=ls, color=color + ) + plt.plot( + theta, xi_psf_sys["quantiles"][1], linestyle=ls, color=color + ) plt.fill_between( theta, xi_psf_sys["quantiles"][0], @@ -961,7 +1019,9 @@ def plot_rho_tau_fits(self): self.versions, self.rho_tau_fits["flat_sample_list"], ): - self.psf_fitter.load_rho_stat(f"rho_stats_{self.basename(ver)}.fits") + self.psf_fitter.load_rho_stat( + f"rho_stats_{self.basename(ver)}.fits" + ) for yscale in ("linear", "log"): out_path = os.path.abspath( f"{out_dir}/xi_psf_sys_terms_{yscale}_{ver}.png" @@ -1013,20 +1073,28 @@ def calculate_scale_dependent_leakage(self): output_path_ab = f"{output_base_path}_a_b.txt" output_path_aa = f"{output_base_path}_a_a.txt" with self.results[ver].temporarily_read_data(): - if os.path.exists(output_path_ab) and os.path.exists(output_path_aa): + if os.path.exists(output_path_ab) and os.path.exists( + output_path_aa + ): self.print_green( f"Skipping computation, reading {output_path_ab} and " f"{output_path_aa} instead" ) - results.r_corr_gp = treecorr.GGCorrelation(self.treecorr_config) + results.r_corr_gp = treecorr.GGCorrelation( + self.treecorr_config + ) results.r_corr_gp.read(output_path_ab) - results.r_corr_pp = treecorr.GGCorrelation(self.treecorr_config) + results.r_corr_pp = treecorr.GGCorrelation( + self.treecorr_config + ) results.r_corr_pp.read(output_path_aa) else: - results.compute_corr_gp_pp_alpha(output_base_path=output_base_path) + results.compute_corr_gp_pp_alpha( + output_base_path=output_base_path + ) results.do_alpha(fast=True) results.do_xi_sys() @@ -1130,7 +1198,9 @@ def plot_scale_dependent_leakage(self): xlabel = r"$\theta$ [arcmin]" ylabel = r"$\xi^{\rm sys}_+(\theta)$" title = "Cross-correlation leakage" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_p.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/xi_sys_p.png" + ) cs_plots.plot_data_1d( theta, y, @@ -1161,7 +1231,9 @@ def plot_scale_dependent_leakage(self): xlabel = r"$\theta$ [arcmin]" ylabel = r"$\xi^{\rm sys}_-(\theta)$" title = "Cross-correlation leakage" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_sys_m.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/xi_sys_m.png" + ) cs_plots.plot_data_1d( theta, y, @@ -1197,9 +1269,6 @@ def calculate_objectwise_leakage(self): results_obj.update_params() results_obj.prepare_output() - # Skip read_data() and copy catalogue from scale leakage instance instead - # results_obj._dat = self.results[ver].dat_shear - out_base = results_obj.get_out_base(mix, order) out_path = f"{out_base}.pkl" if os.path.exists(out_path): @@ -1301,37 +1370,47 @@ def plot_objectwise_leakage(self): ) cs_plots.savefig(out_path, close_fig=False) cs_plots.show() - self.print_done(f"Object-wise leakage coefficients plot saved to {out_path}") + self.print_done( + f"Object-wise leakage coefficients plot saved to {out_path}" + ) - def calculate_objectwise_leakage_aux(self): + def calculate_objectwise_leakage_aux(self): self.print_start("Object-wise leakage auxiliary quantities:") for ver in self.versions: self.print_magenta(ver) + print("Query objectise results") results_obj = self.results_objectwise[ver] + print("Query done") results_obj.check_params() results_obj.update_params() results_obj.prepare_output() - with self.results[ver].temporarily_load_data(): - results_obj._dat = self.results[ver].dat_shear + with self.results_obj.temporarily_load_data(): + results_obj._dat = self.results_obj.dat_shear if not "cols" in results_obj._params: - self.print_green("Skipping object-wise leakage (aux quantities), no input columns for regression found") + self.print_green( + "Skipping object-wise leakage (aux quantities), no input columns for regression found" + ) else: - self.print_cyan(f"Computing object-wise leakage regression with aux quantities: {results_obj._params['cols']}") + self.print_cyan( + f"Computing object-wise leakage regression with aux quantities: {results_obj._params['cols']}" + ) # Run results_obj.obs_leakage() def plot_objectwise_leakage_aux(self): - self.calculate_objectwise_leakage_aux() + self.calculate_objectwise_leakage_aux() def plot_ellipticity(self, nbins=200): out_path = os.path.abspath(f"{self.cc['paths']['output']}/ell_hist.png") if os.path.exists(out_path): - self.print_done(f"Skipping ellipticity histograms, {out_path} exists") + self.print_done( + f"Skipping ellipticity histograms, {out_path} exists" + ) else: self.print_start("Computing ellipticity histograms:") @@ -1342,12 +1421,20 @@ def plot_ellipticity(self, nbins=200): R = self.cc[ver]["shear"]["R"] with self.results[ver].temporarily_read_data(): e1 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] / R + self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e1_col"] + ] + / R ) e2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] / R + self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e2_col"] + ] + / R ) - w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] + w = self.results[ver].dat_shear[ + self.cc[ver]["shear"]["w_col"] + ] axs[0].hist( e1, @@ -1378,7 +1465,9 @@ def plot_ellipticity(self, nbins=200): self.print_done("Ellipticity histograms saved to " + out_path) def plot_weights(self, nbins=200): - out_path = os.path.abspath(f"{self.cc['paths']['output']}/weight_hist.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/weight_hist.png" + ) if os.path.exists(out_path): self.print_done(f"Skipping weight histograms, {out_path} exists") else: @@ -1388,7 +1477,9 @@ def plot_weights(self, nbins=200): for ver in self.versions: self.print_magenta(ver) with self.results[ver].temporarily_read_data(): - w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] + w = self.results[ver].dat_shear[ + self.cc[ver]["shear"]["w_col"] + ] plt.hist( w, @@ -1462,7 +1553,9 @@ def c2(self): self.calculate_additive_bias() return self._c2 - def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): + def calculate_2pcf( + self, ver, npatch=None, save_fits=False, **treecorr_config + ): """ Calculate the two-point correlation function (2PCF) ξ± for a given catalog version with TreeCorr. @@ -1521,8 +1614,12 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): else: # Load data and create a catalog with self.results[ver].temporarily_read_data(): - e1 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] - e2 = self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + e1 = self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e1_col"] + ] + e2 = self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e2_col"] + ] w = self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]] if ver != "DES": R = self.cc[ver]["shear"]["R"] @@ -1552,7 +1649,9 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): ra_units=self.treecorr_config["ra_units"], dec_units=self.treecorr_config["dec_units"], npatch=npatch, - patch_centers=patch_file if os.path.exists(patch_file) else None, + patch_centers=( + patch_file if os.path.exists(patch_file) else None + ), ) # If no patch file exists, save the current patches @@ -1572,13 +1671,17 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) col3 = fits.Column(name="ANGBIN", format="K", array=lst) col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) + col5 = fits.Column( + name="ANG", format="D", unit="arcmin", array=gg.meanr + ) coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") col4 = fits.Column(name="VALUE", format="D", array=gg.xim) coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") + ximinus_hdu = fits.BinTableHDU.from_columns( + coldefs, name="XI_MINUS" + ) # append xi_plus header info xiplus_dict = { @@ -1592,19 +1695,27 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): for key in xiplus_dict: xiplus_hdu.header[key] = xiplus_dict[key] - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) + col1 = fits.Column( + name="BIN1", format="K", array=np.ones(len(lst)) + ) + col2 = fits.Column( + name="BIN2", format="K", array=np.ones(len(lst)) + ) col3 = fits.Column(name="ANGBIN", format="K", array=lst) col4 = fits.Column(name="VALUE", format="D", array=gg.xip) col5 = fits.Column( name="ANG", format="D", unit="arcmin", array=gg.rnom ) coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") + xiplus_hdu = fits.BinTableHDU.from_columns( + coldefs, name="XI_PLUS" + ) col4 = fits.Column(name="VALUE", format="D", array=gg.xim) coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") + ximinus_hdu = fits.BinTableHDU.from_columns( + coldefs, name="XI_MINUS" + ) # append xi_plus header info xiplus_dict = { @@ -1714,7 +1825,8 @@ def plot_2pcf(self): plt.errorbar( self.cat_ggs[ver].meanr, self.cat_ggs[ver].xip * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxip) * self.cat_ggs[ver].meanr, + yerr=np.sqrt(self.cat_ggs[ver].varxip) + * self.cat_ggs[ver].meanr, label=ver, ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], @@ -1725,7 +1837,9 @@ def plot_2pcf(self): plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\theta \xi_+(\theta)$") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_p_theta.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/xi_p_theta.png" + ) cs_plots.savefig(out_path, close_fig=False) cs_plots.show() self.print_done(f"xi_plus_theta plot saved to {out_path}") @@ -1736,7 +1850,8 @@ def plot_2pcf(self): plt.errorbar( self.cat_ggs[ver].meanr * cs_plots.dx(idx, len(ver)), self.cat_ggs[ver].xim * self.cat_ggs[ver].meanr, - yerr=np.sqrt(self.cat_ggs[ver].varxim) * self.cat_ggs[ver].meanr, + yerr=np.sqrt(self.cat_ggs[ver].varxim) + * self.cat_ggs[ver].meanr, label=ver, ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], @@ -1747,7 +1862,9 @@ def plot_2pcf(self): plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.xlim([self.theta_min_plot, self.theta_max_plot]) plt.ylabel(r"$\theta \xi_-(\theta)$") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/xi_m_theta.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/xi_m_theta.png" + ) cs_plots.savefig(out_path, close_fig=False) cs_plots.show() self.print_done(f"xi_minus_theta plot saved to {out_path}") @@ -1797,13 +1914,14 @@ def plot_2pcf(self): ) cs_plots.savefig(out_path, close_fig=False) cs_plots.show() - self.print_done(f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}") + self.print_done( + f"xi_plus_xi_psf_sys {ver} plot saved to {out_path}" + ) def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): fig, _ = plt.subplots(ncols=1, nrows=1, figsize=(10, 7)) - for idx, ver in enumerate(self.versions): xi_psf_sys = self.xi_psf_sys[ver] gg = self.cat_ggs[ver] @@ -1815,7 +1933,7 @@ def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): ) theta = gg.meanr - jittered_theta = theta * (1+idx*offset) + jittered_theta = theta * (1 + idx * offset) plt.errorbar( jittered_theta, @@ -1825,13 +1943,13 @@ def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): ls=self.cc[ver]["ls"], color=self.cc[ver]["colour"], fmt=self.cc[ver].get("marker", None), - capsize=5 + capsize=5, ) plt.fill_between( [self.theta_min_plot, self.theta_max_plot], - - threshold, - + threshold, + -threshold, + +threshold, color="black", alpha=0.1, label=f"{threshold:.0%} threshold", @@ -1840,24 +1958,27 @@ def plot_ratio_xi_sys_xi(self, threshold=0.1, offset=0.02): [self.theta_min_plot, self.theta_max_plot], [threshold, threshold], ls="dashed", - color="black") + color="black", + ) plt.plot( [self.theta_min_plot, self.theta_max_plot], [-threshold, -threshold], ls="dashed", - color="black") + color="black", + ) plt.xscale("log") plt.xlabel(rf"$\theta$ [{self.treecorr_config['sep_units']}]") plt.ylabel(r"$\xi^{\rm psf}_{+, {\rm sys}} / \xi_+$") plt.gca().yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1)) plt.legend() plt.title("Ratio of PSF systematics to cosmic shear signal") - out_path = os.path.abspath(f"{self.cc['paths']['output']}/ratio_xi_sys_xi.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/ratio_xi_sys_xi.png" + ) cs_plots.savefig(out_path, close_fig=False) cs_plots.show() print(f"Ratio of xi_psf_sys to xi plot saved to {out_path}") - def calculate_aperture_mass_dispersion( self, theta_min=0.3, theta_max=200, nbins=500, nbins_map=15, npatch=25 ): @@ -1889,11 +2010,15 @@ def calculate_aperture_mass_dispersion( with self.results[ver].temporarily_read_data(): R = self.cc[ver]["shear"]["R"] g1 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e1_col"]] + self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e1_col"] + ] - self.c1[ver] ) / R g2 = ( - self.results[ver].dat_shear[self.cc[ver]["shear"]["e2_col"]] + self.results[ver].dat_shear[ + self.cc[ver]["shear"]["e2_col"] + ] - self.c2[ver] ) / R cat_gal = treecorr.Catalog( @@ -1901,7 +2026,9 @@ def calculate_aperture_mass_dispersion( dec=self.results[ver].dat_shear["Dec"], g1=g1, g2=g2, - w=self.results[ver].dat_shear[self.cc[ver]["shear"]["w_col"]], + w=self.results[ver].dat_shear[ + self.cc[ver]["shear"]["w_col"] + ], ra_units=self.treecorr_config["ra_units"], dec_units=self.treecorr_config["dec_units"], npatch=npatch, @@ -1960,7 +2087,9 @@ def plot_aperture_mass_dispersion(self): xlabel = r"$\theta$ [arcmin]" ylabel = "dispersion" title = f"Aperture-mass dispersion {mode}" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/{mode}.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/{mode}.png" + ) cs_plots.plot_data_1d( x, y, @@ -1992,7 +2121,9 @@ def plot_aperture_mass_dispersion(self): xlabel = r"$\theta$ [arcmin]" ylabel = "dispersion" title = f"Aperture-mass dispersion mode {mode}" - out_path = os.path.abspath(f"{self.cc['paths']['output']}/{mode}_log.png") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/{mode}_log.png" + ) cs_plots.plot_data_1d( x, y, @@ -2120,7 +2251,9 @@ def calculate_pure_eb( # Calculate correlation functions gg = self.calculate_2pcf(version, npatch=npatch, **treecorr_config) - gg_int = self.calculate_2pcf(version, npatch=npatch, **treecorr_config_int) + gg_int = self.calculate_2pcf( + version, npatch=npatch, **treecorr_config_int + ) # Get redshift distribution if using analytic covariance if cov_path_int is not None: @@ -2137,7 +2270,7 @@ def calculate_pure_eb( cov_path_int=cov_path_int, cosmo_cov=cosmo_cov, n_samples=n_samples, - z_dist=z_dist + z_dist=z_dist, ) return results @@ -2160,7 +2293,7 @@ def plot_pure_eb( cosmo_cov=None, n_samples=1000, results=None, - **kwargs + **kwargs, ): """ Generate comprehensive pure E/B mode analysis plots. @@ -2225,7 +2358,7 @@ def plot_pure_eb( # Use instance defaults for unspecified parameters versions = versions or self.versions - output_dir = output_dir or self.cc['paths']['output'] + output_dir = output_dir or self.cc["paths"]["output"] npatch = npatch or self.npatch # Override var_method to analytic when cov_path_int is provided @@ -2271,26 +2404,26 @@ def plot_pure_eb( # Get or calculate results for this version version_results = results_list[idx] or self.calculate_pure_eb( - version, - min_sep=min_sep, - max_sep=max_sep, - nbins=nbins, - min_sep_int=min_sep_int, - max_sep_int=max_sep_int, - nbins_int=nbins_int, - npatch=npatch, - var_method=var_method, - cov_path_int=cov_path_int, - cosmo_cov=cosmo_cov, - n_samples=n_samples, - ) + version, + min_sep=min_sep, + max_sep=max_sep, + nbins=nbins, + min_sep_int=min_sep_int, + max_sep_int=max_sep_int, + nbins_int=nbins_int, + npatch=npatch, + var_method=var_method, + cov_path_int=cov_path_int, + cosmo_cov=cosmo_cov, + n_samples=n_samples, + ) # Calculate E/B statistics for all bin combinations (only if not provided) version_results = calculate_eb_statistics( version_results, cov_path_int=cov_path_int, n_samples=n_samples, - **kwargs + **kwargs, ) # Generate all plots using specialized plotting functions @@ -2298,9 +2431,7 @@ def plot_pure_eb( # Integration vs Reporting comparison plot plot_integration_vs_reporting( - gg, gg_int, - out_stub + "_integration_vs_reporting.png", - version + gg, gg_int, out_stub + "_integration_vs_reporting.png", version ) # E/B/Ambiguous correlation functions plot @@ -2309,7 +2440,7 @@ def plot_pure_eb( out_stub + "_xis.png", version, fiducial_xip_scale_cut=fiducial_xip_scale_cut, - fiducial_xim_scale_cut=fiducial_xim_scale_cut + fiducial_xim_scale_cut=fiducial_xim_scale_cut, ) # 2D PTE heatmaps plot @@ -2318,7 +2449,7 @@ def plot_pure_eb( version, out_stub + "_ptes.png", fiducial_xip_scale_cut=fiducial_xip_scale_cut, - fiducial_xim_scale_cut=fiducial_xim_scale_cut + fiducial_xim_scale_cut=fiducial_xim_scale_cut, ) # Covariance matrix plot @@ -2326,7 +2457,7 @@ def plot_pure_eb( version_results["cov"], var_method, out_stub + "_covariance.png", - version + version, ) def calculate_cosebis( @@ -2430,7 +2561,7 @@ def calculate_cosebis( scale_cuts = [ (bin_edges[start], bin_edges[stop]) for start in range(nbins) - for stop in range(start+1, nbins+1) + for stop in range(start + 1, nbins + 1) ] print(f"Evaluating {len(scale_cuts)} scale cut combinations") @@ -2453,11 +2584,17 @@ def plot_cosebis( self, version=None, output_dir=None, - min_sep_int=0.5, max_sep_int=500, nbins_int=1000, # Integration binning - npatch=None, nmodes=10, cov_path=None, - evaluate_all_scale_cuts=False, # New parameter - min_sep=None, max_sep=None, nbins=None, # Reporting binning - fiducial_scale_cut=None, # For plotting reference + min_sep_int=0.5, + max_sep_int=500, + nbins_int=1000, # Integration binning + npatch=None, + nmodes=10, + cov_path=None, + evaluate_all_scale_cuts=False, # New parameter + min_sep=None, + max_sep=None, + nbins=None, # Reporting binning + fiducial_scale_cut=None, # For plotting reference results=None, ): """ @@ -2513,8 +2650,8 @@ def plot_cosebis( # Use instance defaults if not specified version = version or self.versions[0] - output_dir = output_dir or self.cc['paths']['output'] - npatch = npatch or self.treecorr_config.get('npatch', 256) + output_dir = output_dir or self.cc["paths"]["output"] + npatch = npatch or self.treecorr_config.get("npatch", 256) # Determine variance method based on whether theoretical covariance is used var_method = "analytic" if cov_path is not None else "jackknife" @@ -2528,7 +2665,9 @@ def plot_cosebis( # Add scale cut info if provided if fiducial_scale_cut is not None: - out_stub += f"_scalecut={fiducial_scale_cut[0]}-{fiducial_scale_cut[1]}" + out_stub += ( + f"_scalecut={fiducial_scale_cut[0]}-{fiducial_scale_cut[1]}" + ) # if evaluate_all_scale_cuts: # out_stub += f"_allcuts_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}" @@ -2552,8 +2691,9 @@ def plot_cosebis( # Generate plots using specialized plotting functions # Extract single result for plotting if multiple scale cuts were evaluated - if (isinstance(results, dict) and - all(isinstance(k, tuple) for k in results.keys())): + if isinstance(results, dict) and all( + isinstance(k, tuple) for k in results.keys() + ): # Multiple scale cuts: use fiducial_scale_cut if provided, otherwise use # full range if fiducial_scale_cut is not None: @@ -2572,20 +2712,19 @@ def plot_cosebis( plot_results, version, out_stub + "_cosebis.png", - fiducial_scale_cut=fiducial_scale_cut + fiducial_scale_cut=fiducial_scale_cut, ) plot_cosebis_covariance_matrix( - plot_results, - version, - var_method, - out_stub + "_covariance.png" + plot_results, version, var_method, out_stub + "_covariance.png" ) # Generate scale cut heatmap if we have multiple scale cuts - if (isinstance(results, dict) and - all(isinstance(k, tuple) for k in results.keys()) and - len(results) > 1): + if ( + isinstance(results, dict) + and all(isinstance(k, tuple) for k in results.keys()) + and len(results) > 1 + ): # Create temporary gg object with correct binning for mapping treecorr_config_temp = { **self.treecorr_config, @@ -2602,10 +2741,9 @@ def plot_cosebis( gg_temp, version, out_stub + "_scalecut_ptes.png", - fiducial_scale_cut=fiducial_scale_cut + fiducial_scale_cut=fiducial_scale_cut, ) - def calculate_pseudo_cl_eb_cov(self): """ Compute a theoretical Gaussian covariance of the Pseudo-Cl for EE, EB and BB. @@ -2624,14 +2762,20 @@ def calculate_pseudo_cl_eb_cov(self): if ver not in self._pseudo_cls.keys(): self._pseudo_cls[ver] = {} - out_path = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_{ver}.fits") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/pseudo_cl_cov_{ver}.fits" + ) if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl covariance calculation, {out_path} exists") - self._pseudo_cls[ver]['cov'] = fits.open(out_path) + self.print_done( + f"Skipping Pseudo-Cl covariance calculation, {out_path} exists" + ) + self._pseudo_cls[ver]["cov"] = fits.open(out_path) else: params = get_params_rho_tau(self.cc[ver], survey=ver) - self.print_cyan(f"Extracting the fiducial power spectrum for {ver}") + self.print_cyan( + f"Extracting the fiducial power spectrum for {ver}" + ) lmax = 2 * self.nside z, dndz = self.get_redshift(ver) @@ -2642,7 +2786,7 @@ def calculate_pseudo_cl_eb_cov(self): "Unexpected pixwin length for lmax=" f"{lmax}: got {pw.shape[0]}, expected {len(ell)+1}" ) - pw = pw[1:len(ell)+1] + pw = pw[1 : len(ell) + 1] # Load redshift distribution and calculate theory C_ell fiducial_cl = ( @@ -2656,51 +2800,84 @@ def calculate_pseudo_cl_eb_cov(self): * pw**2 ) - self.print_cyan("Getting a sample of the fiducial Cl's with noise") + self.print_cyan( + "Getting a sample of the fiducial Cl's with noise" + ) lmin = 8 - lmax = 2*self.nside + lmax = 2 * self.nside b_lmax = lmax - 1 - ells = np.arange(lmin, lmax+1) + ells = np.arange(lmin, lmax + 1) - if self.binning == 'linear': + if self.binning == "linear": # Linear bands of width ell_step, respecting actual lmax bpws = (ells - lmin) // self.ell_step - bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all + bpws = np.minimum( + bpws, bpws[-1] + ) # Ensure last bin captures all b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - elif self.binning == 'powspace': + elif self.binning == "powspace": start = np.power(lmin, self.power) end = np.power(lmax, self.power) - bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) + bins_ell = np.power( + np.linspace(start, end, self.n_ell_bins + 1), + 1 / self.power, + ) - #Get bandpowers + # Get bandpowers bpws = np.digitize(ells.astype(float), bins_ell) - 1 bpws[0] = 0 - bpws[-1] = self.n_ell_bins-1 + bpws[-1] = self.n_ell_bins - 1 b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - #Load data and create shear and noise maps + # Load data and create shear and noise maps cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - n_gal, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) + n_gal, unique_pix, idx, idx_rep = self.get_n_gal_map( + params, nside, cat_gal + ) mask = n_gal != 0 - - cl_noise, f, wsp = self.get_sample(params, self.nside, b_lmax, b, cat_gal, n_gal, mask, unique_pix, idx_rep) - - fiducial_cl = np.array([fiducial_cl, 0.*fiducial_cl, 0.*fiducial_cl, 0.*fiducial_cl])+ np.mean(cl_noise, axis=1, keepdims=True) - + + cl_noise, f, wsp = self.get_sample( + params, + self.nside, + b_lmax, + b, + cat_gal, + n_gal, + mask, + unique_pix, + idx_rep, + ) + + fiducial_cl = np.array( + [ + fiducial_cl, + 0.0 * fiducial_cl, + 0.0 * fiducial_cl, + 0.0 * fiducial_cl, + ] + ) + np.mean(cl_noise, axis=1, keepdims=True) + self.print_cyan("Computing the Pseudo-Cl covariance") cw = nmt.NmtCovarianceWorkspace.from_fields(f, f, f, f) - covar_22_22 = nmt.gaussian_covariance(cw, 2, 2, 2, 2, - fiducial_cl, - fiducial_cl, - fiducial_cl, - fiducial_cl, - wsp, wb=wsp).reshape([self.n_ell_bins, 4, self.n_ell_bins, 4]) + covar_22_22 = nmt.gaussian_covariance( + cw, + 2, + 2, + 2, + 2, + fiducial_cl, + fiducial_cl, + fiducial_cl, + fiducial_cl, + wsp, + wb=wsp, + ).reshape([self.n_ell_bins, 4, self.n_ell_bins, 4]) covar_EE_EE = covar_22_22[:, 0, :, 0] covar_EE_EB = covar_22_22[:, 0, :, 1] @@ -2742,7 +2919,7 @@ def calculate_pseudo_cl_eb_cov(self): hdu.writeto(out_path, overwrite=True) - self._pseudo_cls[ver]['cov'] = hdu + self._pseudo_cls[ver]["cov"] = hdu self.print_done("Done Pseudo-Cl covariance") @@ -2753,51 +2930,88 @@ def calculate_pseudo_cl_onecovariance(self): self.print_start("Computing Pseudo-Cl covariance with OneCovariance") if self.path_onecovariance is None: - raise ValueError("path_onecovariance must be provided to use OneCovariance") - + raise ValueError( + "path_onecovariance must be provided to use OneCovariance" + ) + if not os.path.exists(self.path_onecovariance): - raise ValueError(f"OneCovariance path {self.path_onecovariance} does not exist") - - template_config = os.path.join(self.path_onecovariance, "config_files", "config_3x2pt_pure_Cell_UNIONS.ini") + raise ValueError( + f"OneCovariance path {self.path_onecovariance} does not exist" + ) + + template_config = os.path.join( + self.path_onecovariance, + "config_files", + "config_3x2pt_pure_Cell_UNIONS.ini", + ) if not os.path.exists(template_config): - raise ValueError(f"Template config file {template_config} does not exist") + raise ValueError( + f"Template config file {template_config} does not exist" + ) self._pseudo_cls_onecov = {} for ver in self.versions: self.print_magenta(ver) - out_dir = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_onecov_{ver}/") + out_dir = os.path.abspath( + f"{self.cc['paths']['output']}/pseudo_cl_cov_onecov_{ver}/" + ) os.makedirs(out_dir, exist_ok=True) - if os.path.exists(os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat")): - self.print_done(f"Skipping OneCovariance calculation, {out_dir} exists") + if os.path.exists( + os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat") + ): + self.print_done( + f"Skipping OneCovariance calculation, {out_dir} exists" + ) self._load_onecovariance_cov(out_dir, ver) else: - mask_path = self.cc[ver]['shear']['mask'] - redshift_distr_path = os.path.join(self.data_base_dir, self.cc[ver]['shear']['redshift_distr']) + mask_path = self.cc[ver]["shear"]["mask"] + redshift_distr_path = os.path.join( + self.data_base_dir, self.cc[ver]["shear"]["redshift_distr"] + ) config_path = os.path.join(out_dir, f"config_onecov_{ver}.ini") - self.print_cyan(f"Modifying OneCovariance config file and saving it to {config_path}") - self._modify_onecov_config(template_config, config_path, out_dir, mask_path, redshift_distr_path, ver) + self.print_cyan( + f"Modifying OneCovariance config file and saving it to {config_path}" + ) + self._modify_onecov_config( + template_config, + config_path, + out_dir, + mask_path, + redshift_distr_path, + ver, + ) self.print_cyan("Running OneCovariance...") cmd = f"python {os.path.join(self.path_onecovariance, 'covariance.py')} {config_path}" self.print_cyan(f"Command: {cmd}") ret = os.system(cmd) if ret != 0: - raise RuntimeError(f"OneCovariance command failed with return code {ret}") + raise RuntimeError( + f"OneCovariance command failed with return code {ret}" + ) self.print_cyan("OneCovariance completed successfully.") self._load_onecovariance_cov(out_dir, ver) - + self.print_done("Done Pseudo-Cl covariance with OneCovariance") - def _modify_onecov_config(self, template_config, config_path, out_dir, mask_path, redshift_distr_path, ver): + def _modify_onecov_config( + self, + template_config, + config_path, + out_dir, + mask_path, + redshift_distr_path, + ver, + ): """ - Modify OneCovariance configuration file with correct mask, redshift distribution, + Modify OneCovariance configuration file with correct mask, redshift distribution, and ellipticity dispersion parameters. - + Parameters ---------- template_config : str @@ -2809,30 +3023,38 @@ def _modify_onecov_config(self, template_config, config_path, out_dir, mask_path redshift_distr_path : str Path to the redshift distribution file """ - config = configparser.ConfigParser() + config = configparser.ConfigParser() # Load the template configuration config.read(template_config) - + # Update mask path mask_base = os.path.basename(os.path.abspath(mask_path)) mask_folder = os.path.dirname(os.path.abspath(mask_path)) - config['survey specs']['mask_directory'] = mask_folder - config['survey specs']['mask_file_lensing'] = mask_base - config['survey specs']['survey_area_lensing_in_deg2'] = str(self.area[ver]) - config['survey specs']['ellipticity_dispersion'] = str(self.ellipticity_dispersion[ver]) - config['survey specs']['n_eff_lensing'] = str(self.n_eff_gal[ver]) + config["survey specs"]["mask_directory"] = mask_folder + config["survey specs"]["mask_file_lensing"] = mask_base + config["survey specs"]["survey_area_lensing_in_deg2"] = str( + self.area[ver] + ) + config["survey specs"]["ellipticity_dispersion"] = str( + self.ellipticity_dispersion[ver] + ) + config["survey specs"]["n_eff_lensing"] = str(self.n_eff_gal[ver]) # Update redshift distribution path - redshift_distr_base = os.path.basename(os.path.abspath(redshift_distr_path)) - redshift_distr_folder = os.path.dirname(os.path.abspath(redshift_distr_path)) - config['redshift']['z_directory'] = redshift_distr_folder - config['redshift']['zlens_file'] = redshift_distr_base - - #Update output directory - config['output settings']['directory'] = out_dir - + redshift_distr_base = os.path.basename( + os.path.abspath(redshift_distr_path) + ) + redshift_distr_folder = os.path.dirname( + os.path.abspath(redshift_distr_path) + ) + config["redshift"]["z_directory"] = redshift_distr_folder + config["redshift"]["zlens_file"] = redshift_distr_base + + # Update output directory + config["output settings"]["directory"] = out_dir + # Save the modified configuration - with open(config_path, 'w') as f: + with open(config_path, "w") as f: config.write(f) def get_cov_from_onecov(self, cov_one_cov, gaussian=True): @@ -2843,52 +3065,73 @@ def get_cov_from_onecov(self, cov_one_cov, gaussian=True): for i in range(n_bins): for j in range(n_bins): cov[i, j] = cov_one_cov[i * n_bins + j, index_value] - + return cov def _load_onecovariance_cov(self, out_dir, ver): self.print_cyan(f"Loading OneCovariance results from {out_dir}") - cov_one_cov = np.genfromtxt(os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat")) + cov_one_cov = np.genfromtxt( + os.path.join(out_dir, "covariance_list_3x2pt_pure_Cell.dat") + ) gaussian_one_cov = self.get_cov_from_onecov(cov_one_cov, gaussian=True) all_one_cov = self.get_cov_from_onecov(cov_one_cov, gaussian=False) self._pseudo_cls_onecov[ver] = { - 'gaussian_cov': gaussian_one_cov, - 'all_cov': all_one_cov + "gaussian_cov": gaussian_one_cov, + "all_cov": all_one_cov, } def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): - assert gaussian_part in ["iNKA", "OneCovariance"], "gaussian_part must be 'iNKA' or 'OneCovariance'" - self.print_start(f"Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part") - + assert gaussian_part in [ + "iNKA", + "OneCovariance", + ], "gaussian_part must be 'iNKA' or 'OneCovariance'" + self.print_start( + f"Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" + ) + self._pseudo_cls_cov_g_ng = {} for ver in self.versions: self.print_magenta(ver) - out_file = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_cov_g_ng_{gaussian_part}_{ver}.fits") + out_file = os.path.abspath( + f"{self.cc['paths']['output']}/pseudo_cl_cov_g_ng_{gaussian_part}_{ver}.fits" + ) if os.path.exists(out_file): - self.print_done(f"Skipping Gaussian and Non-Gaussian covariance calculation, {out_file} exists") + self.print_done( + f"Skipping Gaussian and Non-Gaussian covariance calculation, {out_file} exists" + ) cov_hdu = fits.open(out_file) self._pseudo_cls_cov_g_ng[ver] = cov_hdu continue if gaussian_part == "iNKA": - gaussian_cov = self.pseudo_cls[ver]['cov']['COVAR_EE_EE'].data - non_gaussian_cov = self.pseudo_cls_onecov[ver]['all_cov'] - self.pseudo_cls_onecov[ver]['gaussian_cov'] + gaussian_cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data + non_gaussian_cov = ( + self.pseudo_cls_onecov[ver]["all_cov"] + - self.pseudo_cls_onecov[ver]["gaussian_cov"] + ) full_cov = gaussian_cov + non_gaussian_cov elif gaussian_part == "OneCovariance": - gaussian_cov = self.pseudo_cls_onecov[ver]['gaussian_cov'] - non_gaussian_cov = self.pseudo_cls_onecov[ver]['all_cov'] - self.pseudo_cls_onecov[ver]['gaussian_cov'] - full_cov = self.pseudo_cls_onecov[ver]['all_cov'] + gaussian_cov = self.pseudo_cls_onecov[ver]["gaussian_cov"] + non_gaussian_cov = ( + self.pseudo_cls_onecov[ver]["all_cov"] + - self.pseudo_cls_onecov[ver]["gaussian_cov"] + ) + full_cov = self.pseudo_cls_onecov[ver]["all_cov"] else: raise ValueError(f"Unknown gaussian_part: {gaussian_part}") self.print_cyan("Saving Gaussian and Non-Gaussian covariance...") hdu = fits.HDUList() hdu.append(fits.ImageHDU(gaussian_cov, name="COVAR_GAUSSIAN")) - hdu.append(fits.ImageHDU(non_gaussian_cov, name="COVAR_NON_GAUSSIAN")) + hdu.append( + fits.ImageHDU(non_gaussian_cov, name="COVAR_NON_GAUSSIAN") + ) hdu.append(fits.ImageHDU(full_cov, name="COVAR_FULL")) hdu.writeto(out_file, overwrite=True) self._pseudo_cls_cov_g_ng[ver] = hdu - self.print_done(f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part") + self.print_done( + f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" + ) def calculate_pseudo_cl(self): """ @@ -2907,14 +3150,18 @@ def calculate_pseudo_cl(self): self._pseudo_cls[ver] = {} - out_path = os.path.abspath(f"{self.cc['paths']['output']}/pseudo_cl_{ver}.fits") + out_path = os.path.abspath( + f"{self.cc['paths']['output']}/pseudo_cl_{ver}.fits" + ) if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") + self.print_done( + f"Skipping Pseudo-Cl's calculation, {out_path} exists" + ) cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear - elif self.cell_method == 'map': + self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + elif self.cell_method == "map": self.calculate_pseudo_cl_map(ver, nside, out_path) - elif self.cell_method == 'catalog': + elif self.cell_method == "catalog": self.calculate_pseudo_cl_catalog(ver, out_path) else: raise ValueError(f"Unknown cell method: {self.cell_method}") @@ -2924,35 +3171,37 @@ def calculate_pseudo_cl(self): def calculate_pseudo_cl_map(self, ver, nside, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) - #Load data and create shear and noise maps + # Load data and create shear and noise maps cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - w = cat_gal[params['w_col']] + w = cat_gal[params["w_col"]] self.print_cyan("Creating maps and computing Cl's...") - n_gal_map, unique_pix, idx, idx_rep = self.get_n_gal_map(params, nside, cat_gal) + n_gal_map, unique_pix, idx, idx_rep = self.get_n_gal_map( + params, nside, cat_gal + ) mask = n_gal_map != 0 shear_map_e1 = np.zeros(hp.nside2npix(nside)) shear_map_e2 = np.zeros(hp.nside2npix(nside)) - e1 = cat_gal[params['e1_col']] - e2 = cat_gal[params['e2_col']] + e1 = cat_gal[params["e1_col"]] + e2 = cat_gal[params["e2_col"]] del cat_gal - - shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1*w) - shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2*w) + + shear_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1 * w) + shear_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2 * w) shear_map_e1[mask] /= n_gal_map[mask] shear_map_e2[mask] /= n_gal_map[mask] - shear_map = shear_map_e1 + 1j*shear_map_e2 + shear_map = shear_map_e1 + 1j * shear_map_e2 del shear_map_e1, shear_map_e2 ell_eff, cl_shear, wsp = self.get_pseudo_cls_map(shear_map) cl_noise = np.zeros_like(cl_shear) - + for i in range(self.nrandom_cell): noise_map_e1 = np.zeros(hp.nside2npix(nside)) @@ -2960,64 +3209,67 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): e1_rot, e2_rot = self.apply_random_rotation(e1, e2) - - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) + noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot * w) + noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot * w) noise_map_e1[mask] /= n_gal_map[mask] noise_map_e2[mask] /= n_gal_map[mask] - noise_map = noise_map_e1 + 1j*noise_map_e2 + noise_map = noise_map_e1 + 1j * noise_map_e2 del noise_map_e1, noise_map_e2 _, cl_noise_, _ = self.get_pseudo_cls_map(noise_map, wsp) cl_noise += cl_noise_ - + cl_noise /= self.nrandom_cell del e1, e2, w try: del e1_rot, e2_rot - except NameError: #Continue if the random generation has been skipped. + except NameError: # Continue if the random generation has been skipped. pass - del n_gal_map + del n_gal_map - #This is a problem because the measurement depends on the seed. To be fixed. - #cl_shear = cl_shear - np.mean(cl_noise, axis=1, keepdims=True) + # This is a problem because the measurement depends on the seed. To be fixed. + # cl_shear = cl_shear - np.mean(cl_noise, axis=1, keepdims=True) cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") self.save_pseudo_cl(ell_eff, cl_shear, out_path) cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = cl_shear def calculate_pseudo_cl_catalog(self, ver, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) - #Load data and create shear and noise maps + # Load data and create shear and noise maps cat_gal = fits.getdata(self.cc[ver]["shear"]["path"]) - ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog(catalog=cat_gal, params=params) + ell_eff, cl_shear, wsp = self.get_pseudo_cls_catalog( + catalog=cat_gal, params=params + ) self.print_cyan("Saving pseudo-Cl's...") self.save_pseudo_cl(ell_eff, cl_shear, out_path) cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]['pseudo_cl'] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = cl_shear def get_n_gal_map(self, params, nside, cat_gal): """ Compute the galaxy number density map. """ - ra = cat_gal[params['ra_col']] - dec = cat_gal[params['dec_col']] - w = cat_gal[params['w_col']] + ra = cat_gal[params["ra_col"]] + dec = cat_gal[params["dec_col"]] + w = cat_gal[params["w_col"]] - theta = (90. - dec) * np.pi / 180. - phi = ra * np.pi / 180. + theta = (90.0 - dec) * np.pi / 180.0 + phi = ra * np.pi / 180.0 pix = hp.ang2pix(nside, theta, phi) - unique_pix, idx, idx_rep = np.unique(pix, return_index=True, return_inverse=True) + unique_pix, idx, idx_rep = np.unique( + pix, return_index=True, return_inverse=True + ) n_gal = np.zeros(hp.nside2npix(nside)) n_gal[unique_pix] = np.bincount(idx_rep, weights=w) return n_gal, unique_pix, idx, idx_rep @@ -3031,18 +3283,24 @@ def get_gaussian_real( noise_map_e1 = np.zeros(hp.nside2npix(nside)) noise_map_e2 = np.zeros(hp.nside2npix(nside)) - w = cat_gal[params['w_col']] - noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot*w) - noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot*w) + w = cat_gal[params["w_col"]] + noise_map_e1[unique_pix] += np.bincount(idx_rep, weights=e1_rot * w) + noise_map_e2[unique_pix] += np.bincount(idx_rep, weights=e2_rot * w) noise_map_e1[mask] /= n_gal[mask] noise_map_e2[mask] /= n_gal[mask] - return noise_map_e1 + 1j*noise_map_e2 + return noise_map_e1 + 1j * noise_map_e2 - def get_sample(self, params, nside, lmax, b, cat_gal, n_gal, mask, unique_pix, idx_rep): - noise_map = self.get_gaussian_real(params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep) + def get_sample( + self, params, nside, lmax, b, cat_gal, n_gal, mask, unique_pix, idx_rep + ): + noise_map = self.get_gaussian_real( + params, nside, lmax, cat_gal, n_gal, mask, unique_pix, idx_rep + ) - f = nmt.NmtField(mask=mask, maps=[noise_map.real, noise_map.imag], lmax=lmax) + f = nmt.NmtField( + mask=mask, maps=[noise_map.real, noise_map.imag], lmax=lmax + ) wsp = nmt.NmtWorkspace.from_fields(f, f, b) @@ -3050,32 +3308,34 @@ def get_sample(self, params, nside, lmax, b, cat_gal, n_gal, mask, unique_pix, i cl_noise = wsp.decouple_cell(cl_noise) return cl_noise, f, wsp - + def get_pseudo_cls_map(self, map, wsp=None): """ Compute the pseudo-cl for a given map. """ lmin = 8 - lmax = 2*self.nside + lmax = 2 * self.nside b_lmax = lmax - 1 - ells = np.arange(lmin, lmax+1) + ells = np.arange(lmin, lmax + 1) - if self.binning == 'linear': + if self.binning == "linear": # Linear bands of width ell_step, respecting actual lmax bpws = (ells - lmin) // self.ell_step bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - elif self.binning == 'powspace': + elif self.binning == "powspace": start = np.power(lmin, self.power) end = np.power(lmax, self.power) - bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) + bins_ell = np.power( + np.linspace(start, end, self.n_ell_bins + 1), 1 / self.power + ) - #Get bandpowers + # Get bandpowers bpws = np.digitize(ells.astype(float), bins_ell) - 1 bpws[0] = 0 - bpws[-1] = self.n_ell_bins-1 + bpws[-1] = self.n_ell_bins - 1 b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) @@ -3083,11 +3343,13 @@ def get_pseudo_cls_map(self, map, wsp=None): factor = -1 if self.pol_factor else 1 - f_all = nmt.NmtField(mask=(map!=0), maps=[map.real, factor*map.imag], lmax=b_lmax) + f_all = nmt.NmtField( + mask=(map != 0), maps=[map.real, factor * map.imag], lmax=b_lmax + ) if wsp is None: wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) - + cl_coupled = nmt.compute_coupled_cell(f_all, f_all) cl_all = wsp.decouple_cell(cl_coupled) @@ -3099,25 +3361,27 @@ def get_pseudo_cls_catalog(self, catalog, params, wsp=None): """ lmin = 8 - lmax = 2*self.nside + lmax = 2 * self.nside b_lmax = lmax - 1 - ells = np.arange(lmin, lmax+1) + ells = np.arange(lmin, lmax + 1) - if self.binning == 'linear': + if self.binning == "linear": # Linear bands of width ell_step, respecting actual lmax bpws = (ells - lmin) // self.ell_step bpws = np.minimum(bpws, bpws[-1]) # Ensure last bin captures all b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) - elif self.binning == 'powspace': + elif self.binning == "powspace": start = np.power(lmin, self.power) end = np.power(lmax, self.power) - bins_ell = np.power(np.linspace(start, end, self.n_ell_bins+1), 1/self.power) + bins_ell = np.power( + np.linspace(start, end, self.n_ell_bins + 1), 1 / self.power + ) - #Get bandpowers + # Get bandpowers bpws = np.digitize(ells.astype(float), bins_ell) - 1 bpws[0] = 0 - bpws[-1] = self.n_ell_bins-1 + bpws[-1] = self.n_ell_bins - 1 b = nmt.NmtBin(ells=ells, bpws=bpws, lmax=b_lmax) @@ -3125,17 +3389,22 @@ def get_pseudo_cls_catalog(self, catalog, params, wsp=None): factor = -1 if self.pol_factor else 1 - f_all = nmt.NmtFieldCatalog(positions=[catalog[params['ra_col']], catalog[params['dec_col']]], - weights=catalog[params['w_col']], - field=[catalog[params['e1_col']], factor*catalog[params['e2_col']]], - lmax=b_lmax, - lmax_mask=b_lmax, - spin=2, - lonlat=True) - + f_all = nmt.NmtFieldCatalog( + positions=[catalog[params["ra_col"]], catalog[params["dec_col"]]], + weights=catalog[params["w_col"]], + field=[ + catalog[params["e1_col"]], + factor * catalog[params["e2_col"]], + ], + lmax=b_lmax, + lmax_mask=b_lmax, + spin=2, + lonlat=True, + ) + if wsp is None: wsp = nmt.NmtWorkspace.from_fields(f_all, f_all, b) - + cl_coupled = nmt.compute_coupled_cell(f_all, f_all) cl_all = wsp.decouple_cell(cl_coupled) @@ -3160,9 +3429,9 @@ def apply_random_rotation(self, e1, e2): Second component of the rotated ellipticity. """ np.random.seed() - rot_angle = np.random.rand(len(e1))*2*np.pi - e1_out = e1*np.cos(rot_angle) + e2*np.sin(rot_angle) - e2_out = -e1*np.sin(rot_angle) + e2*np.cos(rot_angle) + rot_angle = np.random.rand(len(e1)) * 2 * np.pi + e1_out = e1 * np.cos(rot_angle) + e2 * np.sin(rot_angle) + e2_out = -e1 * np.sin(rot_angle) + e2 * np.cos(rot_angle) return e1_out, e2_out def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): @@ -3176,7 +3445,7 @@ def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): out_path : str Path to save the pseudo-Cl's to. """ - #Create columns of the fits file + # Create columns of the fits file col1 = fits.Column(name="ELL", format="D", array=ell_eff) col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) @@ -3199,126 +3468,183 @@ def plot_pseudo_cl(self): """ self.print_cyan("Plotting pseudo-Cl's") - #Plotting EE + # Plotting EE out_path = os.path.abspath(f"{self.cc['paths']['output']}/cell_ee.png") fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data - ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"], capsize=2) + ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] + cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data + ax[0].errorbar( + ell, + ell * self.pseudo_cls[ver]["pseudo_cl"]["EE"], + yerr=ell * np.sqrt(np.diag(cov)), + fmt=self.cc[ver]["marker"], + label=ver + " EE", + color=self.cc[ver]["colour"], + capsize=2, + ) ax[0].set_ylabel(r"$\ell C_\ell$") - ax[0].set_xlim(ell.min()-10, ell.max()+100) - ax[0].set_xscale('squareroot') + ax[0].set_xlim(ell.min() - 10, ell.max() + 100) + ax[0].set_xscale("squareroot") ax[0].set_xticks(np.array([100, 400, 900, 1600])) ax[0].minorticks_on() - ax[0].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] + ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) + minor_ticks = [i * 10 for i in range(1, 10)] + [ + i * 100 for i in range(1, 21) + ] ax[0].xaxis.set_ticks(minor_ticks, minor=True) for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EE_EE"].data - ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EE"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EE", color=self.cc[ver]["colour"]) + ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] + cov = self.pseudo_cls[ver]["cov"]["COVAR_EE_EE"].data + ax[1].errorbar( + ell, + self.pseudo_cls[ver]["pseudo_cl"]["EE"], + yerr=np.sqrt(np.diag(cov)), + fmt=self.cc[ver]["marker"], + label=ver + " EE", + color=self.cc[ver]["colour"], + ) ax[1].set_xlabel(r"$\ell$") ax[1].set_ylabel(r"$C_\ell$") - ax[1].set_xlim(ell.min()-10, ell.max()+100) - ax[1].set_xscale('squareroot') - ax[1].set_yscale('log') + ax[1].set_xlim(ell.min() - 10, ell.max() + 100) + ax[1].set_xscale("squareroot") + ax[1].set_yscale("log") ax[1].set_xticks(np.array([100, 400, 900, 1600])) ax[1].minorticks_on() - ax[1].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] + ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) + minor_ticks = [i * 10 for i in range(1, 10)] + [ + i * 100 for i in range(1, 21) + ] ax[1].xaxis.set_ticks(minor_ticks, minor=True) - plt.suptitle('Pseudo-Cl EE (Gaussian covariance)') + plt.suptitle("Pseudo-Cl EE (Gaussian covariance)") plt.legend() plt.savefig(out_path) - #Plotting EB + # Plotting EB out_path = os.path.abspath(f"{self.cc['paths']['output']}/cell_eb.png") fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data - ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"], capsize=2) + ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] + cov = self.pseudo_cls[ver]["cov"]["COVAR_EB_EB"].data + ax[0].errorbar( + ell, + ell * self.pseudo_cls[ver]["pseudo_cl"]["EB"], + yerr=ell * np.sqrt(np.diag(cov)), + fmt=self.cc[ver]["marker"], + label=ver + " EB", + color=self.cc[ver]["colour"], + capsize=2, + ) ax[0].axhline(0, color="black", linestyle="--") ax[0].set_ylabel(r"$\ell C_\ell$") - ax[0].set_xlim(ell.min()-10, ell.max()+100) - ax[0].set_xscale('squareroot') + ax[0].set_xlim(ell.min() - 10, ell.max() + 100) + ax[0].set_xscale("squareroot") ax[0].set_xticks(np.array([100, 400, 900, 1600])) ax[0].minorticks_on() - ax[0].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] + ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) + minor_ticks = [i * 10 for i in range(1, 10)] + [ + i * 100 for i in range(1, 21) + ] ax[0].xaxis.set_ticks(minor_ticks, minor=True) for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_EB_EB"].data - ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["EB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" EB", color=self.cc[ver]["colour"]) + ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] + cov = self.pseudo_cls[ver]["cov"]["COVAR_EB_EB"].data + ax[1].errorbar( + ell, + self.pseudo_cls[ver]["pseudo_cl"]["EB"], + yerr=np.sqrt(np.diag(cov)), + fmt=self.cc[ver]["marker"], + label=ver + " EB", + color=self.cc[ver]["colour"], + ) ax[1].set_xlabel(r"$\ell$") ax[1].set_ylabel(r"$C_\ell$") - ax[1].set_xlim(ell.min()-10, ell.max()+100) - ax[1].set_xscale('squareroot') - ax[1].set_yscale('log') + ax[1].set_xlim(ell.min() - 10, ell.max() + 100) + ax[1].set_xscale("squareroot") + ax[1].set_yscale("log") ax[1].set_xticks(np.array([100, 400, 900, 1600])) ax[1].minorticks_on() - ax[1].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] + ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) + minor_ticks = [i * 10 for i in range(1, 10)] + [ + i * 100 for i in range(1, 21) + ] ax[1].xaxis.set_ticks(minor_ticks, minor=True) - plt.suptitle('Pseudo-Cl EB (Gaussian covariance)') + plt.suptitle("Pseudo-Cl EB (Gaussian covariance)") plt.legend() plt.savefig(out_path) - #Plotting BB + # Plotting BB out_path = os.path.abspath(f"{self.cc['paths']['output']}/cell_bb.png") fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(8, 8)) for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data - ax[0].errorbar(ell, ell*self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=ell*np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"], capsize=2) + ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] + cov = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data + ax[0].errorbar( + ell, + ell * self.pseudo_cls[ver]["pseudo_cl"]["BB"], + yerr=ell * np.sqrt(np.diag(cov)), + fmt=self.cc[ver]["marker"], + label=ver + " BB", + color=self.cc[ver]["colour"], + capsize=2, + ) ax[0].axhline(0, color="black", linestyle="--") ax[0].set_ylabel(r"$\ell C_\ell$") - ax[0].set_xlim(ell.min()-10, ell.max()+100) - ax[0].set_xscale('squareroot') + ax[0].set_xlim(ell.min() - 10, ell.max() + 100) + ax[0].set_xscale("squareroot") ax[0].set_xticks(np.array([100, 400, 900, 1600])) ax[0].minorticks_on() - ax[0].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] + ax[0].tick_params(axis="x", which="minor", length=2, width=0.8) + minor_ticks = [i * 10 for i in range(1, 10)] + [ + i * 100 for i in range(1, 21) + ] ax[0].xaxis.set_ticks(minor_ticks, minor=True) for ver in self.versions: - ell = self.pseudo_cls[ver]['pseudo_cl']["ELL"] - cov = self.pseudo_cls[ver]['cov']["COVAR_BB_BB"].data - ax[1].errorbar(ell, self.pseudo_cls[ver]['pseudo_cl']["BB"], yerr=np.sqrt(np.diag(cov)), fmt=self.cc[ver]["marker"], label=ver+" BB", color=self.cc[ver]["colour"]) + ell = self.pseudo_cls[ver]["pseudo_cl"]["ELL"] + cov = self.pseudo_cls[ver]["cov"]["COVAR_BB_BB"].data + ax[1].errorbar( + ell, + self.pseudo_cls[ver]["pseudo_cl"]["BB"], + yerr=np.sqrt(np.diag(cov)), + fmt=self.cc[ver]["marker"], + label=ver + " BB", + color=self.cc[ver]["colour"], + ) ax[1].set_xlabel(r"$\ell$") ax[1].set_ylabel(r"$C_\ell$") - ax[1].set_xlim(ell.min()-10, ell.max()+100) - ax[1].set_xscale('squareroot') - ax[1].set_yscale('log') + ax[1].set_xlim(ell.min() - 10, ell.max() + 100) + ax[1].set_xscale("squareroot") + ax[1].set_yscale("log") ax[1].set_xticks(np.array([100, 400, 900, 1600])) ax[1].minorticks_on() - ax[1].tick_params(axis='x', which='minor', length=2, width=0.8) - minor_ticks = [i*10 for i in range(1, 10)] + [i*100 for i in range(1, 21)] + ax[1].tick_params(axis="x", which="minor", length=2, width=0.8) + minor_ticks = [i * 10 for i in range(1, 10)] + [ + i * 100 for i in range(1, 21) + ] ax[1].xaxis.set_ticks(minor_ticks, minor=True) - plt.suptitle('Pseudo-Cl BB (Gaussian covariance)') + plt.suptitle("Pseudo-Cl BB (Gaussian covariance)") plt.legend() plt.savefig(out_path) From b7650cff0ab52fe10c6b0fe95c773439297a20b0 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 26 Jan 2026 13:41:07 +0100 Subject: [PATCH 13/15] Added v1.6.6 to cosmo_val config --- notebooks/cosmo_val/cat_config.yaml | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/notebooks/cosmo_val/cat_config.yaml b/notebooks/cosmo_val/cat_config.yaml index 5b1c3ed6..87806ec5 100644 --- a/notebooks/cosmo_val/cat_config.yaml +++ b/notebooks/cosmo_val/cat_config.yaml @@ -859,3 +859,49 @@ nz: path: dndz paths: output: ./output +SP_v1.6.6: + subdir: /n17data/UNIONS/WL/v1.6.x + pipeline: SP + colour: orange + getdist_colour: 0.0, 0.5, 1.0 + ls: dashed + marker: d + cov_th: + A: 2405.3892055695346 + n_e: 6.128201234871523 + n_psf: 0.752316232272063 + sigma_e: 0.379587601488189 + mask: /home/guerrini/sp_validation/cosmo_inference/data/mask/mask_map_v1.4.6_nside_8192.fits + psf: + PSF_flag: FLAG_PSF_HSM + PSF_size: SIGMA_PSF_HSM + square_size: true + star_flag: FLAG_STAR_HSM + star_size: SIGMA_STAR_HSM + hdu: 1 + path: unions_shapepipe_psf_2024_v1.6.a.fits + ra_col: RA + dec_col: Dec + e1_PSF_col: E1_PSF_HSM + e1_star_col: E1_STAR_HSM + e2_PSF_col: E2_PSF_HSM + e2_star_col: E2_STAR_HSM + shear: + R: 1.0 + covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt + path: v1.6.6/unions_shapepipe_cut_struc_2024_v1.6.6.fits + redshift_path: /n17data/mkilbing/astro/data/CFIS/v1.0/nz/dndz_SP_A.txt + w_col: w_des + e1_col: e1 + e1_col_corrected: e1_leak_corrected + e1_PSF_col: e1_PSF + e2_col: e2 + e2_col_corrected: e2_leak_corrected + e2_PSF_col: e2_PSF + cols: RA,Dec + star: + ra_col: RA + dec_col: Dec + e1_col: e1 + e2_col: e2 + path: unions_shapepipe_star_2024_v1.6.a.fits From f87dd90e95337e1e33ccec429d5096c4019732d1 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 26 Jan 2026 13:52:01 +0100 Subject: [PATCH 14/15] bug fix on results/results_obj --- src/sp_validation/cosmo_val.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 1dd423df..3e2db0d6 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1387,8 +1387,8 @@ def calculate_objectwise_leakage_aux(self): results_obj.update_params() results_obj.prepare_output() - with self.results_obj.temporarily_load_data(): - results_obj._dat = self.results_obj.dat_shear + with results_obj.temporarily_load_data(): + results_obj._dat = results_obj.dat_shear if not "cols" in results_obj._params: self.print_green( From c8dca03401324782b0c5392a314eeca1a4768168 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 26 Jan 2026 13:52:40 +0100 Subject: [PATCH 15/15] more minor bug fixes --- src/sp_validation/cosmo_val.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sp_validation/cosmo_val.py b/src/sp_validation/cosmo_val.py index 3e2db0d6..751ae056 100644 --- a/src/sp_validation/cosmo_val.py +++ b/src/sp_validation/cosmo_val.py @@ -1387,7 +1387,7 @@ def calculate_objectwise_leakage_aux(self): results_obj.update_params() results_obj.prepare_output() - with results_obj.temporarily_load_data(): + with results_obj.temporarily_read_data(): results_obj._dat = results_obj.dat_shear if not "cols" in results_obj._params: