From 371070d5ddaaf79abc0a4fb834698bd4d6706e5e Mon Sep 17 00:00:00 2001 From: isilber Date: Mon, 13 Oct 2025 22:46:56 +0000 Subject: [PATCH 1/2] DEV: profile eval init and utils module basics Add JSON files for CEIL, RL, MPL, and HSRL --- profiles_eval/configs/ceil.json | 19 +++++++++ profiles_eval/configs/hsrl.json | 41 ++++++++++++++++++++ profiles_eval/configs/mpl.json | 39 +++++++++++++++++++ profiles_eval/configs/rl.json | 38 ++++++++++++++++++ profiles_eval/prof_init.py | 68 +++++++++++++++++++++++++++++++++ profiles_eval/prof_utils.py | 43 +++++++++++++++++++++ 6 files changed, 248 insertions(+) create mode 100644 profiles_eval/configs/ceil.json create mode 100644 profiles_eval/configs/hsrl.json create mode 100644 profiles_eval/configs/mpl.json create mode 100644 profiles_eval/configs/rl.json create mode 100644 profiles_eval/prof_init.py create mode 100644 profiles_eval/prof_utils.py diff --git a/profiles_eval/configs/ceil.json b/profiles_eval/configs/ceil.json new file mode 100644 index 0000000..653acbc --- /dev/null +++ b/profiles_eval/configs/ceil.json @@ -0,0 +1,19 @@ +{ + "instrument_info": { + "name": "CEIL (CL31)" + }, + "variables": { + "time": "time", + "range": "range", + "attenuated_backscatter": "backscatter" + }, + "coordinates": { + "time": "time", + "range": "range" + }, + "units": { + "time": "sec", + "range": "m", + "attenuated_backscatter": "1/(sr*km*10000)" + } +} diff --git a/profiles_eval/configs/hsrl.json b/profiles_eval/configs/hsrl.json new file mode 100644 index 0000000..4ea88d4 --- /dev/null +++ b/profiles_eval/configs/hsrl.json @@ -0,0 +1,41 @@ +{ + "instrument_info": { + "name": "HSRL" + }, + "variables": { + "time": "time", + "range": "range", + "optical_depth": "od_aerosol", + "molecular_attenuated_backscatter": "atten_beta_r_backscatter", + "attenuated_backscatter": "atten_beta_a_backscatter", + "attenuated_backscatter_1064nm": "atten_beta_a_1064nm_backscatter", + "particulate_backscatter": "beta_a_backscatter", + "particulate_backscatter_1064nm": "beta_a_1064_backscatter", + "particulate_extinction": "extinction_aerosol", + "linear_depol_ratio": "linear_depol_ratio", + "linear_depol_ratio_1064nm": "linear_depol_1064", + "molecular_signal_to_noise": "molecular_signal_to_noise", + "particulate_backscatter_signal_to_noise": "particulate_backscatter_signal_to_noise", + "color_ratio": "color_ratio" + }, + "coordinates": { + "time": "time", + "range": "range" + }, + "units": { + "time": "sec", + "range": "m", + "optical_depth": "1", + "molecular_attenuated_backscatter": "1/(m*sr)", + "attenuated_backscatter": "1/(m*sr)", + "attenuated_backscatter_1064nm": "1/(m*sr)", + "particulate_backscatter": "1/(m*sr)", + "particulate_backscatter_1064nm": "1/(m*sr)", + "particulate_extinction": "1/m", + "linear_depol_ratio": "1", + "linear_depol_ratio_1064nm": "1", + "molecular_signal_to_noise": "1", + "particulate_backscatter_signal_to_noise": "1", + "color_ratio": "1" + } +} diff --git a/profiles_eval/configs/mpl.json b/profiles_eval/configs/mpl.json new file mode 100644 index 0000000..964943b --- /dev/null +++ b/profiles_eval/configs/mpl.json @@ -0,0 +1,39 @@ +{ + "instrument_info": { + "name": "MPL" + }, + "variables": { + "time": "time", + "range": "height", + "attenuated_backscatter": "range_corrected_backscatter", + "qc_preprocess_backscatter": "qc_preprocess_backscatter", + "linear_depol_ratio": "linear_depol_ratio", + "qc_linear_depol_ratio": "qc_linear_depol_ratio", + "feature_mask": "cloud_mask", + "qc_feature_mask": "qc_cloud_mask", + "feature_confidence": "cloud_mask_confidence", + "cloud_base": "cloud_base", + "cloud_top": "cloud_top", + "num_cloud_layers": "num_cloud_layers", + "qc_num_cloud_layers": "qc_num_cloud_layers" + }, + "coordinates": { + "time": "time", + "range": "height" + }, + "units": { + "time": "sec", + "range": "km", + "attenuated_backscatter": "km^2*count/us", + "qc_preprocess_backscatter": "1", + "linear_depol_ratio": "1", + "qc_linear_depol_ratio": "1", + "cloud_mask": "1", + "qc_cloud_mask": "1", + "cloud_mask_confidence": "1", + "cloud_base": "km", + "cloud_top": "km", + "num_cloud_layers": "1", + "qc_num_cloud_layers": "1" + } +} \ No newline at end of file diff --git a/profiles_eval/configs/rl.json b/profiles_eval/configs/rl.json new file mode 100644 index 0000000..9407c79 --- /dev/null +++ b/profiles_eval/configs/rl.json @@ -0,0 +1,38 @@ +{ + "instrument_info": { + "name": "RL" + }, + "variables": { + "time": "time", + "range": "height_high", + "range_low": "height_low", + "scattering_ratio": "scattering_ratio_e", + "qc_scattering_ratio": "qc_scattering_ratio_e", + "particulate_backscatter": "particulate_backscatter_be", + "particulate_extinction": "extinction_be", + "lidar_ratio": "lidar_ratio_be", + "linear_depol_ratio": "depolarization_ratio", + "source_lidar_ratio": "source_lidar_ratio_be", + "feature_mask": "feature_mask", + "feature_mask_confidence": "detection_confidence_score_total" + }, + "coordinates": { + "time": "time", + "range": "height_high", + "range_low": "height_low" + }, + "units": { + "time": "sec", + "range": "km", + "range_low": "km", + "scattering_ratio": "1", + "qc_scattering_ratio": "1", + "particulate_backscatter": "1/(km*sr)", + "particulate_extinction": "1/km", + "lidar_ratio": "sr", + "linear_depol_ratio": "1", + "source_lidar_ratio": "1", + "feature_mask": "1", + "feature_mask_confidence": "1" + } +} \ No newline at end of file diff --git a/profiles_eval/prof_init.py b/profiles_eval/prof_init.py new file mode 100644 index 0000000..be1bf10 --- /dev/null +++ b/profiles_eval/prof_init.py @@ -0,0 +1,68 @@ +""" +=============================================================== +Israel Silber +=============================================================== +Lidar profile intercomparison initialization module +=============================================================== +""" + +import json +from pathlib import Path +from typing import Dict, List + + +def load_config(instrument_type: str, config_dir: str = "./configs") -> Dict: + """ + Load configuration for a specific instrument type. + + Parameters + ---------- + instrument_type : str + The instrument type (e.g., 'vaisala_ceilometer', 'hsrl', etc.) + config_dir : str, optional + Directory containing the JSON configuration files, by default "./configs" + + Returns + ------- + Dict + Configuration dictionary + + Raises + ------ + FileNotFoundError + If configuration file is not found + ValueError + If JSON file is invalid + """ + config_file = Path(config_dir) / f"{instrument_type}.json" + + if not config_file.exists(): + raise FileNotFoundError(f"Configuration file not found: {config_file}") + + try: + with open(config_file, 'r') as f: + config = json.load(f) + return config + + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON in config file {config_file}: {e}") + + +def get_variable_mapping(instrument_type: str, config_dir: str = "./configs") -> Dict[str, str]: + """ + Get variable mapping for an instrument type. + + Parameters + ---------- + instrument_type : str + The instrument type + config_dir : str, optional + Directory containing the JSON configuration files, by default "./configs" + + Returns + ------- + Dict[str, str] + Dictionary mapping standard names to instrument-specific names + """ + config = load_config(instrument_type, config_dir) + return config["variables"] \ No newline at end of file diff --git a/profiles_eval/prof_utils.py b/profiles_eval/prof_utils.py new file mode 100644 index 0000000..3c664b8 --- /dev/null +++ b/profiles_eval/prof_utils.py @@ -0,0 +1,43 @@ +""" +=============================================================== +Israel Silber +=============================================================== +Lidar profile intercomparison utility functions +=============================================================== +""" + +from typing import Dict, List, Optional +import prof_init as pi + + +def get_common_variables(instrument_types: List[str], config_dir: str = "./configs") -> List[str]: + """ + Get variables that are common across multiple instrument types. + + Parameters + ---------- + instrument_types : List[str] + List of instrument types to compare + config_dir : str, optional + Directory containing the JSON configuration files, by default "./configs" + + Returns + ------- + List[str] + List of common variable names (standard names) + """ + if not instrument_types: + return [] + + # Load all configs and get variable sets + variable_sets = [] + for inst_type in instrument_types: + config = pi.load_config(inst_type, config_dir) + variable_sets.append(set(config["variables"].keys())) + + # Find intersection + common_vars = variable_sets[0] + for var_set in variable_sets[1:]: + common_vars &= var_set + + return list(common_vars) \ No newline at end of file From e11860edef3f141c9c0cbfbe1f0620e4fb71b070 Mon Sep 17 00:00:00 2001 From: isilber Date: Wed, 15 Oct 2025 16:47:24 +0000 Subject: [PATCH 2/2] DEV: add instrument class and level to JSON files --- profiles_eval/configs/ceil.json | 5 ++++- profiles_eval/configs/hsrl.json | 4 +++- profiles_eval/configs/mpl.json | 7 +++++-- profiles_eval/configs/rl.json | 6 ++++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/profiles_eval/configs/ceil.json b/profiles_eval/configs/ceil.json index 653acbc..4262a85 100644 --- a/profiles_eval/configs/ceil.json +++ b/profiles_eval/configs/ceil.json @@ -1,6 +1,9 @@ { "instrument_info": { - "name": "CEIL (CL31)" + "name": "CEIL (CL31)", + "instrument_class": "ceil10m", + "level": "b1" + }, "variables": { "time": "time", diff --git a/profiles_eval/configs/hsrl.json b/profiles_eval/configs/hsrl.json index 4ea88d4..72d3655 100644 --- a/profiles_eval/configs/hsrl.json +++ b/profiles_eval/configs/hsrl.json @@ -1,6 +1,8 @@ { "instrument_info": { - "name": "HSRL" + "name": "HSRL", + "instrument_class": "hsrl", + "level": "a1" }, "variables": { "time": "time", diff --git a/profiles_eval/configs/mpl.json b/profiles_eval/configs/mpl.json index 964943b..efd350c 100644 --- a/profiles_eval/configs/mpl.json +++ b/profiles_eval/configs/mpl.json @@ -1,6 +1,9 @@ { "instrument_info": { - "name": "MPL" + "name": "MPL", + "instrument_class": "mplcmaskml", + "level": "c1" + }, "variables": { "time": "time", @@ -36,4 +39,4 @@ "num_cloud_layers": "1", "qc_num_cloud_layers": "1" } -} \ No newline at end of file +} diff --git a/profiles_eval/configs/rl.json b/profiles_eval/configs/rl.json index 9407c79..9579b18 100644 --- a/profiles_eval/configs/rl.json +++ b/profiles_eval/configs/rl.json @@ -1,6 +1,8 @@ { "instrument_info": { - "name": "RL" + "name": "RL", + "instrument_class": "rlproffex1thor", + "level": "c0" }, "variables": { "time": "time", @@ -35,4 +37,4 @@ "feature_mask": "1", "feature_mask_confidence": "1" } -} \ No newline at end of file +}