From 52f50cf1429366f4d46b8952b58fee2312954d13 Mon Sep 17 00:00:00 2001 From: Yusra AlSayyad Date: Mon, 8 Dec 2025 08:31:52 -0800 Subject: [PATCH 1/2] Add compression yaml --- .../LSSTCam/DRP/DRP-compression.yaml | 11 +++++++++++ .../DRP/config/datastores/fileDatastore.yaml | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 bps/compression/LSSTCam/DRP/DRP-compression.yaml create mode 100644 bps/compression/LSSTCam/DRP/config/datastores/fileDatastore.yaml diff --git a/bps/compression/LSSTCam/DRP/DRP-compression.yaml b/bps/compression/LSSTCam/DRP/DRP-compression.yaml new file mode 100644 index 00000000..160ab42c --- /dev/null +++ b/bps/compression/LSSTCam/DRP/DRP-compression.yaml @@ -0,0 +1,11 @@ +# There are recommended configs for compression for LSSTCam DRP +# +# Use it by adding +# +# includeConfigs: +# - ${DRP_PIPE_DIR}/bps/compression/LSSTCam/DRP/DRP-compression.yaml +# +# (with no outer indentation) to your BPS config file. + +environment: + DAF_BUTLER_CONFIG_PATH: ${DRP_PIPE_DIR}/bps/compression/LSSTCam/DRP/config:${DAF_BUTLER_CONFIG_PATH} diff --git a/bps/compression/LSSTCam/DRP/config/datastores/fileDatastore.yaml b/bps/compression/LSSTCam/DRP/config/datastores/fileDatastore.yaml new file mode 100644 index 00000000..6a65902d --- /dev/null +++ b/bps/compression/LSSTCam/DRP/config/datastores/fileDatastore.yaml @@ -0,0 +1,17 @@ +formatters: + preliminary_visit_image: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 + visit_image: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 + deep_coadd: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 + template_coadd: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 From 7d518f56d21c18ce710ea40099218cebe072db8f Mon Sep 17 00:00:00 2001 From: Yusra AlSayyad Date: Thu, 18 Dec 2025 19:02:33 -0800 Subject: [PATCH 2/2] Enable composed compression and caching butler configs --- bps/caching/LSSTCam/usdf/DRP-caching.yaml | 6 +++++- bps/compression/LSSTCam/DRP/DRP-compression.yaml | 6 +++++- bps/daf_butler_config_path/compose.yaml | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 bps/daf_butler_config_path/compose.yaml diff --git a/bps/caching/LSSTCam/usdf/DRP-caching.yaml b/bps/caching/LSSTCam/usdf/DRP-caching.yaml index d37b324a..54392a93 100644 --- a/bps/caching/LSSTCam/usdf/DRP-caching.yaml +++ b/bps/caching/LSSTCam/usdf/DRP-caching.yaml @@ -12,11 +12,15 @@ subDirTemplate: "{label}/{detector}" +includeConfigs: + # DAF_BUTLER_CONFIG_PATH is set here: + - ${DRP_PIPE_DIR}/bps/daf_butler_config_path/compose.yaml + environment: # Default caching config # Each pipetask creates its own local cache folder, # $TMPDIR/butler-cache-dir-* - DAF_BUTLER_CONFIG_PATH: ${DRP_PIPE_DIR}/bps/caching/LSSTCam/usdf/config:${DAF_BUTLER_CONFIG_PATH} + CACHING_CONFIG_PATH: ${DRP_PIPE_DIR}/bps/caching/LSSTCam/usdf/config DAF_BUTLER_CACHE_EXPIRATION_MODE: datasets=500 cluster: diff --git a/bps/compression/LSSTCam/DRP/DRP-compression.yaml b/bps/compression/LSSTCam/DRP/DRP-compression.yaml index 160ab42c..541fabc8 100644 --- a/bps/compression/LSSTCam/DRP/DRP-compression.yaml +++ b/bps/compression/LSSTCam/DRP/DRP-compression.yaml @@ -7,5 +7,9 @@ # # (with no outer indentation) to your BPS config file. +includeConfigs: + # DAF_BUTLER_CONFIG_PATH is set here: + - ${DRP_PIPE_DIR}/bps/daf_butler_config_path/compose.yaml + environment: - DAF_BUTLER_CONFIG_PATH: ${DRP_PIPE_DIR}/bps/compression/LSSTCam/DRP/config:${DAF_BUTLER_CONFIG_PATH} + COMPRESSION_CONFIG_PATH: ${DRP_PIPE_DIR}/bps/compression/LSSTCam/DRP/config diff --git a/bps/daf_butler_config_path/compose.yaml b/bps/daf_butler_config_path/compose.yaml new file mode 100644 index 00000000..2b225f15 --- /dev/null +++ b/bps/daf_butler_config_path/compose.yaml @@ -0,0 +1,2 @@ +environment: + DAF_BUTLER_CONFIG_PATH: "{COMPRESSION_CONFIG_PATH}:{CACHING_CONFIG_PATH}:${DAF_BUTLER_CONFIG_PATH}"