diff --git a/bps/caching/LSSTCam/frdf/DRP-caching.yaml b/bps/caching/LSSTCam/frdf/DRP-caching.yaml new file mode 100755 index 00000000..b64dc982 --- /dev/null +++ b/bps/caching/LSSTCam/frdf/DRP-caching.yaml @@ -0,0 +1,21 @@ +# There are recommended configs for caching data read from the S3 +# datastore used by the embargo rack at USDF. +# +# Use it by adding +# +# includeConfigs: +# - ${DRP_PIPE_DIR}/bps/caching/LSSTCam/frdf/DRP-caching.yaml +# +# (with no outer indentation) to your BPS config file. This is meant +# to be used with the clustering definintions in +# ${DRP_PIPE_DIR}/bps/clustering/LSSTCam + +subDirTemplate: "{label}/{detector}" + +environment: + # Default caching config + # Each pipetask creates its own local cache folder, + # $TMPDIR/butler-cache-dir-* + DAF_BUTLER_CONFIG_PATH: ${DRP_PIPE_DIR}/bps/caching/LSSTCam/frdf/config:${DAF_BUTLER_CONFIG_PATH} + DAF_BUTLER_CACHE_EXPIRATION_MODE: size=40_000_000_000 + DAF_BUTLER_CACHE_DIRECTORY: ${TMPDIR} diff --git a/bps/caching/LSSTCam/frdf/config/datastores/fileDatastore.yaml b/bps/caching/LSSTCam/frdf/config/datastores/fileDatastore.yaml new file mode 100755 index 00000000..084d0d54 --- /dev/null +++ b/bps/caching/LSSTCam/frdf/config/datastores/fileDatastore.yaml @@ -0,0 +1,135 @@ +datastore: + formatters: + preliminary_visit_image: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 + visit_image: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 + deep_coadd: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 + template_coadd: + formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter + parameters: + recipe: lossy16 + cached: + # Explicit location of a cache directory root that can be shared + # between multiple Butler instantiations. Without this the Butler + # will use a new temporary location each time. + # Instantiating a Butler from another Butler could propagate temporary + # location. This can be left out or null used + # to indicate a temporary directory. + root: null + expiry: + # Expiry mode and associated threshold. + # Options are: + # - null (no expiry) + # - disabled (no caching) + # - files (threshold is number of files) + # - datasets (threshold is number of datasets) + # - size (threshold is size in bytes) + # - age (threshold is oldest file in seconds) + # Default for everyone is to limit the cache size to 4 datasets. + mode: size + threshold: 40000000000 + # Default cache value. This will be the default decision if no specific + # match is found in the "cacheable" section later on. + default: false + + # For the dataset types to target for caching, we consider the + # pipetasks/clusters that have the largest numbers of butler put + # operations. + # For per-pipetask cache areas, caching is useful when + # * the input dataset types have dataset dimensions along + # the clustered dimensions, + # * the input dataset types have dimensions that can be transformed + # to clustered dimensions, e.g., detector -> patch + # * output datasets are passed directly to a downstream pipetask + # in the same cluster + # Caching of dataset types that only have dimensions that overlap the + # partition dimension of a cluster isn't needed. + # Use a dict over list to simplify merging logic. + cacheable: + # Disable generic storage classes + Exposure: false + ExposureF: false + + # skymap + skyMap: true + + # step1detector (isr, calibrateImage, standardizeSingleVisitStar, ...) + bias: true + dark: true + flat: true + bfk: true + crosstalk: true + post_isr_image: true + single_visit_star_unstandardized: true + camera: true + astrometry_camera: true + # step2d_refitpsf (refitPsfModelDetector) + single_visit_star_footprints: true + isolated_star: true + isolated_star_association: true + single_visit_star_schema: true + + # step2d_visits (recalibrateSingleVisitStar, standardizeRecalibratedStar) + # single_visit_star_footprints already added above + recalibrated_star_unstandardized: true + + # step2d_visit_summary (consolidateRefitPsfModelDetector, updateVisitSummary) + refit_psf_models_detector: true + refit_psf_star_detector: true + refit_psf_models: true + refit_psf_star: true + + # makeWarpTract (makeDirectWarp, makePsfMatchedWarp,makePrettyDirectWarp,makePrettyPsfMatchedWarp) + preliminary_visit_image: true + direct_warp: true + pretty_direct_warp: true + preliminary_visit_summary: true + visit_summary: true + + # coadd (assembleDeepCoadd, assembleTemplateCoadd, detectCoaddPeaks, assemblePrettyCoadd) + psf_matched_warp: true + pretty_psf_matched_warp: true + deep_coadd_visit_selection: true + template_coadd_visit_selection: true + deep_coadd_predetection: true + + # step4a_reprocess_standardize (reprocessVisitImage, standardizeSource) + preliminary_visit_mask: true + preliminary_visit_image_background: true + source_unstandardized: true + + # diffim (rewarpTemplate, subtractImages, detectAndMeasureDiaSource, + # filterDiaSource, computeReliability, standardizeDiaSource) + template_coadd: true + visit_image: true + template_detector: true + difference_image_predetection: true + template_matched: true + difference_image: true + dia_source_unfiltered: true + dia_source_unstandardized: true + pretrainedModelPackage: true + dia_source_reliability: true + difference_kernel_sources: true + + # association (associateDiaSource, calculateDiaObject) + preloaded_DRP_SsObjects: true + dia_source_detector: true + dia_object_pre_calc: true + dia_source_patch: true + + # step4c_forced_phot (forcedPhotObjectDetector) + object_patch: true + object_forced_source_unstandardized: true + + # step4c_forced_phot_dia (forcedPhotDiaObjectDetector) + dia_object_patch: true + dia_object_forced_source_unstandardized: true