From 3b2a55623eadfedf62fa5b201b99785b37d4bf24 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Thu, 6 Mar 2025 23:14:16 -0800 Subject: [PATCH 01/10] Update to use new IsrTaskLSST --- python/lsst/summit/utils/bestEffort.py | 6 ++-- python/lsst/summit/utils/quickLook.py | 31 +++++++------------ .../utils/resources/config/quickLookIsr.py | 8 +++-- 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/python/lsst/summit/utils/bestEffort.py b/python/lsst/summit/utils/bestEffort.py index cfcf56f5..ad7d3b45 100644 --- a/python/lsst/summit/utils/bestEffort.py +++ b/python/lsst/summit/utils/bestEffort.py @@ -26,7 +26,7 @@ import lsst.afw.image as afwImage import lsst.daf.butler as dafButler from lsst.daf.butler.registry import ConflictingDefinitionError -from lsst.ip.isr import IsrTask +from lsst.ip.isr import IsrTaskLSST from lsst.pex.config import Config from lsst.summit.utils.butlerUtils import getLatissDefaultCollections from lsst.summit.utils.quickLook import QuickLookIsrTask @@ -58,7 +58,7 @@ class BestEffortIsr: Extra collections to add to the butler init. Collections are prepended. defaultExtraIsrOptions : `dict`, optional A dict of extra isr config options to apply. Each key should be an - attribute of an isrTaskConfigClass. + attribute of an isrTaskLSSTConfigClass. doRepairCosmics : `bool`, optional Repair cosmic ray hits? doWrite : `bool`, optional @@ -238,7 +238,7 @@ def getExposure( raise RuntimeError(f"Failed to retrieve raw for exp {dataId}") from None # default options that are probably good for most engineering time - isrConfig = IsrTask.ConfigClass() + isrConfig = IsrTaskLSST.ConfigClass() with importlib.resources.path("lsst.summit.utils", "resources/config/quickLookIsr.py") as cfgPath: isrConfig.load(cfgPath) diff --git a/python/lsst/summit/utils/quickLook.py b/python/lsst/summit/utils/quickLook.py index c309961e..c46c9c6c 100644 --- a/python/lsst/summit/utils/quickLook.py +++ b/python/lsst/summit/utils/quickLook.py @@ -31,15 +31,15 @@ import lsst.pex.config as pexConfig import lsst.pipe.base as pipeBase import lsst.pipe.base.connectionTypes as cT -from lsst.ip.isr import IsrTask -from lsst.ip.isr.isrTask import IsrTaskConnections +from lsst.ip.isr import IsrTaskLSST +from lsst.ip.isr.isrTaskLSST import IsrTaskLSSTConnections from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask __all__ = ["QuickLookIsrTask", "QuickLookIsrTaskConfig"] -class QuickLookIsrTaskConnections(IsrTaskConnections): +class QuickLookIsrTaskConnections(IsrTaskLSSTConnections): """Copy isrTask's connections, changing prereq min values to zero. Copy all the connections directly for IsrTask, keeping ccdExposure as @@ -50,7 +50,9 @@ class QuickLookIsrTaskConnections(IsrTaskConnections): def __init__(self, *, config: Any = None): # programatically clone all of the connections from isrTask # setting minimum values to zero for everything except the ccdExposure - super().__init__(config=IsrTask.ConfigClass()) # need a dummy config, isn't used other than for ctor + super().__init__( + config=IsrTaskLSST.ConfigClass() + ) # need a dummy config, isn't used other than for ctor for name, connection in self.allConnections.items(): if hasattr(connection, "minimum"): setattr( @@ -95,13 +97,13 @@ class QuickLookIsrTask(pipeBase.PipelineTask): config: QuickLookIsrTaskConfig _DefaultName = "quickLook" - def __init__(self, isrTask: IsrTask = IsrTask, **kwargs: Any): + def __init__(self, isrTask: IsrTaskLSST = IsrTaskLSST, **kwargs: Any): super().__init__(**kwargs) # Pass in IsrTask so that we can modify it slightly for unit tests. # Note that this is not an instance of the IsrTask class, but the class # itself, which is then instantiated later on, in the run() method, # with the dynamically generated config. - self.isrTask = IsrTask + self.isrTask = IsrTaskLSST def run( self, @@ -119,7 +121,7 @@ def run( newBFKernel: ipIsr.BrighterFatterKernel | None = None, ptc: ipIsr.PhotonTransferCurveDataset | None = None, crosstalkSources: list | None = None, - isrBaseConfig: ipIsr.IsrTaskConfig | None = None, + isrBaseConfig: ipIsr.IsrTaskLSSTConfig | None = None, filterTransmission: afwImage.TransmissionCurve | None = None, opticsTransmission: afwImage.TransmissionCurve | None = None, strayLightData: Any | None = None, @@ -169,7 +171,7 @@ def run( and read noise. crosstalkSources : `list`, optional List of possible crosstalk sources. - isrBaseConfig : `lsst.ip.isr.IsrTaskConfig`, optional + isrBaseConfig : `lsst.ip.isr.IsrTaskLSSTConfig`, optional An isrTask config to act as the base configuration. Options which involve applying a calibration product are ignored, but this allows for the configuration of e.g. the number of overscan columns. @@ -204,7 +206,7 @@ def run( The ISRed and cosmic-ray-repaired exposure. """ if not isrBaseConfig: - isrConfig = IsrTask.ConfigClass() + isrConfig = IsrTaskLSST.ConfigClass() with importlib.resources.path("lsst.summit.utils", "resources/config/quickLookIsr.py") as cfgPath: isrConfig.load(cfgPath) else: @@ -213,12 +215,10 @@ def run( isrConfig.doBias = False isrConfig.doDark = False isrConfig.doFlat = False - isrConfig.doFringe = False isrConfig.doDefect = False isrConfig.doLinearize = False isrConfig.doCrosstalk = False isrConfig.doBrighterFatter = False - isrConfig.usePtcGains = False if bias: isrConfig.doBias = True @@ -232,10 +232,6 @@ def run( isrConfig.doFlat = True self.log.info("Running with flat correction") - if fringes: - isrConfig.doFringe = True - self.log.info("Running with fringe correction") - if defects: isrConfig.doDefect = True self.log.info("Running with defect correction") @@ -259,11 +255,6 @@ def run( isrConfig.doBrighterFatter = True self.log.info("Running with brighter-fatter correction") - if ptc: - isrConfig.usePtcGains = True - self.log.info("Running with ptc correction") - - isrConfig.doWrite = False isrTask = self.isrTask(config=isrConfig) if fringes: diff --git a/python/lsst/summit/utils/resources/config/quickLookIsr.py b/python/lsst/summit/utils/resources/config/quickLookIsr.py index f08a75d1..2d8ea268 100644 --- a/python/lsst/summit/utils/resources/config/quickLookIsr.py +++ b/python/lsst/summit/utils/resources/config/quickLookIsr.py @@ -2,7 +2,9 @@ config.doWrite = False # this task writes separately, no need for this config.doSaturation = True # saturation very important for roundness measurement in qfm -config.doSaturationInterpolation = True -config.overscan.fitType = "MEDIAN_PER_ROW" -config.overscan.doParallelOverscan = True config.brighterFatterMaxIter = 2 # Uncomment this to remove test warning +config.doDeferredCharge = False # no calib for this yet +config.doBootstrap = True +config.doApplyGains = False +config.doSuspect = False +config.defaultSaturationSource = "CAMERAMODEL" From 997525a3733c56a0f987cbbd73056c8785866b76 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Thu, 13 Nov 2025 14:15:41 -0800 Subject: [PATCH 02/10] Remove doBootstrap config option --- python/lsst/summit/utils/resources/config/quickLookIsr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/summit/utils/resources/config/quickLookIsr.py b/python/lsst/summit/utils/resources/config/quickLookIsr.py index 2d8ea268..1a464c5c 100644 --- a/python/lsst/summit/utils/resources/config/quickLookIsr.py +++ b/python/lsst/summit/utils/resources/config/quickLookIsr.py @@ -4,7 +4,7 @@ config.doSaturation = True # saturation very important for roundness measurement in qfm config.brighterFatterMaxIter = 2 # Uncomment this to remove test warning config.doDeferredCharge = False # no calib for this yet -config.doBootstrap = True +config.doBootstrap = False config.doApplyGains = False config.doSuspect = False config.defaultSaturationSource = "CAMERAMODEL" From 0931d66af43609e07d89e4ebb49788820c0d4793 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Fri, 14 Nov 2025 15:41:16 -0800 Subject: [PATCH 03/10] Turn gain correction on --- python/lsst/summit/utils/resources/config/quickLookIsr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/summit/utils/resources/config/quickLookIsr.py b/python/lsst/summit/utils/resources/config/quickLookIsr.py index 1a464c5c..60d63c96 100644 --- a/python/lsst/summit/utils/resources/config/quickLookIsr.py +++ b/python/lsst/summit/utils/resources/config/quickLookIsr.py @@ -5,6 +5,6 @@ config.brighterFatterMaxIter = 2 # Uncomment this to remove test warning config.doDeferredCharge = False # no calib for this yet config.doBootstrap = False -config.doApplyGains = False +config.doApplyGains = True config.doSuspect = False config.defaultSaturationSource = "CAMERAMODEL" From 8095070f406d07c79c6334366ea311d22ca042b1 Mon Sep 17 00:00:00 2001 From: Christopher Waters Date: Thu, 20 Nov 2025 15:50:28 -0800 Subject: [PATCH 04/10] CZW/RB: Working through IsrTaskLSST issues. --- pipelines/quickLook.yaml | 2 + python/lsst/summit/utils/quickLook.py | 93 +++++++------------- tests/test_bestEffortIsr.py | 2 +- tests/test_quickLook.py | 118 +++++++++----------------- 4 files changed, 74 insertions(+), 141 deletions(-) diff --git a/pipelines/quickLook.yaml b/pipelines/quickLook.yaml index a48c0cce..ba7be659 100644 --- a/pipelines/quickLook.yaml +++ b/pipelines/quickLook.yaml @@ -4,3 +4,5 @@ tasks: class: lsst.summit.utils.quickLook.QuickLookIsrTask config: doRepairCosmics: true + doCorrectGains: false + doDeferredCharge: false diff --git a/python/lsst/summit/utils/quickLook.py b/python/lsst/summit/utils/quickLook.py index c46c9c6c..9f3ad19f 100644 --- a/python/lsst/summit/utils/quickLook.py +++ b/python/lsst/summit/utils/quickLook.py @@ -23,7 +23,6 @@ import importlib.resources from typing import Any -import numpy as np import lsst.afw.cameraGeom as camGeom import lsst.afw.image as afwImage @@ -103,6 +102,9 @@ def __init__(self, isrTask: IsrTaskLSST = IsrTaskLSST, **kwargs: Any): # Note that this is not an instance of the IsrTask class, but the class # itself, which is then instantiated later on, in the run() method, # with the dynamically generated config. + # import pdb; pdb.set_trace() + if IsrTaskLSST._DefaultName != 'isrLSST': + raise RuntimeError("QuickLookIsrTask should now always use IsrTaskLSST for processing.") self.isrTask = IsrTaskLSST def run( @@ -113,22 +115,14 @@ def run( bias: afwImage.Exposure | None = None, dark: afwImage.Exposure | None = None, flat: afwImage.Exposure | None = None, - fringes: afwImage.Exposure | None = None, defects: ipIsr.Defects | None = None, linearizer: ipIsr.linearize.LinearizeBase | None = None, crosstalk: ipIsr.crosstalk.CrosstalkCalib | None = None, - bfKernel: np.ndarray | None = None, - newBFKernel: ipIsr.BrighterFatterKernel | None = None, + bfKernel: ipIsr.BrighterFatterKernel | None = None, ptc: ipIsr.PhotonTransferCurveDataset | None = None, - crosstalkSources: list | None = None, isrBaseConfig: ipIsr.IsrTaskLSSTConfig | None = None, - filterTransmission: afwImage.TransmissionCurve | None = None, - opticsTransmission: afwImage.TransmissionCurve | None = None, - strayLightData: Any | None = None, - sensorTransmission: afwImage.TransmissionCurve | None = None, - atmosphereTransmission: afwImage.TransmissionCurve | None = None, deferredChargeCalib: Any | None = None, - illumMaskedImage: afwImage.MaskedImage | None = None, + gainCorrection: ipIsr.IsrCalib | None = None, ) -> pipeBase.Struct: """Run isr and cosmic ray repair using, doing as much isr as possible. @@ -162,41 +156,17 @@ def run( Functor for linearization. crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional Calibration for crosstalk. - bfKernel : `numpy.ndarray`, optional - Brighter-fatter kernel. - newBFKernel : `ipIsr.BrighterFatterKernel`, optional + bfKernel : `ipIsr.BrighterFatterKernel`, optional New Brighter-fatter kernel. ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional Photon transfer curve dataset, with, e.g., gains and read noise. - crosstalkSources : `list`, optional - List of possible crosstalk sources. + cti : `lsst.ip.isr.DeferredChargeCalib`, optional + Charge transfer inefficiency correction calibration. isrBaseConfig : `lsst.ip.isr.IsrTaskLSSTConfig`, optional An isrTask config to act as the base configuration. Options which involve applying a calibration product are ignored, but this allows for the configuration of e.g. the number of overscan columns. - filterTransmission : `lsst.afw.image.TransmissionCurve` - A ``TransmissionCurve`` that represents the throughput of the - filter itself, to be evaluated in focal-plane coordinates. - opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional - A ``TransmissionCurve`` that represents the throughput of the, - optics, to be evaluated in focal-plane coordinates. - strayLightData : `object`, optional - Opaque object containing calibration information for stray-light - correction. If `None`, no correction will be performed. - sensorTransmission : `lsst.afw.image.TransmissionCurve` - A ``TransmissionCurve`` that represents the throughput of the - sensor itself, to be evaluated in post-assembly trimmed detector - coordinates. - atmosphereTransmission : `lsst.afw.image.TransmissionCurve` - A ``TransmissionCurve`` that represents the throughput of the - atmosphere, assumed to be spatially constant. - illumMaskedImage : `lsst.afw.image.MaskedImage`, optional - Illumination correction image. - bfGains : `dict` of `float`, optional - Gains used to override the detector's nominal gains for the - brighter-fatter correction. A dict keyed by amplifier name for - the detector in question. Returns ------- @@ -219,6 +189,8 @@ def run( isrConfig.doLinearize = False isrConfig.doCrosstalk = False isrConfig.doBrighterFatter = False + isrConfig.doDeferredCharge = False + isrConfig.doCorrectGains = False if bias: isrConfig.doBias = True @@ -244,26 +216,31 @@ def run( isrConfig.doCrosstalk = True self.log.info("Running with crosstalk correction") - if newBFKernel is not None: - bfGains = newBFKernel.gain - isrConfig.doBrighterFatter = True - self.log.info("Running with new brighter-fatter correction") - else: - bfGains = None - - if bfKernel is not None and bfGains is None: + if bfKernel is not None: isrConfig.doBrighterFatter = True self.log.info("Running with brighter-fatter correction") + if deferredChargeCalib is not None: + isrConfig.doDeferredCharge = True + self.log.info("Running with CTI correction") + + if gainCorrection is not None: + isrConfig.doCorrectGains = True + self.log.info("Running with Gain corrections") + + if ptc is None: + raise RuntimeError("IsrTaskLSST requires a PTC.") + isrTask = self.isrTask(config=isrConfig) - if fringes: - # Must be run after isrTask is instantiated. - isrTask.fringe.loadFringes( - fringes, - expId=ccdExposure.info.id, - assembler=isrTask.assembleCcd if isrConfig.doAssembleIsrExposures else None, - ) + # DM-47959: TODO Add fringe correction to IsrTaskLSST. + # if fringes: + # # Must be run after isrTask is instantiated. + # isrTask.fringe.loadFringes( + # fringes, + # expId=ccdExposure.info.id, + # assembler=isrTask.assembleCcd if isrConfig.doAssembleIsrExposures else None, + # ) result = isrTask.run( ccdExposure, @@ -271,21 +248,13 @@ def run( bias=bias, dark=dark, flat=flat, - fringes=fringes, defects=defects, linearizer=linearizer, crosstalk=crosstalk, bfKernel=bfKernel, - bfGains=bfGains, ptc=ptc, - crosstalkSources=crosstalkSources, - filterTransmission=filterTransmission, - opticsTransmission=opticsTransmission, - sensorTransmission=sensorTransmission, - atmosphereTransmission=atmosphereTransmission, - strayLightData=strayLightData, deferredChargeCalib=deferredChargeCalib, - illumMaskedImage=illumMaskedImage, + gainCorrection=gainCorrection, ) postIsr = result.exposure diff --git a/tests/test_bestEffortIsr.py b/tests/test_bestEffortIsr.py index 82f38de8..2bd2cad4 100644 --- a/tests/test_bestEffortIsr.py +++ b/tests/test_bestEffortIsr.py @@ -40,7 +40,7 @@ def setUpClass(cls): # NCSA - LATISS/raw/all # TTS - LATISS-test-data-tts # summit - LATISS_test_data - cls.dataId = {"day_obs": 20210121, "seq_num": 743, "detector": 0} + cls.dataId = {"day_obs": 20251021, "seq_num": 443, "detector": 0} def test_getExposure(self): # in most locations this will load a pre-made image diff --git a/tests/test_quickLook.py b/tests/test_quickLook.py index 3757d51b..90e7ee63 100644 --- a/tests/test_quickLook.py +++ b/tests/test_quickLook.py @@ -23,13 +23,11 @@ import tempfile import unittest -import numpy as np - import lsst.afw.cameraGeom.testUtils as afwTestUtils import lsst.afw.image as afwImage import lsst.daf.butler.tests as butlerTests import lsst.ip.isr as ipIsr -import lsst.ip.isr.isrMock as isrMock +import lsst.ip.isr.isrMockLSST as isrMock import lsst.pex.exceptions import lsst.pipe.base as pipeBase import lsst.pipe.base.testUtils @@ -42,25 +40,23 @@ class QuickLookIsrTaskTestCase(unittest.TestCase): """Tests of the run method with fake data.""" def setUp(self): - self.mockConfig = isrMock.IsrMockConfig() - self.dataContainer = isrMock.MockDataContainer(config=self.mockConfig) - self.camera = isrMock.IsrMock(config=self.mockConfig).getCamera() + self.mockConfig = isrMock.IsrMockLSSTConfig() + self.camera = isrMock.IsrMockLSST(config=self.mockConfig).getCamera() - self.ccdExposure = isrMock.RawMock(config=self.mockConfig).run() + self.ccdExposure = isrMock.RawMockLSST(config=self.mockConfig).run() self.detector = self.ccdExposure.getDetector() amps = self.detector.getAmplifiers() ampNames = [amp.getName() for amp in amps] # # Mock other optional parameters - self.bias = self.dataContainer.get("bias") - self.dark = self.dataContainer.get("dark") - self.flat = self.dataContainer.get("flat") - self.defects = self.dataContainer.get("defects") + self.bias = isrMock.BiasMockLSST(config=self.mockConfig).run() + self.dark = isrMock.DarkMockLSST(config=self.mockConfig).run() + self.flat = isrMock.FlatMockLSST(config=self.mockConfig).run() + self.defects = isrMock.DefectMockLSST(config=self.mockConfig).run() self.ptc = ipIsr.PhotonTransferCurveDataset(ampNames=ampNames) # Mock PTC dataset - self.bfKernel = self.dataContainer.get("bfKernel") - self.newBFKernel = pipeBase.Struct(gain={}) - for amp_i, amp in enumerate(ampNames): - self.newBFKernel.gain[amp] = 0.9 + 0.1 * amp_i + for amp, gain in self.mockConfig.gainDict.items(): + self.ptc.gain[amp] = 1.0 + self.bfKernel = isrMock.BfKernelMockLSST(config=self.mockConfig).run() self.task = QuickLookIsrTask(config=QuickLookIsrTaskConfig()) def test_runQuickLook(self): @@ -75,9 +71,7 @@ def test_runQuickLook(self): linearizer=None, crosstalk=None, bfKernel=self.bfKernel, - newBFKernel=self.newBFKernel, ptc=self.ptc, - crosstalkSources=None, ) self.assertIsNotNone(result, "Result of run method should not be None") self.assertIsInstance(result, pipeBase.Struct, "Result should be of type lsst.pipe.base.Struct") @@ -88,8 +82,8 @@ def test_runQuickLook(self): ) def test_runQuickLookMissingData(self): - # Test without any inputs other than the exposure - result = self.task.run(self.ccdExposure) + # Test without any inputs other than the exposure. And the PTC. + result = self.task.run(self.ccdExposure, ptc=self.ptc) self.assertIsInstance(result.exposure, afwImage.Exposure) def test_runQuickLookBadDark(self): @@ -130,23 +124,16 @@ def setUp(self): dark = "dark" flat = "flat" defects = "defects" - bfKernel = "bfKernel" - newBFKernel = "brighterFatterKernel" + bfKernel = "bfk" ptc = "ptc" - filterTransmission = "transmission_filter" - deferredChargeCalib = "cpCtiCalib" - opticsTransmission = "transmission_optics" - strayLightData = "yBackground" - atmosphereTransmission = "transmission_atmosphere" + deferredChargeCalib = "cti" crosstalk = "crosstalk" - illumMaskedImage = "illum" linearizer = "linearizer" - fringes = "fringe" - sensorTransmission = "transmission_sensor" - crosstalkSources = "isrOverscanCorrected" + gainCorrection = "gain_correction" # outputs outputExposure = "postISRCCD" + outputStatistics = "isrStatistics" # quickLook-only outputs exposure = "quickLookExp" @@ -171,36 +158,17 @@ def setUp(self): butlerTests.addDatasetType(self.repo, defects, {"instrument", "detector"}, "Defects") butlerTests.addDatasetType(self.repo, linearizer, {"instrument", "detector"}, "Linearizer") butlerTests.addDatasetType(self.repo, crosstalk, {"instrument", "detector"}, "CrosstalkCalib") - butlerTests.addDatasetType(self.repo, bfKernel, {"instrument"}, "NumpyArray") - butlerTests.addDatasetType(self.repo, newBFKernel, {"instrument", "detector"}, "BrighterFatterKernel") + butlerTests.addDatasetType(self.repo, bfKernel, {"instrument", "detector"}, "BrighterFatterKernel") butlerTests.addDatasetType(self.repo, ptc, {"instrument", "detector"}, "PhotonTransferCurveDataset") - butlerTests.addDatasetType( - self.repo, filterTransmission, {"instrument", "physical_filter"}, "TransmissionCurve" - ) - butlerTests.addDatasetType(self.repo, opticsTransmission, {"instrument"}, "TransmissionCurve") butlerTests.addDatasetType(self.repo, deferredChargeCalib, {"instrument", "detector"}, "IsrCalib") - butlerTests.addDatasetType( - self.repo, strayLightData, {"instrument", "physical_filter", "detector"}, "Exposure" - ) - butlerTests.addDatasetType(self.repo, atmosphereTransmission, {"instrument"}, "TransmissionCurve") - butlerTests.addDatasetType( - self.repo, illumMaskedImage, {"instrument", "physical_filter", "detector"}, "MaskedImage" - ) - butlerTests.addDatasetType( - self.repo, fringes, {"instrument", "physical_filter", "detector"}, "Exposure" - ) - butlerTests.addDatasetType( - self.repo, sensorTransmission, {"instrument", "detector"}, "TransmissionCurve" - ) - butlerTests.addDatasetType( - self.repo, crosstalkSources, {"instrument", "exposure", "detector"}, "Exposure" - ) + butlerTests.addDatasetType(self.repo, gainCorrection, {"instrument", "detector"}, "IsrCalib") # outputs butlerTests.addDatasetType( self.repo, outputExposure, {"instrument", "exposure", "detector"}, "Exposure" ) butlerTests.addDatasetType(self.repo, exposure, {"instrument", "exposure", "detector"}, "Exposure") + butlerTests.addDatasetType(self.repo, outputStatistics, {"instrument", "exposure", "detector"}, "StructuredDataDict") # dataIds self.exposure_id = self.repo.registry.expandDataId( @@ -221,9 +189,6 @@ def setUp(self): ) # put empty data - transmissionCurve = TransmissionCurve.makeSpatiallyConstant( - np.ones(2), np.linspace(0, 1, 2), 0.0, 0.0 - ) self.butler = butlerTests.makeTestCollection(self.repo) self.butler.put(afwImage.ExposureF(), ccdExposure, self.exposure_id) self.butler.put(afwTestUtils.CameraWrapper().camera, camera, self.instrument_id) @@ -231,22 +196,14 @@ def setUp(self): self.butler.put(afwImage.ExposureF(), dark, self.detector_id) self.butler.put(afwImage.ExposureF(), flat, self.flat_id) self.butler.put(lsst.ip.isr.Defects(), defects, self.detector_id) - self.butler.put(np.zeros(2), bfKernel, self.instrument_id) self.butler.put( - lsst.ip.isr.brighterFatterKernel.BrighterFatterKernel(), newBFKernel, self.detector_id + lsst.ip.isr.brighterFatterKernel.BrighterFatterKernel(), bfKernel, self.detector_id ) self.butler.put(ipIsr.PhotonTransferCurveDataset(), ptc, self.detector_id) - self.butler.put(transmissionCurve, filterTransmission, self.filter_id) self.butler.put(lsst.ip.isr.calibType.IsrCalib(), deferredChargeCalib, self.detector_id) - self.butler.put(transmissionCurve, opticsTransmission, self.instrument_id) - self.butler.put(afwImage.ExposureF(), strayLightData, self.flat_id) - self.butler.put(transmissionCurve, atmosphereTransmission, self.instrument_id) self.butler.put(lsst.ip.isr.crosstalk.CrosstalkCalib(), crosstalk, self.detector_id) - self.butler.put(afwImage.ExposureF().maskedImage, illumMaskedImage, self.flat_id) self.butler.put(lsst.ip.isr.linearize.Linearizer(), linearizer, self.detector_id) - self.butler.put(afwImage.ExposureF(), fringes, self.flat_id) - self.butler.put(transmissionCurve, sensorTransmission, self.detector_id) - self.butler.put(afwImage.ExposureF(), crosstalkSources, self.exposure_id) + self.butler.put(lsst.ip.isr.calibType.IsrCalib(), gainCorrection, self.detector_id) def tearDown(self): del self.repo_path # this removes the temporary directory @@ -261,10 +218,8 @@ def test_runQuantum(self): config.doCalculateStatistics = False # Turn on all optional inputs - config.doAttachTransmissionCurve = True - config.doIlluminationCorrection = True - config.doStrayLight = True - config.doDeferredCharge = True + config.doDeferredCharge = False # There is no CTI calibration + # available for LATISS. config.usePtcReadNoise = True config.doCrosstalk = True config.doBrighterFatter = True @@ -288,22 +243,16 @@ def test_runQuantum(self): "dark": self.detector_id, "flat": self.flat_id, "defects": self.detector_id, - "bfKernel": self.instrument_id, - "newBFKernel": self.detector_id, + "bfKernel": self.detector_id, "ptc": self.detector_id, - "filterTransmission": self.filter_id, "deferredChargeCalib": self.detector_id, - "opticsTransmission": self.instrument_id, - "strayLightData": self.flat_id, - "atmosphereTransmission": self.instrument_id, "crosstalk": self.detector_id, - "illumMaskedImage": self.flat_id, "linearizer": self.detector_id, "fringes": self.flat_id, - "sensorTransmission": self.detector_id, - "crosstalkSources": [self.exposure_id, self.exposure_id], + "gainCorrection": self.detector_id, # outputs "outputExposure": self.exposure_id, + "outputStatistics": self.exposure_id, "exposure": self.exposure_id, }, ) @@ -321,3 +270,16 @@ class ExitMockError(Exception): """A custom exception to catch during a unit test.""" pass + + +class TestMemory(lsst.utils.tests.MemoryTestCase): + pass + + +def setup_module(module): + lsst.utils.tests.init() + + +if __name__ == "__main__": + lsst.utils.tests.init() + unittest.main() From ad3d8941ccd683ee5f66bb0d8cb04f87ef21ff00 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Mon, 26 Jan 2026 12:20:25 -0800 Subject: [PATCH 05/10] Remove non-existent doWrite option --- python/lsst/summit/utils/resources/config/quickLookIsr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/lsst/summit/utils/resources/config/quickLookIsr.py b/python/lsst/summit/utils/resources/config/quickLookIsr.py index 60d63c96..af7ba969 100644 --- a/python/lsst/summit/utils/resources/config/quickLookIsr.py +++ b/python/lsst/summit/utils/resources/config/quickLookIsr.py @@ -1,6 +1,5 @@ # mypy: disable-error-code="name-defined" -config.doWrite = False # this task writes separately, no need for this config.doSaturation = True # saturation very important for roundness measurement in qfm config.brighterFatterMaxIter = 2 # Uncomment this to remove test warning config.doDeferredCharge = False # no calib for this yet From 14bf0e96134637161864676d45766d6ec5e52d32 Mon Sep 17 00:00:00 2001 From: Christopher Waters Date: Wed, 6 May 2026 13:29:23 -0700 Subject: [PATCH 06/10] Remove unneeded import. --- tests/test_quickLook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_quickLook.py b/tests/test_quickLook.py index 90e7ee63..662a768f 100644 --- a/tests/test_quickLook.py +++ b/tests/test_quickLook.py @@ -32,7 +32,7 @@ import lsst.pipe.base as pipeBase import lsst.pipe.base.testUtils import lsst.utils.tests -from lsst.afw.image import TransmissionCurve + from lsst.summit.utils.quickLook import QuickLookIsrTask, QuickLookIsrTaskConfig From 0f4719d25a16127c67b2a9d900b8e6dfa616e64e Mon Sep 17 00:00:00 2001 From: Christopher Waters Date: Wed, 6 May 2026 13:29:46 -0700 Subject: [PATCH 07/10] Use IsrMockLSST data to avoid issues. --- tests/test_quickLook.py | 53 +++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/tests/test_quickLook.py b/tests/test_quickLook.py index 662a768f..1597c960 100644 --- a/tests/test_quickLook.py +++ b/tests/test_quickLook.py @@ -109,7 +109,29 @@ class QuickLookIsrTaskRunQuantumTests(lsst.utils.tests.TestCase): """ def setUp(self): - instrument = "testCam" + # These need to be real, not empty: + self.mockConfig = isrMock.IsrMockLSSTConfig() + self.camera = isrMock.IsrMockLSST(config=self.mockConfig).getCamera() + + self.ccdExposure = isrMock.RawMockLSST(config=self.mockConfig).run() + self.bias = isrMock.BiasMockLSST(config=self.mockConfig).run() + self.dark = isrMock.DarkMockLSST(config=self.mockConfig).run() + self.flat = isrMock.FlatMockLSST(config=self.mockConfig).run() + self.defects = isrMock.DefectMockLSST(config=self.mockConfig).run() + + amps = self.ccdExposure.getDetector().getAmplifiers() + ampNames = [amp.getName() for amp in amps] + self.ptc = ipIsr.PhotonTransferCurveDataset(ampNames=ampNames) # Mock PTC dataset + for amp, gain in self.mockConfig.gainDict.items(): + self.ptc.gain[amp] = 1.0 + self.crosstalk = lsst.ip.isr.crosstalk.CrosstalkCalib(nAmp=len(ampNames)) + self.crosstalk.hasCrosstalk = True + self.cti = isrMock.DeferredChargeMockLSST(config=self.mockConfig).run() + self.mockConfig.doDeferredCharge = False # TODO: DM-54880 + self.bfKernel = isrMock.BfKernelMockLSST(config=self.mockConfig).run() + + # dataId values: + instrument = self.camera.getName() exposureId = 100 visit = 100101 detector = 0 @@ -190,20 +212,25 @@ def setUp(self): # put empty data self.butler = butlerTests.makeTestCollection(self.repo) - self.butler.put(afwImage.ExposureF(), ccdExposure, self.exposure_id) - self.butler.put(afwTestUtils.CameraWrapper().camera, camera, self.instrument_id) - self.butler.put(afwImage.ExposureF(), bias, self.detector_id) - self.butler.put(afwImage.ExposureF(), dark, self.detector_id) - self.butler.put(afwImage.ExposureF(), flat, self.flat_id) - self.butler.put(lsst.ip.isr.Defects(), defects, self.detector_id) + self.butler.put(self.ccdExposure, ccdExposure, self.exposure_id) + self.butler.put(self.camera, camera, self.instrument_id) + self.butler.put(self.bias, bias, self.detector_id) + self.butler.put(self.dark, dark, self.detector_id) + self.butler.put(self.flat, flat, self.flat_id) + self.butler.put(self.defects, defects, self.detector_id) + self.butler.put(self.bfKernel, bfKernel, self.detector_id) + self.butler.put(self.ptc, ptc, self.detector_id) + self.butler.put(self.cti, deferredChargeCalib, self.detector_id) + self.butler.put(self.crosstalk, crosstalk, self.detector_id) + self.butler.put(lsst.ip.isr.linearize.Linearizer(), linearizer, self.detector_id) self.butler.put( - lsst.ip.isr.brighterFatterKernel.BrighterFatterKernel(), bfKernel, self.detector_id + lsst.ip.isr.GainCorrection( + ampNames=ampNames, + gainAdjustments=[1.0 for x in ampNames], + ), + gainCorrection, + self.detector_id ) - self.butler.put(ipIsr.PhotonTransferCurveDataset(), ptc, self.detector_id) - self.butler.put(lsst.ip.isr.calibType.IsrCalib(), deferredChargeCalib, self.detector_id) - self.butler.put(lsst.ip.isr.crosstalk.CrosstalkCalib(), crosstalk, self.detector_id) - self.butler.put(lsst.ip.isr.linearize.Linearizer(), linearizer, self.detector_id) - self.butler.put(lsst.ip.isr.calibType.IsrCalib(), gainCorrection, self.detector_id) def tearDown(self): del self.repo_path # this removes the temporary directory From ca9cffa2d3c02c6a9ca3c88357ed7df771a02bd1 Mon Sep 17 00:00:00 2001 From: Christopher Waters Date: Wed, 6 May 2026 13:43:14 -0700 Subject: [PATCH 08/10] Remove another unneeded import. --- tests/test_quickLook.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_quickLook.py b/tests/test_quickLook.py index 1597c960..e3ea2b55 100644 --- a/tests/test_quickLook.py +++ b/tests/test_quickLook.py @@ -23,7 +23,6 @@ import tempfile import unittest -import lsst.afw.cameraGeom.testUtils as afwTestUtils import lsst.afw.image as afwImage import lsst.daf.butler.tests as butlerTests import lsst.ip.isr as ipIsr From 9fa7fb7273324c2aac81863d536e4f2a9dd558cf Mon Sep 17 00:00:00 2001 From: Christopher Waters Date: Wed, 6 May 2026 13:55:47 -0700 Subject: [PATCH 09/10] Fix linting errors. --- python/lsst/summit/utils/quickLook.py | 10 +--------- tests/test_quickLook.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/python/lsst/summit/utils/quickLook.py b/python/lsst/summit/utils/quickLook.py index 9f3ad19f..7c5a4d6d 100644 --- a/python/lsst/summit/utils/quickLook.py +++ b/python/lsst/summit/utils/quickLook.py @@ -23,7 +23,6 @@ import importlib.resources from typing import Any - import lsst.afw.cameraGeom as camGeom import lsst.afw.image as afwImage import lsst.ip.isr as ipIsr @@ -103,7 +102,7 @@ def __init__(self, isrTask: IsrTaskLSST = IsrTaskLSST, **kwargs: Any): # itself, which is then instantiated later on, in the run() method, # with the dynamically generated config. # import pdb; pdb.set_trace() - if IsrTaskLSST._DefaultName != 'isrLSST': + if IsrTaskLSST._DefaultName != "isrLSST": raise RuntimeError("QuickLookIsrTask should now always use IsrTaskLSST for processing.") self.isrTask = IsrTaskLSST @@ -234,13 +233,6 @@ def run( isrTask = self.isrTask(config=isrConfig) # DM-47959: TODO Add fringe correction to IsrTaskLSST. - # if fringes: - # # Must be run after isrTask is instantiated. - # isrTask.fringe.loadFringes( - # fringes, - # expId=ccdExposure.info.id, - # assembler=isrTask.assembleCcd if isrConfig.doAssembleIsrExposures else None, - # ) result = isrTask.run( ccdExposure, diff --git a/tests/test_quickLook.py b/tests/test_quickLook.py index e3ea2b55..6c99bb93 100644 --- a/tests/test_quickLook.py +++ b/tests/test_quickLook.py @@ -31,7 +31,6 @@ import lsst.pipe.base as pipeBase import lsst.pipe.base.testUtils import lsst.utils.tests - from lsst.summit.utils.quickLook import QuickLookIsrTask, QuickLookIsrTaskConfig @@ -189,7 +188,9 @@ def setUp(self): self.repo, outputExposure, {"instrument", "exposure", "detector"}, "Exposure" ) butlerTests.addDatasetType(self.repo, exposure, {"instrument", "exposure", "detector"}, "Exposure") - butlerTests.addDatasetType(self.repo, outputStatistics, {"instrument", "exposure", "detector"}, "StructuredDataDict") + butlerTests.addDatasetType( + self.repo, outputStatistics, {"instrument", "exposure", "detector"}, "StructuredDataDict" + ) # dataIds self.exposure_id = self.repo.registry.expandDataId( @@ -228,7 +229,7 @@ def setUp(self): gainAdjustments=[1.0 for x in ampNames], ), gainCorrection, - self.detector_id + self.detector_id, ) def tearDown(self): @@ -243,9 +244,10 @@ def test_runQuantum(self): config.qa.doThumbnailFlattened = False config.doCalculateStatistics = False - # Turn on all optional inputs - config.doDeferredCharge = False # There is no CTI calibration - # available for LATISS. + # Turn on all optional inputs, except CTI, as that isn't + # defined for LATISS. + config.doDeferredCharge = False + config.usePtcReadNoise = True config.doCrosstalk = True config.doBrighterFatter = True From db73762969adba61d32c1296398a2a357bfc633f Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Thu, 7 May 2026 05:11:32 -0700 Subject: [PATCH 10/10] Update collection comment and remove (inactive) breakpoint --- python/lsst/summit/utils/quickLook.py | 1 - tests/test_bestEffortIsr.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/python/lsst/summit/utils/quickLook.py b/python/lsst/summit/utils/quickLook.py index 7c5a4d6d..e47b6b1d 100644 --- a/python/lsst/summit/utils/quickLook.py +++ b/python/lsst/summit/utils/quickLook.py @@ -101,7 +101,6 @@ def __init__(self, isrTask: IsrTaskLSST = IsrTaskLSST, **kwargs: Any): # Note that this is not an instance of the IsrTask class, but the class # itself, which is then instantiated later on, in the run() method, # with the dynamically generated config. - # import pdb; pdb.set_trace() if IsrTaskLSST._DefaultName != "isrLSST": raise RuntimeError("QuickLookIsrTask should now always use IsrTaskLSST for processing.") self.isrTask = IsrTaskLSST diff --git a/tests/test_bestEffortIsr.py b/tests/test_bestEffortIsr.py index 2bd2cad4..f15accf0 100644 --- a/tests/test_bestEffortIsr.py +++ b/tests/test_bestEffortIsr.py @@ -37,9 +37,7 @@ def setUpClass(cls): raise unittest.SkipTest("Skipping tests that require the LATISS butler repo.") # chosen as this is available in the following locations - collections: - # NCSA - LATISS/raw/all - # TTS - LATISS-test-data-tts - # summit - LATISS_test_data + # main@USDF - LATISS/raw/all cls.dataId = {"day_obs": 20251021, "seq_num": 443, "detector": 0} def test_getExposure(self):