From 41a9f28de64b0a3f283fd1b9217e9ab750e632cf Mon Sep 17 00:00:00 2001 From: plazas Date: Wed, 17 Jan 2024 10:28:56 -0800 Subject: [PATCH 1/4] Rename keys for consistency in verification quantities and boolean --- python/lsst/cp/verify/verifyPtc.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/lsst/cp/verify/verifyPtc.py b/python/lsst/cp/verify/verifyPtc.py index e7ef784..7ba8c57 100644 --- a/python/lsst/cp/verify/verifyPtc.py +++ b/python/lsst/cp/verify/verifyPtc.py @@ -177,8 +177,8 @@ def verify(self, calib, statisticsDict, camera=None): # The fractional relative difference between the fitted PTC and the # nominal amplifier gain and readout noise values should be less # than a certain threshold (default: 5%). - verify['GAIN'] = bool(diffGain < self.config.gainThreshold) - verify['NOISE'] = bool(diffNoise < self.config.noiseThreshold) + verify['PTC_GAIN'] = bool(diffGain < self.config.gainThreshold) + verify['PTC_NOISE'] = bool(diffNoise < self.config.noiseThreshold) # DMTN-101: 16.3 # Check that the measured PTC turnoff is at least greater than the @@ -200,11 +200,11 @@ def verify(self, calib, statisticsDict, camera=None): if detVendor == 'ITL': a00Max = self.config.a00MaxITL a00Min = self.config.a00MinITL - verify['BFE_A00'] = bool(a00 > a00Min and a00 < a00Max) + verify['PTC_BFE_A00'] = bool(a00 > a00Min and a00 < a00Max) elif detVendor == 'E2V': a00Max = self.config.a00MaxE2V a00Min = self.config.a00MinE2V - verify['BFE_A00'] = bool(a00 > a00Min and a00 < a00Max) + verify['PTC_BFE_A00'] = bool(a00 > a00Min and a00 < a00Max) else: raise RuntimeError(f"Detector type {detVendor} not one of 'ITL' or 'E2V'") # Overall success among all tests for this amp. @@ -215,7 +215,7 @@ def verify(self, calib, statisticsDict, camera=None): verifyStats[ampName] = verify # Loop over amps to make a detector summary. - verifyDetStats = {'GAIN': [], 'NOISE': [], 'PTC_TURNOFF': [], 'BFE_A00': []} + verifyDetStats = {'PTC_GAIN': [], 'PTC_NOISE': [], 'PTC_TURNOFF': [], 'PTC_BFE_A00': []} for amp in verifyStats: for testName in verifyStats[amp]: if testName == 'SUCCESS': @@ -223,8 +223,8 @@ def verify(self, calib, statisticsDict, camera=None): verifyDetStats[testName].append(verifyStats[amp][testName]) # If ptc model did not fit for a00 (e.g., POLYNOMIAL) - if not len(verifyDetStats['BFE_A00']): - verifyDetStats.pop('BFE_A00') + if not len(verifyDetStats['PTC_BFE_A00']): + verifyDetStats.pop('PTC_BFE_A00') # VerifyDetStatsFinal has final boolean test over all amps verifyDetStatsFinal = {} From 3062077924f8026789fbb4d7445db847ebbd70b7 Mon Sep 17 00:00:00 2001 From: plazas Date: Wed, 17 Jan 2024 10:43:24 -0800 Subject: [PATCH 2/4] Add row means var and max raw means --- python/lsst/cp/verify/verifyPtc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/lsst/cp/verify/verifyPtc.py b/python/lsst/cp/verify/verifyPtc.py index 7ba8c57..b61daff 100644 --- a/python/lsst/cp/verify/verifyPtc.py +++ b/python/lsst/cp/verify/verifyPtc.py @@ -129,6 +129,8 @@ def amplifierStatistics(self, inputCalib, camera=None): outputStatistics[ampName]['AMP_NOISE'] = amp.getReadNoise() outputStatistics[ampName]['PTC_TURNOFF'] = inputCalib.ptcTurnoff[ampName] outputStatistics[ampName]['PTC_FIT_TYPE'] = ptcFitType + outputStatistics[ampName]['PTC_ROW_MEAN_VARIANCE'] = inputCalib.rowMeanVariance[ampName] + outputStatistics[ampName]['PTC_MAX_RAW_MEANS'] = np.max(inputCalib.rawMeans[ampName]) if ptcFitType == 'EXPAPPROXIMATION': outputStatistics[ampName]['PTC_BFE_A00'] = float(inputCalib.ptcFitPars[ampName][0]) if ptcFitType == 'FULLCOVARIANCE': From 3c9835f1156007cd5f6f624bfb7091504f81b4b0 Mon Sep 17 00:00:00 2001 From: plazas Date: Fri, 19 Jan 2024 12:40:21 -0800 Subject: [PATCH 3/4] Add slope calculation Fix serialization problems --- python/lsst/cp/verify/verifyPtc.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/python/lsst/cp/verify/verifyPtc.py b/python/lsst/cp/verify/verifyPtc.py index b61daff..636cf53 100644 --- a/python/lsst/cp/verify/verifyPtc.py +++ b/python/lsst/cp/verify/verifyPtc.py @@ -123,18 +123,31 @@ def amplifierStatistics(self, inputCalib, camera=None): outputStatistics = {amp.getName(): {} for amp in detector} for amp in detector: ampName = amp.getName() - outputStatistics[ampName]['PTC_GAIN'] = inputCalib.gain[ampName] + calibGain = inputCalib.gain[ampName] + outputStatistics[ampName]['PTC_GAIN'] = calibGain outputStatistics[ampName]['AMP_GAIN'] = amp.getGain() outputStatistics[ampName]['PTC_NOISE'] = inputCalib.noise[ampName] outputStatistics[ampName]['AMP_NOISE'] = amp.getReadNoise() outputStatistics[ampName]['PTC_TURNOFF'] = inputCalib.ptcTurnoff[ampName] outputStatistics[ampName]['PTC_FIT_TYPE'] = ptcFitType - outputStatistics[ampName]['PTC_ROW_MEAN_VARIANCE'] = inputCalib.rowMeanVariance[ampName] - outputStatistics[ampName]['PTC_MAX_RAW_MEANS'] = np.max(inputCalib.rawMeans[ampName]) + outputStatistics[ampName]['PTC_ROW_MEAN_VARIANCE'] = inputCalib.rowMeanVariance[ampName].tolist() + outputStatistics[ampName]['PTC_MAX_RAW_MEANS'] = float(np.max(inputCalib.rawMeans[ampName])) if ptcFitType == 'EXPAPPROXIMATION': outputStatistics[ampName]['PTC_BFE_A00'] = float(inputCalib.ptcFitPars[ampName][0]) if ptcFitType == 'FULLCOVARIANCE': outputStatistics[ampName]['PTC_BFE_A00'] = float(inputCalib.aMatrix[ampName][0][0]) + + # Test from eo_pipe: github.com/lsst-camera-dh/eo-pipe; + # ptcPlotTask.py + # Slope of [variance of means of rows](electrons^2) + # vs [2*signal(electrons)/numCols] + numCols = amp.getBBox().width + mask = inputCalib.expIdMask[ampName] + rowMeanVar = inputCalib.rowMeanVariance[ampName][mask]*calibGain**2 + signal = inputCalib.rawMeans[ampName][mask]*calibGain + slope = sum(rowMeanVar) / sum(2.*signal/numCols) + outputStatistics[ampName]['ROW_MEAN_VARIANCE_SLOPE'] = float(slope) + return outputStatistics def verify(self, calib, statisticsDict, camera=None): @@ -172,7 +185,9 @@ def verify(self, calib, statisticsDict, camera=None): for amp in detector: verify = {} ampName = amp.getName() - diffGain = (np.abs(calib.gain[ampName] - amp.getGain()) / amp.getGain())*100 + calibGain = calib.gain[ampName] + + diffGain = (np.abs(calibGain - amp.getGain()) / amp.getGain())*100 diffNoise = (np.abs(calib.noise[ampName] - amp.getReadNoise()) / amp.getReadNoise())*100 # DMTN-101: 16.1 and 16.2 @@ -186,7 +201,7 @@ def verify(self, calib, statisticsDict, camera=None): # Check that the measured PTC turnoff is at least greater than the # full-well requirement of 90k e-. turnoffCut = self.config.turnoffThreshold - verify['PTC_TURNOFF'] = bool(calib.ptcTurnoff[ampName]*calib.gain[ampName] > turnoffCut) + verify['PTC_TURNOFF'] = bool(calib.ptcTurnoff[ampName]*calibGain > turnoffCut) # DMTN-101: 16.4 # Check the a00 value (brighter-fatter effect). # This is a purely electrostatic parameter that should not change @@ -209,6 +224,7 @@ def verify(self, calib, statisticsDict, camera=None): verify['PTC_BFE_A00'] = bool(a00 > a00Min and a00 < a00Max) else: raise RuntimeError(f"Detector type {detVendor} not one of 'ITL' or 'E2V'") + # Overall success among all tests for this amp. verify['SUCCESS'] = bool(np.all(list(verify.values()))) if verify['SUCCESS'] is False: From 77fc50009e4a6d7b35524764f5e72da6f6d4ec94 Mon Sep 17 00:00:00 2001 From: plazas Date: Mon, 22 Jan 2024 07:16:19 -0800 Subject: [PATCH 4/4] Handle dividion by zero in slope --- python/lsst/cp/verify/verifyPtc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/lsst/cp/verify/verifyPtc.py b/python/lsst/cp/verify/verifyPtc.py index 636cf53..e149eb0 100644 --- a/python/lsst/cp/verify/verifyPtc.py +++ b/python/lsst/cp/verify/verifyPtc.py @@ -145,7 +145,10 @@ def amplifierStatistics(self, inputCalib, camera=None): mask = inputCalib.expIdMask[ampName] rowMeanVar = inputCalib.rowMeanVariance[ampName][mask]*calibGain**2 signal = inputCalib.rawMeans[ampName][mask]*calibGain - slope = sum(rowMeanVar) / sum(2.*signal/numCols) + try: + slope = sum(rowMeanVar) / sum(2.*signal/numCols) + except ZeroDivisionError: + slope = np.nan outputStatistics[ampName]['ROW_MEAN_VARIANCE_SLOPE'] = float(slope) return outputStatistics