Skip to content

Commit

Permalink
Add merged detector results for calibs
Browse files Browse the repository at this point in the history
  • Loading branch information
@Alex-Broughton
Alex-Broughton committed Jan 6, 2025
1 parent f924e43 commit 7945d88
Show file tree
Hide file tree
Showing 5 changed files with 216 additions and 9 deletions.
20 changes: 16 additions & 4 deletions pipelines/_ingredients/verifyFlat.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ imports:
- location: $ANALYSIS_TOOLS_DIR/pipelines/cpCore.yaml
include:
- analyzeFlatCore
- analyzeFlatDetCore
- analyzeFlatDetMergeCore
tasks:
verifyFlatIsr:
class: lsst.ip.isr.IsrTask
Expand All @@ -22,6 +22,16 @@ tasks:
connections.outputStats: "verifyFlatDetStats"
connections.outputResults: "verifyFlatDetResults"
useIsrStatistics: true
verifyFlatDetMerge:
class: lsst.cp.verify.CpVerifyDetMergeByFilterTask
config:
connections.inputStats: "verifyFlatDetStats"
connections.inputResults: "verifyFlatDetResults"
connections.outputStats: "verifyFlatDetMergeStats"
connections.outputResults: "verifyFlatDetMergeResults"
hasInputResults: true
hasMatrixCatalog: false
stageName: "flat"
verifyFlatExp:
class: lsst.cp.verify.CpVerifyFlatExpMergeTask
config:
Expand All @@ -46,14 +56,16 @@ tasks:
config:
connections.data: "verifyFlatResults"
connections.outputName: "verifyFlatAnalysis"
analyzeFlatDetCore:
analyzeFlatDetMergeCore:
class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
config:
connections.data: "verifyFlatDetResults"
connections.outputName: "verifyFlatDetAnalysis"
connections.data: "verifyFlatDetMergeResults"
connections.outputName: "verifyFlatDetMergeAnalysis"
contracts:
- verifyFlatDet.connections.inputExp == verifyFlatIsr.connections.outputExposure
- verifyFlatExp.connections.inputStats == verifyFlatDet.connections.outputStats
- verifyFlatExp.connections.inputResults == verifyFlatDet.connections.outputResults
- verifyFlat.connections.inputStats == verifyFlatExp.connections.outputStats
- verifyFlat.connections.inputResults == verifyFlatExp.connections.outputResults
- analyzeFlatCore.connections.data == verifyFlat.connections.outputResults
- analyzeFlatDetMergeCore.connections.data == verifyFlatDetMerge.connections.outputResults
22 changes: 17 additions & 5 deletions pipelines/_ingredients/verifyFlatLSST.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ imports:
- location: $ANALYSIS_TOOLS_DIR/pipelines/cpCore.yaml
include:
- analyzeFlatCore
- analyzeFlatDetCore
- analyzeFlatDetMergeCore
tasks:
verifyFlatIsr:
class: lsst.ip.isr.IsrTaskLSST
Expand Down Expand Up @@ -35,6 +35,16 @@ tasks:
connections.outputStats: "verifyFlatDetStats"
connections.outputResults: "verifyFlatDetResults"
useIsrStatistics: true
verifyFlatDetMerge:
class: lsst.cp.verify.CpVerifyDetMergeByFilterTask
config:
connections.inputStats: "verifyFlatDetStats"
connections.inputResults: "verifyFlatDetResults"
connections.outputStats: "verifyFlatDetMergeStats"
connections.outputResults: "verifyFlatDetMergeResults"
hasInputResults: true
hasMatrixCatalog: false
stageName: "flat"
verifyFlatExp:
class: lsst.cp.verify.CpVerifyFlatExpMergeTask
config:
Expand All @@ -46,7 +56,7 @@ tasks:
hasMatrixCatalog: false
stageName: "flat"
verifyFlat:
class: lsst.cp.verify.CpVerifyRunMergeByFilterTaskByFilter
class: lsst.cp.verify.CpVerifyRunMergeByFilterTask
config:
connections.inputStats: "verifyFlatExpStats"
connections.inputResults: "verifyFlatExpResults"
Expand All @@ -59,14 +69,16 @@ tasks:
config:
connections.data: "verifyFlatResults"
connections.outputName: "verifyFlatAnalysis"
analyzeFlatDetCore:
analyzeFlatDetMergeCore:
class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
config:
connections.data: "verifyFlatDetResults"
connections.outputName: "verifyFlatDetAnalysis"
connections.data: "verifyFlatDetMergeResults"
connections.outputName: "verifyFlatDetMergeAnalysis"
contracts:
- verifyFlatDet.connections.inputExp == verifyFlatIsr.connections.outputExposure
- verifyFlatExp.connections.inputStats == verifyFlatDet.connections.outputStats
- verifyFlatExp.connections.inputResults == verifyFlatDet.connections.outputResults
- verifyFlat.connections.inputStats == verifyFlatExp.connections.outputStats
- verifyFlat.connections.inputResults == verifyFlatExp.connections.outputResults
- analyzeFlatCore.connections.data == verifyFlat.connections.outputResults
- analyzeFlatDetMergeCore.connections.data == verifyFlatDetMerge.connections.outputResults
12 changes: 12 additions & 0 deletions python/lsst/cp/verify/configs/report.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,18 @@ stages:
description: "Processing configuration for the combined flat construction."
verifyFlatApply_config:
description: "Processing configuration for combined flat verification."
cpFlatCore_flatMeanPerAmp_FocalPlaneGeometryPlot:
description: "Focal plane mosaic of the combined flat means per amp."
cpFlatCore_flatNoisePerAmp_FocalPlaneGeometryPlot:
description: "Focal plane mosaic of the combined flat noise per amp."
cpFlatDetMergeCore_flatTestsByDate_GridPlot:
description: "Verfiy flat noise boolean by MJD per amp."
cpFlatDetMergeCore_flatMeansByDate_GridPlot:
description: "Flat mean by MJD per amp."
cpFlatDetMergeCore_flatNoiseByDate_GridPlot:
description: "Flat noise by MJD per amp."
cpFlatDetMergeCore_flatNoiseByMean_GridPlot:
description: "Verify flat noise by mean per amp."
defects:
defects:
description: "The combined defects calibration."
Expand Down
168 changes: 168 additions & 0 deletions python/lsst/cp/verify/mergeResults.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@

__all__ = ['CpVerifyExpMergeConfig', 'CpVerifyExpMergeTask',
'CpVerifyRunMergeConfig', 'CpVerifyRunMergeTask',
'CpVerifyDetMergeConfig', 'CpVerifyDetMergeTask',
'CpVerifyDetMergeByFilterConfig', 'CpVerifyDetMergeByFilterTask',
'CpVerifyExpMergeByFilterConfig', 'CpVerifyExpMergeByFilterTask',
'CpVerifyRunMergeByFilterConfig', 'CpVerifyRunMergeByFilterTask',
'CpVerifyVisitExpMergeConfig', 'CpVerifyVisitExpMergeTask',
Expand Down Expand Up @@ -477,6 +479,172 @@ class CpVerifyRunMergeTask(CpVerifyExpMergeTask):
# End ExpMerge/RunMerge


class CpVerifyDetMergeConnections(pipeBase.PipelineTaskConnections,
dimensions={"instrument", "detector"},
defaultTemplates={}):
inputStats = cT.Input(
name="detectorStats",
doc="Input statistics to merge.",
storageClass="StructuredDataDict",
dimensions=["instrument", "exposure", "detector"],
multiple=True,
)
inputResults = cT.Input(
name="detectorResults",
doc="Input results to merge.",
storageClass="ArrowAstropy",
dimensions=["instrument", "exposure", "detector"],
multiple=True,
)
inputMatrix = cT.Input(
name="detectorMatrix",
doc="Input matrix to merge.",
storageClass="ArrowAstropy",
dimensions=["instrument", "exposure", "detector"],
multiple=True,
)
camera = cT.PrerequisiteInput(
name="camera",
storageClass="Camera",
doc="Input camera.",
dimensions=["instrument", ],
isCalibration=True,
)

outputStats = cT.Output(
name="exposureStats",
doc="Output statistics.",
storageClass="StructuredDataDict",
dimensions=["instrument", "detector"],
)
outputResults = cT.Output(
name="exposureResults",
doc="Output results.",
storageClass="ArrowAstropy",
dimensions=["instrument", "detector"],
)
outputMatrix = cT.Output(
name="exposureMatrix",
doc="Output matrix.",
storageClass="ArrowAstropy",
dimensions=["instrument", "detector"],
)

def __init__(self, *, config=None):
super().__init__(config=config)

if not self.config.hasMatrixCatalog:
self.inputs.remove("inputMatrix")
self.outputs.remove("outputMatrix")
if not self.config.hasInputResults:
self.inputs.remove("inputResults")
if self.config.doDropStats:
self.outputs.remove("outputStats")


class CpVerifyDetMergeConfig(CpVerifyExpMergeConfig,
pipelineConnections=CpVerifyDetMergeConnections):
"""Configuration paramters for exposure stats merging.
"""
mergeDimension = pexConfig.Field(
dtype=str,
doc="Dimension name for this input.",
default="detector",
)


class CpVerifyDetMergeTask(CpVerifyExpMergeTask):
"""Merge statistics from detectors together.
"""
ConfigClass = CpVerifyDetMergeConfig
_DefaultName = 'cpVerifyDetMerge'

pass


class CpVerifyDetMergeByFilterConnections(pipeBase.PipelineTaskConnections,
dimensions={"instrument", "detector", "physical_filter"},
defaultTemplates={}):
inputStats = cT.Input(
name="detectorStats",
doc="Input statistics to merge.",
storageClass="StructuredDataDict",
dimensions=["instrument", "exposure", "detector", "physical_filter"],
multiple=True,
)
inputResults = cT.Input(
name="detectorResults",
doc="Input results to merge.",
storageClass="ArrowAstropy",
dimensions=["instrument", "exposure", "detector", "physical_filter"],
multiple=True,
)
inputMatrix = cT.Input(
name="detectorMatrix",
doc="Input matrix to merge.",
storageClass="ArrowAstropy",
dimensions=["instrument", "exposure", "detector", "physical_filter"],
multiple=True,
)
camera = cT.PrerequisiteInput(
name="camera",
storageClass="Camera",
doc="Input camera.",
dimensions=["instrument", ],
isCalibration=True,
)

outputStats = cT.Output(
name="exposureStats",
doc="Output statistics.",
storageClass="StructuredDataDict",
dimensions=["instrument", "detector", "physical_filter"],
)
outputResults = cT.Output(
name="exposureResults",
doc="Output results.",
storageClass="ArrowAstropy",
dimensions=["instrument", "detector", "physical_filter"],
)
outputMatrix = cT.Output(
name="exposureMatrix",
doc="Output matrix.",
storageClass="ArrowAstropy",
dimensions=["instrument", "detector", "physical_filter"],
)

def __init__(self, *, config=None):
super().__init__(config=config)

if not self.config.hasMatrixCatalog:
self.inputs.remove("inputMatrix")
self.outputs.remove("outputMatrix")
if not self.config.hasInputResults:
self.inputs.remove("inputResults")
if self.config.doDropStats:
self.outputs.remove("outputStats")


class CpVerifyDetMergeByFilterConfig(CpVerifyExpMergeConfig,
pipelineConnections=CpVerifyDetMergeByFilterConnections):
"""Configuration paramters for exposure stats merging.
"""
mergeDimension = pexConfig.Field(
dtype=str,
doc="Dimension name for this input.",
default="detector",
)


class CpVerifyDetMergeByFilterTask(CpVerifyExpMergeTask):
"""Merge statistics from detectors together.
"""
ConfigClass = CpVerifyDetMergeByFilterConfig
_DefaultName = 'cpVerifyDetMergeByFilter'

pass


# Begin ExpMergeByFilter/RunMergeByFilter
class CpVerifyExpMergeByFilterConnections(pipeBase.PipelineTaskConnections,
dimensions={"instrument", "exposure", "physical_filter"},
Expand Down
3 changes: 3 additions & 0 deletions python/lsst_cp_verify.dist-info/METADATA
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Metadata-Version: 1.0
Name: lsst-cp-verify
Version: w.2025.01-3-g990dc87

0 comments on commit 7945d88

Please sign in to comment.