From 0dfe9e515b4e06b4e23c4621c08f4b0701262d75 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Thu, 24 Oct 2019 16:08:49 -0700 Subject: [PATCH 1/2] add code to skip bad frames --- .../v0/producer_BOT_acq_recovery.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py b/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py index 76ac4c9..9d72dd6 100755 --- a/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py +++ b/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py @@ -23,17 +23,30 @@ # lcatr.cfg. acq_run = os.environ['LCATR_ACQ_RUN'] +# Get any bad frames from a text file specified in lcatr.cfg. +try: + with open(os.environ['LCATR_BAD_FRAME_LIST'], 'r') as fd: + bad_frames = [_.strip() for _ in fd] +except KeyError: + bad_frames = [] + staging_dir = os.path.join(os.environ['LCATR_STAGE_ROOT'], siteUtils.getUnitType(), siteUtils.getUnitId()) outdir = '.' acqs_dir = os.path.join(staging_dir, acq_run, 'BOT_acq', 'v0') job_id_dirs = sorted(glob.glob(os.path.join(acqs_dir, '[0-9]*')), reverse=True) for job_id_dir in job_id_dirs: - items = glob.glob(os.path.join(job_id_dir, '*')) - for item in items: - dest = os.path.join(outdir, os.path.basename(item)) - if ((os.path.islink(item) or item.endswith('.cfg')) - and not os.path.lexists(dest)): - shutil.copyfile(item, dest, follow_symlinks=False) + frame_dirs = glob.glob(os.path.join(job_id_dir, '*')) + for frame_dir in frame_dirs: + if bad_frames: + fits_images = [] + for bad_frame in bad_frames: + pattern = os.path.join(frame_dir, f'*{bad_frame}*.fits') + fits_images.extend(glob.glob(pattern)) + dest = os.path.join(outdir, os.path.basename(frame_dir)) + if ((os.path.islink(frame_dir) or frame_dir.endswith('.cfg')) + and not os.path.lexists(dest) + and not fits_images): + shutil.copyfile(frame_dir, dest, follow_symlinks=False) pathlib.Path('PRESERVE_SYMLINKS').touch() From 50237721e567512998ff608cfdf868d6577af847 Mon Sep 17 00:00:00 2001 From: James Chiang Date: Thu, 24 Oct 2019 18:08:50 -0700 Subject: [PATCH 2/2] it's more efficient to delete the bad symlinks after making all of them first --- .../v0/producer_BOT_acq_recovery.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py b/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py index 9d72dd6..03be25f 100755 --- a/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py +++ b/harnessed_jobs/BOT_acq_recovery/v0/producer_BOT_acq_recovery.py @@ -38,15 +38,19 @@ for job_id_dir in job_id_dirs: frame_dirs = glob.glob(os.path.join(job_id_dir, '*')) for frame_dir in frame_dirs: - if bad_frames: - fits_images = [] - for bad_frame in bad_frames: - pattern = os.path.join(frame_dir, f'*{bad_frame}*.fits') - fits_images.extend(glob.glob(pattern)) dest = os.path.join(outdir, os.path.basename(frame_dir)) if ((os.path.islink(frame_dir) or frame_dir.endswith('.cfg')) - and not os.path.lexists(dest) - and not fits_images): + and not os.path.lexists(dest)): shutil.copyfile(frame_dir, dest, follow_symlinks=False) +# Delete any folders with bad data +for bad_frame in bad_frames: + fits_images = [] + for bad_frame in bad_frames: + pattern = os.path.join('*', f'*{bad_frame}*.fits') + fits_images.extend(glob.glob(pattern)) +bad_symlinks = set([os.path.dirname(_) for _ in fits_images]) +for bad_symlink in bad_symlinks: + os.remove(bad_symlink) + pathlib.Path('PRESERVE_SYMLINKS').touch()